llm_cost_tracker 0.8.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +108 -0
- data/README.md +12 -5
- data/app/assets/llm_cost_tracker/application.css +65 -5
- data/app/controllers/llm_cost_tracker/application_controller.rb +25 -33
- data/app/controllers/llm_cost_tracker/assets_controller.rb +1 -1
- data/app/controllers/llm_cost_tracker/calls_controller.rb +5 -7
- data/app/controllers/llm_cost_tracker/data_quality_controller.rb +4 -0
- data/app/controllers/llm_cost_tracker/reconciliation_controller.rb +106 -0
- data/app/controllers/llm_cost_tracker/tags_controller.rb +15 -1
- data/app/helpers/llm_cost_tracker/application_helper.rb +10 -0
- data/app/helpers/llm_cost_tracker/inline_style_helper.rb +28 -0
- data/app/helpers/llm_cost_tracker/reconciliation_helper.rb +13 -0
- data/app/helpers/llm_cost_tracker/token_usage_helper.rb +5 -1
- data/app/models/llm_cost_tracker/call.rb +0 -3
- data/app/models/llm_cost_tracker/call_line_item.rb +1 -5
- data/app/models/llm_cost_tracker/call_rollup.rb +0 -3
- data/app/models/llm_cost_tracker/call_tag.rb +0 -4
- data/app/models/llm_cost_tracker/ingestion/inbox_entry.rb +0 -4
- data/app/models/llm_cost_tracker/ingestion/lease.rb +0 -3
- data/app/models/llm_cost_tracker/provider_invoice.rb +7 -3
- data/app/models/llm_cost_tracker/provider_invoice_import.rb +24 -0
- data/app/services/llm_cost_tracker/dashboard/data_quality.rb +33 -4
- data/app/services/llm_cost_tracker/dashboard/filter.rb +6 -4
- data/app/views/layouts/llm_cost_tracker/application.html.erb +6 -1
- data/app/views/llm_cost_tracker/calls/show.html.erb +25 -40
- data/app/views/llm_cost_tracker/dashboard/index.html.erb +9 -9
- data/app/views/llm_cost_tracker/data_quality/index.html.erb +91 -52
- data/app/views/llm_cost_tracker/reconciliation/index.html.erb +183 -0
- data/app/views/llm_cost_tracker/shared/_bar.html.erb +1 -1
- data/app/views/llm_cost_tracker/shared/_filters.html.erb +3 -0
- data/app/views/llm_cost_tracker/shared/_metric_stack.html.erb +1 -1
- data/app/views/llm_cost_tracker/tags/show.html.erb +60 -0
- data/config/routes.rb +3 -2
- data/lib/llm_cost_tracker/billing/components.rb +45 -3
- data/lib/llm_cost_tracker/billing/components.yml +71 -0
- data/lib/llm_cost_tracker/billing/line_item.rb +1 -1
- data/lib/llm_cost_tracker/budget.rb +4 -2
- data/lib/llm_cost_tracker/capture/stream_collector.rb +93 -20
- data/lib/llm_cost_tracker/capture/stream_tracker.rb +40 -5
- data/lib/llm_cost_tracker/configuration.rb +53 -1
- data/lib/llm_cost_tracker/dashboard_setup_state.rb +109 -0
- data/lib/llm_cost_tracker/doctor/cost_drift_check.rb +2 -0
- data/lib/llm_cost_tracker/doctor/ingestion_check.rb +26 -0
- data/lib/llm_cost_tracker/doctor/invoice_reconciliation_check.rb +164 -0
- data/lib/llm_cost_tracker/doctor/schema_check.rb +5 -2
- data/lib/llm_cost_tracker/doctor.rb +72 -3
- data/lib/llm_cost_tracker/engine.rb +9 -0
- data/lib/llm_cost_tracker/event.rb +1 -1
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/call_rollups_generator.rb +43 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/durable_ingestion_generator.rb +43 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/install_generator.rb +13 -3
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/reconciliation_generator.rb +34 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/create_llm_cost_tracker_call_rollups.rb.erb +15 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/create_llm_cost_tracker_calls.rb.erb +5 -58
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/create_llm_cost_tracker_durable_ingestion.rb.erb +29 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/create_llm_cost_tracker_reconciliation.rb.erb +55 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/initializer.rb.erb +28 -25
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_call_rollups_provider.rb.erb +20 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_call_tags_key_value_index.rb.erb +32 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_image_tokens.rb.erb +18 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/upgrade_call_rollups_provider_generator.rb +38 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/upgrade_call_tags_key_value_index_generator.rb +30 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/upgrade_image_tokens_generator.rb +29 -0
- data/lib/llm_cost_tracker/ingestion/inbox.rb +0 -1
- data/lib/llm_cost_tracker/ingestion/inline.rb +22 -0
- data/lib/llm_cost_tracker/ingestion/worker.rb +10 -2
- data/lib/llm_cost_tracker/ingestion.rb +48 -10
- data/lib/llm_cost_tracker/integrations/anthropic.rb +24 -5
- data/lib/llm_cost_tracker/integrations/base.rb +22 -5
- data/lib/llm_cost_tracker/integrations/openai.rb +300 -66
- data/lib/llm_cost_tracker/integrations/ruby_llm.rb +105 -6
- data/lib/llm_cost_tracker/integrations.rb +19 -1
- data/lib/llm_cost_tracker/ledger/period/totals.rb +21 -5
- data/lib/llm_cost_tracker/ledger/rollups.rb +24 -10
- data/lib/llm_cost_tracker/ledger/schema/call_line_items.rb +30 -1
- data/lib/llm_cost_tracker/ledger/schema/call_rollups.rb +3 -3
- data/lib/llm_cost_tracker/ledger/schema/call_tags.rb +17 -2
- data/lib/llm_cost_tracker/ledger/schema/calls.rb +2 -0
- data/lib/llm_cost_tracker/ledger/schema/ingestion_inbox_entries.rb +47 -0
- data/lib/llm_cost_tracker/ledger/schema/ingestion_leases.rb +42 -0
- data/lib/llm_cost_tracker/ledger/schema/provider_invoice_imports.rb +46 -0
- data/lib/llm_cost_tracker/ledger/schema/provider_invoices.rb +2 -2
- data/lib/llm_cost_tracker/ledger/store.rb +14 -14
- data/lib/llm_cost_tracker/ledger/tags/encoding.rb +37 -0
- data/lib/llm_cost_tracker/ledger/tags/query.rb +2 -1
- data/lib/llm_cost_tracker/ledger.rb +2 -1
- data/lib/llm_cost_tracker/masking.rb +39 -0
- data/lib/llm_cost_tracker/middleware/faraday.rb +88 -29
- data/lib/llm_cost_tracker/parsers/anthropic.rb +22 -7
- data/lib/llm_cost_tracker/parsers/base.rb +5 -1
- data/lib/llm_cost_tracker/parsers/gemini.rb +4 -0
- data/lib/llm_cost_tracker/parsers/openai.rb +16 -2
- data/lib/llm_cost_tracker/parsers/openai_compatible.rb +5 -1
- data/lib/llm_cost_tracker/parsers/openai_service_charges.rb +49 -10
- data/lib/llm_cost_tracker/parsers/openai_usage.rb +124 -53
- data/lib/llm_cost_tracker/prices.json +110 -19
- data/lib/llm_cost_tracker/pricing/effective_prices.rb +5 -36
- data/lib/llm_cost_tracker/pricing/lookup.rb +36 -3
- data/lib/llm_cost_tracker/pricing/mode.rb +76 -0
- data/lib/llm_cost_tracker/pricing/registry.rb +3 -1
- data/lib/llm_cost_tracker/pricing/service_charges.rb +9 -3
- data/lib/llm_cost_tracker/pricing/sync/registry_writer.rb +50 -1
- data/lib/llm_cost_tracker/pricing/sync.rb +3 -1
- data/lib/llm_cost_tracker/pricing.rb +47 -19
- data/lib/llm_cost_tracker/railtie.rb +6 -0
- data/lib/llm_cost_tracker/reconcile_tasks.rb +134 -0
- data/lib/llm_cost_tracker/reconciliation/diff.rb +428 -0
- data/lib/llm_cost_tracker/reconciliation/diff_result.rb +48 -0
- data/lib/llm_cost_tracker/reconciliation/import_result.rb +19 -0
- data/lib/llm_cost_tracker/reconciliation/importer.rb +253 -0
- data/lib/llm_cost_tracker/reconciliation/sources/anthropic_usage.rb +171 -0
- data/lib/llm_cost_tracker/reconciliation/sources/fingerprint.rb +20 -0
- data/lib/llm_cost_tracker/reconciliation/sources/openai_usage.rb +142 -0
- data/lib/llm_cost_tracker/reconciliation.rb +118 -0
- data/lib/llm_cost_tracker/report/data.rb +4 -1
- data/lib/llm_cost_tracker/retention.rb +15 -2
- data/lib/llm_cost_tracker/tags/context.rb +3 -4
- data/lib/llm_cost_tracker/tags/sanitizer.rb +60 -4
- data/lib/llm_cost_tracker/token_usage.rb +10 -2
- data/lib/llm_cost_tracker/tracker.rb +45 -18
- data/lib/llm_cost_tracker/version.rb +1 -1
- data/lib/llm_cost_tracker.rb +9 -0
- data/lib/tasks/llm_cost_tracker.rake +25 -2
- metadata +36 -1
|
@@ -13,6 +13,8 @@ require_relative "doctor/pricing_snapshot_drift_check"
|
|
|
13
13
|
|
|
14
14
|
module LlmCostTracker
|
|
15
15
|
class Doctor
|
|
16
|
+
autoload :InvoiceReconciliationCheck, "llm_cost_tracker/doctor/invoice_reconciliation_check"
|
|
17
|
+
|
|
16
18
|
class << self
|
|
17
19
|
def call
|
|
18
20
|
new.checks
|
|
@@ -41,10 +43,10 @@ module LlmCostTracker
|
|
|
41
43
|
table: "llm_cost_tracker_call_line_items").call,
|
|
42
44
|
SchemaCheck.new(name: "call tags", schema: Ledger::Schema::CallTags,
|
|
43
45
|
table: "llm_cost_tracker_call_tags").call,
|
|
44
|
-
|
|
45
|
-
table: "llm_cost_tracker_provider_invoices").call,
|
|
46
|
+
*reconciliation_schema_checks,
|
|
46
47
|
CostDriftCheck.new.call,
|
|
47
48
|
PricingSnapshotDriftCheck.new.call,
|
|
49
|
+
*reconciliation_invoice_check,
|
|
48
50
|
LegacyBillingStatusCheck.new.call,
|
|
49
51
|
LegacyAuditCheck.new.call,
|
|
50
52
|
call_rollups_check,
|
|
@@ -56,6 +58,26 @@ module LlmCostTracker
|
|
|
56
58
|
|
|
57
59
|
private
|
|
58
60
|
|
|
61
|
+
def reconciliation_schema_checks
|
|
62
|
+
return [] unless LlmCostTracker.reconciliation_enabled?
|
|
63
|
+
|
|
64
|
+
LlmCostTracker.const_get(:Reconciliation) # autoload reconciliation + its ledger schemas
|
|
65
|
+
Reconciliation::SCHEMA_TABLES.map do |schema, table|
|
|
66
|
+
SchemaCheck.new(name: humanize_table(table), schema: schema, table: table,
|
|
67
|
+
optional: false, install_command: "llm_cost_tracker:reconciliation").call
|
|
68
|
+
end.compact
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def humanize_table(table)
|
|
72
|
+
table.delete_prefix("llm_cost_tracker_").tr("_", " ")
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def reconciliation_invoice_check
|
|
76
|
+
return [] unless LlmCostTracker.reconciliation_enabled?
|
|
77
|
+
|
|
78
|
+
Array(InvoiceReconciliationCheck.new.call)
|
|
79
|
+
end
|
|
80
|
+
|
|
59
81
|
def configuration_check
|
|
60
82
|
config = LlmCostTracker.configuration
|
|
61
83
|
Check.new(:ok, "configuration", "active_record ledger enabled=#{config.enabled}")
|
|
@@ -120,9 +142,10 @@ module LlmCostTracker
|
|
|
120
142
|
|
|
121
143
|
def call_rollups_check
|
|
122
144
|
return unless llm_cost_tracker_calls_table?
|
|
145
|
+
return live_rollups_check unless LlmCostTracker.configuration.cache_rollups
|
|
123
146
|
|
|
124
147
|
errors = LlmCostTracker::Ledger::Schema::CallRollups.current_schema_errors
|
|
125
|
-
return
|
|
148
|
+
return rollups_drift_check if errors.empty?
|
|
126
149
|
|
|
127
150
|
Check.new(
|
|
128
151
|
:error,
|
|
@@ -131,6 +154,52 @@ module LlmCostTracker
|
|
|
131
154
|
)
|
|
132
155
|
end
|
|
133
156
|
|
|
157
|
+
ROLLUPS_DRIFT_TOLERANCE_PERCENT = 1.0
|
|
158
|
+
private_constant :ROLLUPS_DRIFT_TOLERANCE_PERCENT
|
|
159
|
+
|
|
160
|
+
def rollups_drift_check
|
|
161
|
+
drift_window = Time.now.utc.beginning_of_day
|
|
162
|
+
calls_total = LlmCostTracker::Call
|
|
163
|
+
.where(tracked_at: drift_window..)
|
|
164
|
+
.where.not(total_cost: nil)
|
|
165
|
+
.sum(:total_cost)
|
|
166
|
+
rollup_total = LlmCostTracker::CallRollup
|
|
167
|
+
.where(period: "day", period_start: drift_window.to_date)
|
|
168
|
+
.sum(:total_cost)
|
|
169
|
+
return Check.new(:ok, "call rollups", "llm_cost_tracker_call_rollups exists") if calls_total.zero?
|
|
170
|
+
|
|
171
|
+
drift_percent = ((calls_total - rollup_total).abs * 100.0 / calls_total)
|
|
172
|
+
if drift_percent > ROLLUPS_DRIFT_TOLERANCE_PERCENT
|
|
173
|
+
return Check.new(
|
|
174
|
+
:warn, "call rollups",
|
|
175
|
+
"rollups drift detected: today's calls SUM=#{calls_total} vs rollups SUM=#{rollup_total} " \
|
|
176
|
+
"(#{drift_percent.round(2)}% > #{ROLLUPS_DRIFT_TOLERANCE_PERCENT}% threshold). " \
|
|
177
|
+
"Cached budget reads may understate spend until a rebuild."
|
|
178
|
+
)
|
|
179
|
+
end
|
|
180
|
+
|
|
181
|
+
Check.new(:ok, "call rollups", "llm_cost_tracker_call_rollups exists")
|
|
182
|
+
rescue StandardError => e
|
|
183
|
+
Check.new(:warn, "call rollups", "rollups drift check failed: #{e.class}: #{e.message}")
|
|
184
|
+
end
|
|
185
|
+
|
|
186
|
+
def live_rollups_check
|
|
187
|
+
if Probe.table_exists?("llm_cost_tracker_call_rollups")
|
|
188
|
+
Check.new(
|
|
189
|
+
:warn,
|
|
190
|
+
"call rollups",
|
|
191
|
+
"cache_rollups=false but llm_cost_tracker_call_rollups exists. " \
|
|
192
|
+
"Set config.cache_rollups = true to keep budget reads on the rollups fast path or drop the table."
|
|
193
|
+
)
|
|
194
|
+
else
|
|
195
|
+
Check.new(
|
|
196
|
+
:ok,
|
|
197
|
+
"call rollups",
|
|
198
|
+
"cache_rollups=false; budget reads aggregate from llm_cost_tracker_calls directly"
|
|
199
|
+
)
|
|
200
|
+
end
|
|
201
|
+
end
|
|
202
|
+
|
|
134
203
|
def calls_check
|
|
135
204
|
return unless llm_cost_tracker_calls_table?
|
|
136
205
|
|
|
@@ -3,10 +3,19 @@
|
|
|
3
3
|
require "rails"
|
|
4
4
|
require_relative "../llm_cost_tracker"
|
|
5
5
|
require_relative "assets"
|
|
6
|
+
require_relative "dashboard_setup_state"
|
|
6
7
|
require "rack/files"
|
|
7
8
|
|
|
8
9
|
module LlmCostTracker
|
|
9
10
|
class Engine < ::Rails::Engine
|
|
10
11
|
isolate_namespace LlmCostTracker
|
|
12
|
+
|
|
13
|
+
initializer "llm_cost_tracker.filter_parameters" do |app|
|
|
14
|
+
app.config.filter_parameters += %i[tag tag_value]
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
initializer "llm_cost_tracker.dashboard_setup_state" do |app|
|
|
18
|
+
app.reloader.to_prepare { LlmCostTracker::DashboardSetupState.reset! }
|
|
19
|
+
end
|
|
11
20
|
end
|
|
12
21
|
end
|
|
@@ -29,7 +29,7 @@ module LlmCostTracker
|
|
|
29
29
|
def to_h
|
|
30
30
|
super.merge(
|
|
31
31
|
token_usage: token_usage.to_h,
|
|
32
|
-
cost: cost
|
|
32
|
+
cost: cost && cost.to_h.transform_values { |v| v.is_a?(BigDecimal) ? v.to_f : v },
|
|
33
33
|
tags: tags ? tags.to_h : {},
|
|
34
34
|
line_items: (line_items || []).map(&:to_h)
|
|
35
35
|
)
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "rails/generators"
|
|
4
|
+
require "rails/generators/active_record"
|
|
5
|
+
|
|
6
|
+
module LlmCostTracker
|
|
7
|
+
module Generators
|
|
8
|
+
class CallRollupsGenerator < Rails::Generators::Base
|
|
9
|
+
include ActiveRecord::Generators::Migration
|
|
10
|
+
|
|
11
|
+
source_root File.expand_path("templates", __dir__)
|
|
12
|
+
|
|
13
|
+
desc "Creates the optional llm_cost_tracker_call_rollups table for fast budget reads. " \
|
|
14
|
+
"Required when config.cache_rollups = true."
|
|
15
|
+
|
|
16
|
+
def create_migration_file
|
|
17
|
+
migration_template(
|
|
18
|
+
"create_llm_cost_tracker_call_rollups.rb.erb",
|
|
19
|
+
"db/migrate/create_llm_cost_tracker_call_rollups.rb"
|
|
20
|
+
)
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def warn_about_config_flag
|
|
24
|
+
say(<<~MSG, :yellow)
|
|
25
|
+
After migrating, set the following in config/initializers/llm_cost_tracker.rb:
|
|
26
|
+
|
|
27
|
+
LlmCostTracker.configure do |config|
|
|
28
|
+
config.cache_rollups = true
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
Without it Tracker keeps reading budget totals as live SUM aggregates over
|
|
32
|
+
llm_cost_tracker_calls. The doctor check warns about an unused rollups table.
|
|
33
|
+
MSG
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
private
|
|
37
|
+
|
|
38
|
+
def migration_version
|
|
39
|
+
"[#{ActiveRecord::VERSION::MAJOR}.#{ActiveRecord::VERSION::MINOR}]"
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
end
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "rails/generators"
|
|
4
|
+
require "rails/generators/active_record"
|
|
5
|
+
|
|
6
|
+
module LlmCostTracker
|
|
7
|
+
module Generators
|
|
8
|
+
class DurableIngestionGenerator < Rails::Generators::Base
|
|
9
|
+
include ActiveRecord::Generators::Migration
|
|
10
|
+
|
|
11
|
+
source_root File.expand_path("templates", __dir__)
|
|
12
|
+
|
|
13
|
+
desc "Creates the durable ingestion tables (llm_cost_tracker_ingestion_inbox_entries + _leases). " \
|
|
14
|
+
"Required when config.durable_ingestion = true."
|
|
15
|
+
|
|
16
|
+
def create_migration_file
|
|
17
|
+
migration_template(
|
|
18
|
+
"create_llm_cost_tracker_durable_ingestion.rb.erb",
|
|
19
|
+
"db/migrate/create_llm_cost_tracker_durable_ingestion.rb"
|
|
20
|
+
)
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def warn_about_config_flag
|
|
24
|
+
say(<<~MSG, :yellow)
|
|
25
|
+
After migrating, set the following in config/initializers/llm_cost_tracker.rb:
|
|
26
|
+
|
|
27
|
+
LlmCostTracker.configure do |config|
|
|
28
|
+
config.durable_ingestion = true
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
Without it the durable inbox tables stay unused and Tracker keeps writing
|
|
32
|
+
inline. The doctor check warns about unused durable tables.
|
|
33
|
+
MSG
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
private
|
|
37
|
+
|
|
38
|
+
def migration_version
|
|
39
|
+
"[#{ActiveRecord::VERSION::MAJOR}.#{ActiveRecord::VERSION::MINOR}]"
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
end
|
|
@@ -35,15 +35,25 @@ module LlmCostTracker
|
|
|
35
35
|
def create_prices_file
|
|
36
36
|
return unless options[:prices]
|
|
37
37
|
|
|
38
|
-
|
|
38
|
+
require_relative "prices_generator"
|
|
39
|
+
invoke LlmCostTracker::Generators::PricesGenerator
|
|
39
40
|
end
|
|
40
41
|
|
|
41
42
|
def mount_engine
|
|
42
43
|
return unless options[:dashboard]
|
|
43
44
|
|
|
44
45
|
add_engine_require
|
|
45
|
-
|
|
46
|
-
|
|
46
|
+
say(<<~MSG, :yellow)
|
|
47
|
+
The LLM Cost Tracker dashboard ships without authentication.
|
|
48
|
+
Mount it in config/routes.rb behind your app's admin auth, e.g.:
|
|
49
|
+
|
|
50
|
+
authenticate :admin do
|
|
51
|
+
mount LlmCostTracker::Engine => "/llm-costs"
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
The generator does NOT add a route automatically — leaving the dashboard
|
|
55
|
+
unauthenticated would expose spend, tags, and provider IDs to anyone.
|
|
56
|
+
MSG
|
|
47
57
|
end
|
|
48
58
|
|
|
49
59
|
private
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "rails/generators"
|
|
4
|
+
require "rails/generators/active_record"
|
|
5
|
+
|
|
6
|
+
module LlmCostTracker
|
|
7
|
+
module Generators
|
|
8
|
+
class ReconciliationGenerator < Rails::Generators::Base
|
|
9
|
+
include ActiveRecord::Generators::Migration
|
|
10
|
+
|
|
11
|
+
source_root File.expand_path("templates", __dir__)
|
|
12
|
+
|
|
13
|
+
desc "Creates the optional invoice reconciliation tables. Requires provider admin/org-level API keys."
|
|
14
|
+
|
|
15
|
+
def create_migration_file
|
|
16
|
+
migration_template(
|
|
17
|
+
"create_llm_cost_tracker_reconciliation.rb.erb",
|
|
18
|
+
"db/migrate/create_llm_cost_tracker_reconciliation.rb"
|
|
19
|
+
)
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def warn_about_admin_keys
|
|
23
|
+
say "Reconciliation requires admin/org-level API keys (OpenAI sk-admin-..., Anthropic admin keys, " \
|
|
24
|
+
"GCP billing.viewer service accounts). Do NOT use the runtime inference key.", :yellow
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
private
|
|
28
|
+
|
|
29
|
+
def migration_version
|
|
30
|
+
"[#{ActiveRecord::VERSION::MAJOR}.#{ActiveRecord::VERSION::MINOR}]"
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
class CreateLlmCostTrackerCallRollups < ActiveRecord::Migration<%= migration_version %>
|
|
2
|
+
def change
|
|
3
|
+
create_table :llm_cost_tracker_call_rollups do |t|
|
|
4
|
+
t.string :period, null: false
|
|
5
|
+
t.date :period_start, null: false
|
|
6
|
+
t.string :currency, null: false, default: "USD"
|
|
7
|
+
t.string :provider, null: false, default: ""
|
|
8
|
+
t.decimal :total_cost, precision: 20, scale: 8, null: false, default: 0
|
|
9
|
+
|
|
10
|
+
t.timestamps
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
add_index :llm_cost_tracker_call_rollups, [:period, :period_start, :currency, :provider], unique: true
|
|
14
|
+
end
|
|
15
|
+
end
|
data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/create_llm_cost_tracker_calls.rb.erb
CHANGED
|
@@ -34,36 +34,6 @@ class CreateLlmCostTrackerCalls < ActiveRecord::Migration<%= migration_version %
|
|
|
34
34
|
t.timestamps
|
|
35
35
|
end
|
|
36
36
|
|
|
37
|
-
create_table :llm_cost_tracker_call_rollups do |t|
|
|
38
|
-
t.string :period, null: false
|
|
39
|
-
t.date :period_start, null: false
|
|
40
|
-
t.string :currency, null: false, default: "USD"
|
|
41
|
-
t.decimal :total_cost, precision: 20, scale: 8, null: false, default: 0
|
|
42
|
-
|
|
43
|
-
t.timestamps
|
|
44
|
-
end
|
|
45
|
-
|
|
46
|
-
create_table :llm_cost_tracker_ingestion_inbox_entries do |t|
|
|
47
|
-
t.string :event_id, null: false
|
|
48
|
-
t.decimal :total_cost, precision: 20, scale: 8
|
|
49
|
-
t.datetime :tracked_at, null: false
|
|
50
|
-
t.text :payload, null: false
|
|
51
|
-
t.datetime :locked_at
|
|
52
|
-
t.string :locked_by
|
|
53
|
-
t.integer :attempts, null: false, default: 0
|
|
54
|
-
t.text :last_error
|
|
55
|
-
|
|
56
|
-
t.timestamps
|
|
57
|
-
end
|
|
58
|
-
|
|
59
|
-
create_table :llm_cost_tracker_ingestion_leases do |t|
|
|
60
|
-
t.string :name, null: false
|
|
61
|
-
t.string :locked_by
|
|
62
|
-
t.datetime :locked_until
|
|
63
|
-
|
|
64
|
-
t.timestamps
|
|
65
|
-
end
|
|
66
|
-
|
|
67
37
|
create_table :llm_cost_tracker_call_line_items do |t|
|
|
68
38
|
t.references :llm_cost_tracker_call,
|
|
69
39
|
null: false,
|
|
@@ -107,25 +77,6 @@ class CreateLlmCostTrackerCalls < ActiveRecord::Migration<%= migration_version %
|
|
|
107
77
|
t.text :value, null: false
|
|
108
78
|
end
|
|
109
79
|
|
|
110
|
-
create_table :llm_cost_tracker_provider_invoices do |t|
|
|
111
|
-
t.string :source, null: false
|
|
112
|
-
t.date :period_start, null: false
|
|
113
|
-
t.date :period_end, null: false
|
|
114
|
-
t.string :external_id, null: false
|
|
115
|
-
t.decimal :billed_amount, precision: 20, scale: 8
|
|
116
|
-
t.string :currency, null: false, default: "USD"
|
|
117
|
-
if postgresql?
|
|
118
|
-
t.jsonb :metadata, null: false, default: {}
|
|
119
|
-
elsif mysql?
|
|
120
|
-
t.json :metadata, null: false
|
|
121
|
-
else
|
|
122
|
-
raise "LLM Cost Tracker supports PostgreSQL and MySQL only"
|
|
123
|
-
end
|
|
124
|
-
t.datetime :imported_at, null: false
|
|
125
|
-
|
|
126
|
-
t.timestamps
|
|
127
|
-
end
|
|
128
|
-
|
|
129
80
|
add_index :llm_cost_tracker_calls, :event_id, unique: true
|
|
130
81
|
add_index :llm_cost_tracker_calls, :tracked_at
|
|
131
82
|
add_index :llm_cost_tracker_calls, [:provider, :tracked_at]
|
|
@@ -133,16 +84,12 @@ class CreateLlmCostTrackerCalls < ActiveRecord::Migration<%= migration_version %
|
|
|
133
84
|
add_index :llm_cost_tracker_calls, :cost_status
|
|
134
85
|
add_index :llm_cost_tracker_calls, :provider_response_id
|
|
135
86
|
add_index :llm_cost_tracker_call_line_items, [:llm_cost_tracker_call_id, :position]
|
|
136
|
-
add_index :llm_cost_tracker_call_line_items, :kind
|
|
137
87
|
add_index :llm_cost_tracker_call_tags, :llm_cost_tracker_call_id
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
add_index :llm_cost_tracker_ingestion_leases, :name, unique: true
|
|
144
|
-
add_index :llm_cost_tracker_provider_invoices, :external_id, unique: true
|
|
145
|
-
add_index :llm_cost_tracker_provider_invoices, [:source, :period_start]
|
|
88
|
+
if postgresql?
|
|
89
|
+
add_index :llm_cost_tracker_call_tags, [:key, :value]
|
|
90
|
+
elsif mysql?
|
|
91
|
+
add_index :llm_cost_tracker_call_tags, [:key, :value], length: { value: 191 }
|
|
92
|
+
end
|
|
146
93
|
end
|
|
147
94
|
|
|
148
95
|
private
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
class CreateLlmCostTrackerDurableIngestion < ActiveRecord::Migration<%= migration_version %>
|
|
2
|
+
def change
|
|
3
|
+
create_table :llm_cost_tracker_ingestion_inbox_entries do |t|
|
|
4
|
+
t.string :event_id, null: false
|
|
5
|
+
t.decimal :total_cost, precision: 20, scale: 8
|
|
6
|
+
t.datetime :tracked_at, null: false
|
|
7
|
+
t.text :payload, null: false
|
|
8
|
+
t.datetime :locked_at
|
|
9
|
+
t.string :locked_by
|
|
10
|
+
t.integer :attempts, null: false, default: 0
|
|
11
|
+
t.text :last_error
|
|
12
|
+
|
|
13
|
+
t.timestamps
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
create_table :llm_cost_tracker_ingestion_leases do |t|
|
|
17
|
+
t.string :name, null: false
|
|
18
|
+
t.string :locked_by
|
|
19
|
+
t.datetime :locked_until
|
|
20
|
+
|
|
21
|
+
t.timestamps
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
add_index :llm_cost_tracker_ingestion_inbox_entries, :event_id, unique: true
|
|
25
|
+
add_index :llm_cost_tracker_ingestion_inbox_entries, [:tracked_at, :attempts]
|
|
26
|
+
add_index :llm_cost_tracker_ingestion_inbox_entries, [:locked_at, :id]
|
|
27
|
+
add_index :llm_cost_tracker_ingestion_leases, :name, unique: true
|
|
28
|
+
end
|
|
29
|
+
end
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
require "llm_cost_tracker/ledger/schema/adapter"
|
|
2
|
+
|
|
3
|
+
class CreateLlmCostTrackerReconciliation < ActiveRecord::Migration<%= migration_version %>
|
|
4
|
+
def change
|
|
5
|
+
create_table :llm_cost_tracker_provider_invoices, if_not_exists: true do |t|
|
|
6
|
+
t.string :source, null: false
|
|
7
|
+
t.date :period_start, null: false
|
|
8
|
+
t.date :period_end, null: false
|
|
9
|
+
t.string :external_id, null: false
|
|
10
|
+
t.decimal :billed_amount, precision: 20, scale: 8
|
|
11
|
+
t.string :currency, null: false, default: "USD"
|
|
12
|
+
if postgresql?
|
|
13
|
+
t.jsonb :metadata, null: false, default: {}
|
|
14
|
+
elsif mysql?
|
|
15
|
+
t.json :metadata, null: false
|
|
16
|
+
else
|
|
17
|
+
raise "LLM Cost Tracker supports PostgreSQL and MySQL only"
|
|
18
|
+
end
|
|
19
|
+
t.datetime :imported_at, null: false
|
|
20
|
+
|
|
21
|
+
t.timestamps
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
create_table :llm_cost_tracker_provider_invoice_imports, if_not_exists: true do |t|
|
|
25
|
+
t.string :source, null: false
|
|
26
|
+
t.string :cursor
|
|
27
|
+
t.date :window_start
|
|
28
|
+
t.date :window_end
|
|
29
|
+
t.string :state, null: false
|
|
30
|
+
t.text :last_error
|
|
31
|
+
t.integer :rows_imported, null: false, default: 0
|
|
32
|
+
t.datetime :started_at, null: false
|
|
33
|
+
t.datetime :finished_at
|
|
34
|
+
|
|
35
|
+
t.timestamps
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
add_index :llm_cost_tracker_provider_invoices, :external_id, unique: true,
|
|
39
|
+
if_not_exists: true
|
|
40
|
+
add_index :llm_cost_tracker_provider_invoices, %i[source currency period_start],
|
|
41
|
+
if_not_exists: true
|
|
42
|
+
add_index :llm_cost_tracker_provider_invoice_imports, %i[source started_at],
|
|
43
|
+
if_not_exists: true
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
private
|
|
47
|
+
|
|
48
|
+
def postgresql?
|
|
49
|
+
LlmCostTracker::Ledger::Schema::Adapter.postgresql?(connection)
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def mysql?
|
|
53
|
+
LlmCostTracker::Ledger::Schema::Adapter.mysql?(connection)
|
|
54
|
+
end
|
|
55
|
+
end
|
|
@@ -4,7 +4,10 @@ LlmCostTracker.configure do |config|
|
|
|
4
4
|
# Set to false to temporarily disable tracking without removing middleware.
|
|
5
5
|
config.enabled = true
|
|
6
6
|
|
|
7
|
-
#
|
|
7
|
+
# LLM Cost Tracker logs warnings through Rails.logger when available.
|
|
8
|
+
config.log_level = :info
|
|
9
|
+
|
|
10
|
+
# Tags merged into every event. Use a callable for request/job-time context.
|
|
8
11
|
config.default_tags = -> { { environment: Rails.env } }
|
|
9
12
|
|
|
10
13
|
# Tag guardrails keep accidental high-cardinality or sensitive values out of the ledger.
|
|
@@ -18,40 +21,40 @@ LlmCostTracker.configure do |config|
|
|
|
18
21
|
# config.instrument :anthropic
|
|
19
22
|
# config.instrument :ruby_llm
|
|
20
23
|
|
|
21
|
-
#
|
|
22
|
-
#
|
|
23
|
-
config.budget_exceeded_behavior = :notify
|
|
24
|
-
|
|
25
|
-
# Unknown pricing records token usage with nil cost by default. Use :raise if
|
|
26
|
-
# every model must have known pricing before it can be used.
|
|
27
|
-
config.unknown_pricing_behavior = :warn
|
|
28
|
-
|
|
29
|
-
# LLM Cost Tracker logs warnings through Rails.logger when available.
|
|
30
|
-
config.log_level = :info
|
|
24
|
+
# Pricing — local file refreshed via bin/rails llm_cost_tracker:prices:refresh
|
|
25
|
+
# plus inline overrides. Prices are USD per 1M tokens.
|
|
31
26
|
<% if options[:prices] -%>
|
|
32
|
-
|
|
33
|
-
# Local JSON/YAML pricing file generated by --prices. Keep it in source control
|
|
34
|
-
# and refresh it with bin/rails llm_cost_tracker:prices:refresh.
|
|
35
27
|
config.prices_file = Rails.root.join("config/llm_cost_tracker_prices.yml")
|
|
28
|
+
<% else -%>
|
|
29
|
+
# config.prices_file = Rails.root.join("config/llm_cost_tracker_prices.yml")
|
|
36
30
|
<% end -%>
|
|
31
|
+
# config.pricing_overrides = {
|
|
32
|
+
# "my-custom-model" => { input: 1.00, output: 2.00 }
|
|
33
|
+
# }
|
|
34
|
+
# :warn (default) records token usage with nil cost when a model has no rate.
|
|
35
|
+
# Use :raise to require known pricing for every model.
|
|
36
|
+
config.unknown_pricing_behavior = :warn
|
|
37
37
|
|
|
38
|
-
#
|
|
38
|
+
# Budget guardrails — cumulative monthly/daily and per-call ceilings in USD,
|
|
39
|
+
# plus behavior on crossing (:notify default fires on_budget_exceeded; :raise
|
|
40
|
+
# raises after recording; :block_requests preflights supported requests) and
|
|
41
|
+
# an optional callback. Cap evaluation reads from llm_cost_tracker_calls live;
|
|
42
|
+
# flip cache_rollups to true at high volume so reads hit the rollups table
|
|
43
|
+
# instead — generate the table with `bin/rails generate llm_cost_tracker:call_rollups`.
|
|
39
44
|
# config.monthly_budget = 100.00
|
|
40
45
|
# config.daily_budget = 10.00
|
|
41
46
|
# config.per_call_budget = 1.00
|
|
42
|
-
|
|
43
|
-
# Called when :notify is selected and a monthly, daily, or per-call budget is exceeded.
|
|
47
|
+
config.budget_exceeded_behavior = :notify
|
|
44
48
|
# config.on_budget_exceeded = ->(data) {
|
|
45
|
-
# Rails.logger.warn(
|
|
46
|
-
# "LLM #{data[:budget_type]} budget exceeded: $#{data[:total]} / $#{data[:budget]}"
|
|
47
|
-
# )
|
|
49
|
+
# Rails.logger.warn("LLM #{data[:budget_type]} budget exceeded: $#{data[:total]} / $#{data[:budget]}")
|
|
48
50
|
# }
|
|
51
|
+
# config.cache_rollups = true
|
|
49
52
|
|
|
50
|
-
#
|
|
51
|
-
#
|
|
52
|
-
#
|
|
53
|
-
#
|
|
54
|
-
#
|
|
53
|
+
# Ingestion path — false (default) writes events synchronously from the request
|
|
54
|
+
# thread. Flip to true for a write-ahead inbox + background worker that batches
|
|
55
|
+
# inserts and survives caller transaction rollbacks. Requires the optional
|
|
56
|
+
# inbox/leases tables created by `bin/rails generate llm_cost_tracker:durable_ingestion`.
|
|
57
|
+
# config.durable_ingestion = true
|
|
55
58
|
|
|
56
59
|
# Register OpenAI-compatible gateway hosts and choose extra tag breakdowns
|
|
57
60
|
# for bin/rails llm_cost_tracker:report.
|
data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_call_rollups_provider.rb.erb
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
class UpgradeLlmCostTrackerCallRollupsProvider < ActiveRecord::Migration<%= migration_version %>
|
|
2
|
+
TABLE = :llm_cost_tracker_call_rollups
|
|
3
|
+
OLD_INDEX = %i[period period_start currency].freeze
|
|
4
|
+
NEW_INDEX = %i[period period_start currency provider].freeze
|
|
5
|
+
|
|
6
|
+
def up
|
|
7
|
+
unless column_exists?(TABLE, :provider)
|
|
8
|
+
execute "DELETE FROM #{TABLE}"
|
|
9
|
+
add_column TABLE, :provider, :string, null: false, default: ""
|
|
10
|
+
end
|
|
11
|
+
remove_index TABLE, column: OLD_INDEX, unique: true if index_exists?(TABLE, OLD_INDEX, unique: true)
|
|
12
|
+
add_index TABLE, NEW_INDEX, unique: true unless index_exists?(TABLE, NEW_INDEX, unique: true)
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def down
|
|
16
|
+
remove_index TABLE, column: NEW_INDEX, unique: true if index_exists?(TABLE, NEW_INDEX, unique: true)
|
|
17
|
+
add_index TABLE, OLD_INDEX, unique: true unless index_exists?(TABLE, OLD_INDEX, unique: true)
|
|
18
|
+
remove_column TABLE, :provider if column_exists?(TABLE, :provider)
|
|
19
|
+
end
|
|
20
|
+
end
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
require "llm_cost_tracker/ledger/schema/adapter"
|
|
2
|
+
|
|
3
|
+
class UpgradeLlmCostTrackerCallTagsKeyValueIndex < ActiveRecord::Migration<%= migration_version %>
|
|
4
|
+
TABLE = :llm_cost_tracker_call_tags
|
|
5
|
+
INDEX_COLUMNS = %i[key value].freeze
|
|
6
|
+
|
|
7
|
+
def up
|
|
8
|
+
return if index_exists?(TABLE, INDEX_COLUMNS)
|
|
9
|
+
|
|
10
|
+
if postgresql?
|
|
11
|
+
add_index TABLE, INDEX_COLUMNS
|
|
12
|
+
elsif mysql?
|
|
13
|
+
add_index TABLE, INDEX_COLUMNS, length: { value: 191 }
|
|
14
|
+
else
|
|
15
|
+
raise "LLM Cost Tracker supports PostgreSQL and MySQL only"
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def down
|
|
20
|
+
remove_index TABLE, column: INDEX_COLUMNS if index_exists?(TABLE, INDEX_COLUMNS)
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
private
|
|
24
|
+
|
|
25
|
+
def postgresql?
|
|
26
|
+
LlmCostTracker::Ledger::Schema::Adapter.postgresql?(connection)
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def mysql?
|
|
30
|
+
LlmCostTracker::Ledger::Schema::Adapter.mysql?(connection)
|
|
31
|
+
end
|
|
32
|
+
end
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
class UpgradeLlmCostTrackerImageTokens < ActiveRecord::Migration<%= migration_version %>
|
|
2
|
+
TABLE = :llm_cost_tracker_calls
|
|
3
|
+
COLUMNS = %i[image_input_tokens image_output_tokens].freeze
|
|
4
|
+
|
|
5
|
+
def up
|
|
6
|
+
COLUMNS.each do |column|
|
|
7
|
+
next if column_exists?(TABLE, column)
|
|
8
|
+
|
|
9
|
+
add_column TABLE, column, :integer, null: false, default: 0
|
|
10
|
+
end
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
def down
|
|
14
|
+
COLUMNS.each do |column|
|
|
15
|
+
remove_column TABLE, column if column_exists?(TABLE, column)
|
|
16
|
+
end
|
|
17
|
+
end
|
|
18
|
+
end
|
data/lib/llm_cost_tracker/generators/llm_cost_tracker/upgrade_call_rollups_provider_generator.rb
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "rails/generators"
|
|
4
|
+
require "rails/generators/active_record"
|
|
5
|
+
|
|
6
|
+
module LlmCostTracker
|
|
7
|
+
module Generators
|
|
8
|
+
class UpgradeCallRollupsProviderGenerator < Rails::Generators::Base
|
|
9
|
+
include ActiveRecord::Generators::Migration
|
|
10
|
+
|
|
11
|
+
source_root File.expand_path("templates", __dir__)
|
|
12
|
+
|
|
13
|
+
desc "Adds the v0.9 provider column and unique index to llm_cost_tracker_call_rollups."
|
|
14
|
+
|
|
15
|
+
def create_migration_file
|
|
16
|
+
migration_template(
|
|
17
|
+
"upgrade_call_rollups_provider.rb.erb",
|
|
18
|
+
"db/migrate/upgrade_llm_cost_tracker_call_rollups_provider.rb"
|
|
19
|
+
)
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def warn_about_rollups_truncation
|
|
23
|
+
say(<<~MSG, :yellow)
|
|
24
|
+
The migration clears existing llm_cost_tracker_call_rollups rows before adding the
|
|
25
|
+
provider column. Budget reads fall back to live aggregation from
|
|
26
|
+
llm_cost_tracker_calls until new events repopulate the rollups under their provider
|
|
27
|
+
keys. See docs/upgrading.md for details.
|
|
28
|
+
MSG
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
private
|
|
32
|
+
|
|
33
|
+
def migration_version
|
|
34
|
+
"[#{ActiveRecord::VERSION::MAJOR}.#{ActiveRecord::VERSION::MINOR}]"
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|