llm_cost_tracker 0.8.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +136 -0
- data/README.md +14 -6
- data/app/assets/llm_cost_tracker/application.css +65 -5
- data/app/controllers/llm_cost_tracker/application_controller.rb +25 -33
- data/app/controllers/llm_cost_tracker/assets_controller.rb +1 -1
- data/app/controllers/llm_cost_tracker/calls_controller.rb +21 -11
- data/app/controllers/llm_cost_tracker/data_quality_controller.rb +4 -0
- data/app/controllers/llm_cost_tracker/reconciliation_controller.rb +106 -0
- data/app/controllers/llm_cost_tracker/tags_controller.rb +15 -1
- data/app/helpers/llm_cost_tracker/application_helper.rb +11 -1
- data/app/helpers/llm_cost_tracker/inline_style_helper.rb +28 -0
- data/app/helpers/llm_cost_tracker/reconciliation_helper.rb +13 -0
- data/app/helpers/llm_cost_tracker/token_usage_helper.rb +5 -1
- data/app/models/llm_cost_tracker/call.rb +0 -3
- data/app/models/llm_cost_tracker/call_line_item.rb +1 -5
- data/app/models/llm_cost_tracker/call_rollup.rb +0 -3
- data/app/models/llm_cost_tracker/call_tag.rb +0 -4
- data/app/models/llm_cost_tracker/ingestion/inbox_entry.rb +0 -4
- data/app/models/llm_cost_tracker/ingestion/lease.rb +0 -3
- data/app/models/llm_cost_tracker/provider_invoice.rb +7 -3
- data/app/models/llm_cost_tracker/provider_invoice_import.rb +29 -0
- data/app/services/llm_cost_tracker/dashboard/data_quality.rb +33 -4
- data/app/services/llm_cost_tracker/dashboard/filter.rb +6 -4
- data/app/services/llm_cost_tracker/dashboard/setup_state.rb +110 -0
- data/app/views/layouts/llm_cost_tracker/application.html.erb +6 -1
- data/app/views/llm_cost_tracker/calls/show.html.erb +26 -41
- data/app/views/llm_cost_tracker/dashboard/index.html.erb +9 -9
- data/app/views/llm_cost_tracker/data_quality/index.html.erb +92 -53
- data/app/views/llm_cost_tracker/reconciliation/index.html.erb +183 -0
- data/app/views/llm_cost_tracker/shared/_bar.html.erb +1 -1
- data/app/views/llm_cost_tracker/shared/_filters.html.erb +3 -0
- data/app/views/llm_cost_tracker/shared/_metric_stack.html.erb +1 -1
- data/app/views/llm_cost_tracker/tags/show.html.erb +60 -0
- data/config/routes.rb +3 -2
- data/lib/llm_cost_tracker/billing/components.rb +45 -3
- data/lib/llm_cost_tracker/billing/components.yml +71 -0
- data/lib/llm_cost_tracker/billing/cost_status.rb +21 -25
- data/lib/llm_cost_tracker/billing/line_item.rb +16 -50
- data/lib/llm_cost_tracker/budget.rb +31 -7
- data/lib/llm_cost_tracker/capture/stream_collector.rb +113 -34
- data/lib/llm_cost_tracker/capture/stream_tracker.rb +40 -5
- data/lib/llm_cost_tracker/configuration.rb +72 -17
- data/lib/llm_cost_tracker/doctor/capture_verifier.rb +1 -1
- data/lib/llm_cost_tracker/doctor/cost_drift_check.rb +2 -0
- data/lib/llm_cost_tracker/doctor/ingestion_check.rb +30 -4
- data/lib/llm_cost_tracker/doctor/invoice_reconciliation_check.rb +164 -0
- data/lib/llm_cost_tracker/doctor/legacy_audit_check.rb +0 -2
- data/lib/llm_cost_tracker/doctor/legacy_billing_status_check.rb +0 -2
- data/lib/llm_cost_tracker/doctor/schema_check.rb +5 -2
- data/lib/llm_cost_tracker/doctor.rb +72 -14
- data/lib/llm_cost_tracker/engine.rb +8 -0
- data/lib/llm_cost_tracker/errors.rb +3 -2
- data/lib/llm_cost_tracker/event.rb +48 -1
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/async_ingestion_generator.rb +43 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/call_rollups_generator.rb +43 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/install_generator.rb +17 -26
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/reconciliation_generator.rb +34 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/create_llm_cost_tracker_async_ingestion.rb.erb +29 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/create_llm_cost_tracker_call_rollups.rb.erb +15 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/create_llm_cost_tracker_calls.rb.erb +5 -58
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/create_llm_cost_tracker_reconciliation.rb.erb +60 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/initializer.rb.erb +35 -25
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_call_rollups_provider.rb.erb +35 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_call_tags_key_value_index.rb.erb +32 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_image_tokens.rb.erb +18 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_provider_invoice_imports_provider.rb.erb +32 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_provider_invoices_metadata_index.rb.erb +25 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/upgrade_call_rollups_provider_generator.rb +29 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/upgrade_call_tags_key_value_index_generator.rb +30 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/upgrade_image_tokens_generator.rb +29 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/upgrade_provider_invoice_imports_provider_generator.rb +31 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/upgrade_provider_invoices_metadata_index_generator.rb +31 -0
- data/lib/llm_cost_tracker/ingestion/batch.rb +5 -2
- data/lib/llm_cost_tracker/ingestion/inbox.rb +3 -25
- data/lib/llm_cost_tracker/ingestion/pool.rb +44 -0
- data/lib/llm_cost_tracker/ingestion/worker.rb +28 -34
- data/lib/llm_cost_tracker/ingestion.rb +48 -11
- data/lib/llm_cost_tracker/integrations/anthropic.rb +31 -26
- data/lib/llm_cost_tracker/integrations/base.rb +35 -15
- data/lib/llm_cost_tracker/integrations/openai.rb +345 -84
- data/lib/llm_cost_tracker/integrations/ruby_llm.rb +111 -14
- data/lib/llm_cost_tracker/integrations.rb +33 -14
- data/lib/llm_cost_tracker/ledger/period/totals.rb +25 -7
- data/lib/llm_cost_tracker/ledger/rollups.rb +22 -17
- data/lib/llm_cost_tracker/ledger/schema/call_line_items.rb +41 -1
- data/lib/llm_cost_tracker/ledger/schema/call_rollups.rb +16 -6
- data/lib/llm_cost_tracker/ledger/schema/call_tags.rb +28 -2
- data/lib/llm_cost_tracker/ledger/schema/calls.rb +2 -4
- data/lib/llm_cost_tracker/ledger/schema/ingestion_inbox_entries.rb +57 -0
- data/lib/llm_cost_tracker/ledger/schema/ingestion_leases.rb +52 -0
- data/lib/llm_cost_tracker/ledger/schema/provider_invoice_imports.rb +56 -0
- data/lib/llm_cost_tracker/ledger/schema/provider_invoices.rb +28 -13
- data/lib/llm_cost_tracker/ledger/store.rb +34 -31
- data/lib/llm_cost_tracker/ledger/tags/encoding.rb +37 -0
- data/lib/llm_cost_tracker/ledger/tags/query.rb +2 -2
- data/lib/llm_cost_tracker/ledger.rb +2 -1
- data/lib/llm_cost_tracker/logging.rb +0 -4
- data/lib/llm_cost_tracker/masking.rb +39 -0
- data/lib/llm_cost_tracker/middleware/faraday.rb +120 -33
- data/lib/llm_cost_tracker/parsers/anthropic.rb +36 -28
- data/lib/llm_cost_tracker/parsers/azure.rb +46 -0
- data/lib/llm_cost_tracker/parsers/base.rb +53 -43
- data/lib/llm_cost_tracker/parsers/gemini.rb +24 -22
- data/lib/llm_cost_tracker/parsers/openai.rb +20 -38
- data/lib/llm_cost_tracker/parsers/openai_compatible.rb +26 -39
- data/lib/llm_cost_tracker/parsers/openai_service_charges.rb +81 -13
- data/lib/llm_cost_tracker/parsers/openai_usage.rb +126 -59
- data/lib/llm_cost_tracker/parsers.rb +31 -4
- data/lib/llm_cost_tracker/prices.json +572 -493
- data/lib/llm_cost_tracker/pricing/backfill.rb +140 -0
- data/lib/llm_cost_tracker/pricing/effective_prices.rb +7 -40
- data/lib/llm_cost_tracker/pricing/estimator.rb +33 -0
- data/lib/llm_cost_tracker/pricing/explainer.rb +4 -1
- data/lib/llm_cost_tracker/pricing/lookup.rb +73 -5
- data/lib/llm_cost_tracker/pricing/mode.rb +76 -0
- data/lib/llm_cost_tracker/pricing/registry.rb +3 -8
- data/lib/llm_cost_tracker/pricing/service_charges.rb +14 -12
- data/lib/llm_cost_tracker/pricing/{sync_change_printer.rb → sync/change_printer.rb} +3 -3
- data/lib/llm_cost_tracker/pricing/sync/registry_writer.rb +62 -1
- data/lib/llm_cost_tracker/pricing/sync.rb +4 -10
- data/lib/llm_cost_tracker/pricing/unknown.rb +5 -2
- data/lib/llm_cost_tracker/pricing.rb +117 -44
- data/lib/llm_cost_tracker/providers/anthropic/tier_classification.rb +22 -0
- data/lib/llm_cost_tracker/providers/azure/hosts.rb +17 -0
- data/lib/llm_cost_tracker/providers/gemini/model_families.rb +17 -0
- data/lib/llm_cost_tracker/providers/openai/hosts.rb +35 -0
- data/lib/llm_cost_tracker/providers/openai/model_families.rb +51 -0
- data/lib/llm_cost_tracker/railtie.rb +8 -0
- data/lib/llm_cost_tracker/reconcile_tasks.rb +134 -0
- data/lib/llm_cost_tracker/reconciliation/diff.rb +409 -0
- data/lib/llm_cost_tracker/reconciliation/diff_result.rb +44 -0
- data/lib/llm_cost_tracker/reconciliation/import_result.rb +19 -0
- data/lib/llm_cost_tracker/reconciliation/importer.rb +254 -0
- data/lib/llm_cost_tracker/reconciliation/sources/anthropic_usage.rb +172 -0
- data/lib/llm_cost_tracker/reconciliation/sources/fingerprint.rb +20 -0
- data/lib/llm_cost_tracker/reconciliation/sources/openai_usage.rb +142 -0
- data/lib/llm_cost_tracker/reconciliation.rb +118 -0
- data/lib/llm_cost_tracker/report/data.rb +4 -1
- data/lib/llm_cost_tracker/report.rb +0 -4
- data/lib/llm_cost_tracker/retention.rb +31 -6
- data/lib/llm_cost_tracker/tags/context.rb +3 -4
- data/lib/llm_cost_tracker/tags/sanitizer.rb +73 -21
- data/lib/llm_cost_tracker/token_usage.rb +14 -2
- data/lib/llm_cost_tracker/tracker.rb +41 -55
- data/lib/llm_cost_tracker/version.rb +1 -1
- data/lib/llm_cost_tracker.rb +19 -14
- data/lib/tasks/llm_cost_tracker.rake +41 -4
- metadata +49 -3
- data/lib/llm_cost_tracker/usage_capture.rb +0 -58
|
@@ -7,7 +7,21 @@ module LlmCostTracker
|
|
|
7
7
|
end
|
|
8
8
|
|
|
9
9
|
def show
|
|
10
|
-
|
|
10
|
+
scope = Dashboard::Filter.call(params: params)
|
|
11
|
+
@value = params[:tag_value].to_s
|
|
12
|
+
|
|
13
|
+
if @value.empty?
|
|
14
|
+
@breakdown = Dashboard::TagBreakdown.call(scope: scope, key: params[:key])
|
|
15
|
+
else
|
|
16
|
+
@key = LlmCostTracker::Tags::Key.validate!(
|
|
17
|
+
params[:key],
|
|
18
|
+
error_class: LlmCostTracker::InvalidFilterError
|
|
19
|
+
)
|
|
20
|
+
value_scope = scope.by_tag(@key, @value)
|
|
21
|
+
@value_total_cost = value_scope.sum(:total_cost).to_f
|
|
22
|
+
@value_calls = value_scope.count
|
|
23
|
+
@value_points = Dashboard::TimeSeries.call(scope: value_scope)
|
|
24
|
+
end
|
|
11
25
|
end
|
|
12
26
|
end
|
|
13
27
|
end
|
|
@@ -13,6 +13,7 @@ module LlmCostTracker
|
|
|
13
13
|
include ChartHelper
|
|
14
14
|
include PaginationHelper
|
|
15
15
|
include TokenUsageHelper
|
|
16
|
+
include InlineStyleHelper
|
|
16
17
|
|
|
17
18
|
def coverage_percent(numerator, denominator)
|
|
18
19
|
denominator = denominator.to_f
|
|
@@ -37,7 +38,7 @@ module LlmCostTracker
|
|
|
37
38
|
end
|
|
38
39
|
|
|
39
40
|
def number(value)
|
|
40
|
-
number_with_delimiter(value
|
|
41
|
+
number_with_delimiter(value)
|
|
41
42
|
end
|
|
42
43
|
|
|
43
44
|
def format_date(value)
|
|
@@ -104,6 +105,15 @@ module LlmCostTracker
|
|
|
104
105
|
value.to_s
|
|
105
106
|
end
|
|
106
107
|
|
|
108
|
+
def masked_metadata_hash(value)
|
|
109
|
+
return value if value.is_a?(Hash)
|
|
110
|
+
return {} if value.nil?
|
|
111
|
+
|
|
112
|
+
JSON.parse(value.to_s)
|
|
113
|
+
rescue JSON::ParserError, TypeError
|
|
114
|
+
{}
|
|
115
|
+
end
|
|
116
|
+
|
|
107
117
|
def tag_chip_entries(tags, limit: 3)
|
|
108
118
|
normalized = normalized_tags(tags)
|
|
109
119
|
return [] if normalized.empty?
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module LlmCostTracker
|
|
4
|
+
module InlineStyleHelper
|
|
5
|
+
UNSAFE_CSS_CHARS = /[<>{}"]/
|
|
6
|
+
|
|
7
|
+
def inline_style(declarations)
|
|
8
|
+
registry = inline_style_registry
|
|
9
|
+
token = "lct-i-#{registry.length}"
|
|
10
|
+
registry << [token, declarations.to_s.gsub(UNSAFE_CSS_CHARS, "")]
|
|
11
|
+
token
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def inline_style_block
|
|
15
|
+
registry = inline_style_registry
|
|
16
|
+
return "".html_safe if registry.empty?
|
|
17
|
+
|
|
18
|
+
rules = registry.map { |token, decl| %([data-lct-style="#{token}"]{#{decl}}) }.join("\n")
|
|
19
|
+
content_tag(:style, rules.html_safe, nonce: dashboard_csp_nonce)
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
private
|
|
23
|
+
|
|
24
|
+
def inline_style_registry
|
|
25
|
+
@inline_style_registry ||= []
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
end
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module LlmCostTracker
|
|
4
|
+
module ReconciliationHelper
|
|
5
|
+
def attribution_summary(attribution)
|
|
6
|
+
LlmCostTracker::Masking.format_attribution(attribution)
|
|
7
|
+
end
|
|
8
|
+
|
|
9
|
+
def mask_secret(value)
|
|
10
|
+
LlmCostTracker::Masking.mask_value(:provider_api_key_id, value)
|
|
11
|
+
end
|
|
12
|
+
end
|
|
13
|
+
end
|
|
@@ -8,8 +8,10 @@ module LlmCostTracker
|
|
|
8
8
|
cache_write_input_tokens: "Cache write",
|
|
9
9
|
cache_write_extended_input_tokens: "Extended cache write",
|
|
10
10
|
audio_input_tokens: "Audio input",
|
|
11
|
+
image_input_tokens: "Image input",
|
|
11
12
|
output_tokens: "Output",
|
|
12
13
|
audio_output_tokens: "Audio output",
|
|
14
|
+
image_output_tokens: "Image output",
|
|
13
15
|
hidden_output_tokens: "Hidden output"
|
|
14
16
|
}.freeze
|
|
15
17
|
QUALITY_LABELS = COMPONENT_LABELS.merge(
|
|
@@ -24,8 +26,10 @@ module LlmCostTracker
|
|
|
24
26
|
cache_write_input_tokens: "lct-stack-fill-cache-write",
|
|
25
27
|
cache_write_extended_input_tokens: "lct-stack-fill-cache-write-extended",
|
|
26
28
|
audio_input_tokens: "lct-stack-fill-audio-input",
|
|
29
|
+
image_input_tokens: "lct-stack-fill-image-input",
|
|
27
30
|
output_tokens: "lct-stack-fill-output",
|
|
28
|
-
audio_output_tokens: "lct-stack-fill-audio-output"
|
|
31
|
+
audio_output_tokens: "lct-stack-fill-audio-output",
|
|
32
|
+
image_output_tokens: "lct-stack-fill-image-output"
|
|
29
33
|
}.freeze
|
|
30
34
|
|
|
31
35
|
def token_usage_stack_components
|
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require "active_record"
|
|
4
3
|
require "securerandom"
|
|
5
4
|
|
|
6
5
|
require "llm_cost_tracker/billing/cost_status"
|
|
@@ -9,8 +8,6 @@ require "llm_cost_tracker/ledger/tags/sql"
|
|
|
9
8
|
|
|
10
9
|
module LlmCostTracker
|
|
11
10
|
class Call < ActiveRecord::Base
|
|
12
|
-
self.table_name = "llm_cost_tracker_calls"
|
|
13
|
-
|
|
14
11
|
before_validation :assign_event_id
|
|
15
12
|
|
|
16
13
|
PERIOD_FORMATS = {
|
|
@@ -1,17 +1,13 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require "active_record"
|
|
4
|
-
|
|
5
3
|
module LlmCostTracker
|
|
6
4
|
class CallLineItem < ActiveRecord::Base
|
|
7
|
-
self.table_name = "llm_cost_tracker_call_line_items"
|
|
8
|
-
|
|
9
5
|
belongs_to :call,
|
|
10
6
|
class_name: "LlmCostTracker::Call",
|
|
11
7
|
foreign_key: :llm_cost_tracker_call_id,
|
|
12
8
|
inverse_of: :line_items
|
|
13
9
|
|
|
14
|
-
scope :tokens, -> { where(
|
|
10
|
+
scope :tokens, -> { where(unit: "token") }
|
|
15
11
|
scope :by_kind, ->(kind) { where(kind: kind.to_s) }
|
|
16
12
|
scope :by_direction, ->(direction) { where(direction: direction.to_s) }
|
|
17
13
|
scope :by_modality, ->(modality) { where(modality: modality.to_s) }
|
|
@@ -1,11 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require "active_record"
|
|
4
|
-
|
|
5
3
|
module LlmCostTracker
|
|
6
4
|
class CallTag < ActiveRecord::Base
|
|
7
|
-
self.table_name = "llm_cost_tracker_call_tags"
|
|
8
|
-
|
|
9
5
|
belongs_to :call,
|
|
10
6
|
class_name: "LlmCostTracker::Call",
|
|
11
7
|
foreign_key: :llm_cost_tracker_call_id,
|
|
@@ -1,12 +1,8 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require "active_record"
|
|
4
|
-
|
|
5
3
|
module LlmCostTracker
|
|
6
4
|
module Ingestion
|
|
7
5
|
class InboxEntry < ActiveRecord::Base
|
|
8
|
-
self.table_name = "llm_cost_tracker_ingestion_inbox_entries"
|
|
9
|
-
|
|
10
6
|
MAX_ATTEMPTS_BEFORE_QUARANTINE = 5
|
|
11
7
|
end
|
|
12
8
|
end
|
|
@@ -1,9 +1,13 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require "active_record"
|
|
4
|
-
|
|
5
3
|
module LlmCostTracker
|
|
6
4
|
class ProviderInvoice < ActiveRecord::Base
|
|
7
|
-
|
|
5
|
+
before_validation :normalize_currency
|
|
6
|
+
|
|
7
|
+
private
|
|
8
|
+
|
|
9
|
+
def normalize_currency
|
|
10
|
+
self.currency = currency.to_s.upcase if currency.present?
|
|
11
|
+
end
|
|
8
12
|
end
|
|
9
13
|
end
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module LlmCostTracker
|
|
4
|
+
class ProviderInvoiceImport < ActiveRecord::Base
|
|
5
|
+
STATE_RUNNING = "running"
|
|
6
|
+
STATE_COMPLETED = "completed"
|
|
7
|
+
STATE_FAILED = "failed"
|
|
8
|
+
STATES = [STATE_RUNNING, STATE_COMPLETED, STATE_FAILED].freeze
|
|
9
|
+
|
|
10
|
+
scope :for_source, ->(source) { where(source: source.to_s) }
|
|
11
|
+
scope :for_provider, ->(provider) { where(provider: provider.to_s) }
|
|
12
|
+
scope :running, -> { where(state: STATE_RUNNING) }
|
|
13
|
+
scope :completed, -> { where(state: STATE_COMPLETED) }
|
|
14
|
+
scope :failed, -> { where(state: STATE_FAILED) }
|
|
15
|
+
scope :latest, -> { order(started_at: :desc, id: :desc) }
|
|
16
|
+
|
|
17
|
+
def self.resume_cursor_for(source, provider: nil)
|
|
18
|
+
scope = for_source(source)
|
|
19
|
+
scope = scope.for_provider(provider) if provider
|
|
20
|
+
scope.latest.limit(1).pick(:cursor)
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def self.last_completed_window_for(source, provider: nil)
|
|
24
|
+
scope = for_source(source)
|
|
25
|
+
scope = scope.for_provider(provider) if provider
|
|
26
|
+
scope.completed.latest.limit(1).pick(:window_start, :window_end)
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
end
|
|
@@ -6,7 +6,8 @@ require "llm_cost_tracker/ledger/schema/adapter"
|
|
|
6
6
|
module LlmCostTracker
|
|
7
7
|
module Dashboard
|
|
8
8
|
class DataQuality
|
|
9
|
-
UnknownPricingRow = ::Data.define(:model, :calls, :share_percent)
|
|
9
|
+
UnknownPricingRow = ::Data.define(:provider, :model, :calls, :share_percent)
|
|
10
|
+
StreamingHealthRow = ::Data.define(:provider, :streams, :with_usage, :unknown, :unknown_share)
|
|
10
11
|
Summary = ::Data.define(:total, :unknown_pricing_count, :untagged_calls_count, :missing_latency_count,
|
|
11
12
|
:streaming_count, :streaming_missing_usage, :missing_provider_response_id_count,
|
|
12
13
|
:calls_with_pricing, :tagged_calls, :calls_with_latency, :streams_with_usage,
|
|
@@ -48,13 +49,14 @@ module LlmCostTracker
|
|
|
48
49
|
|
|
49
50
|
def unknown_pricing_by_model(scope, total_calls:)
|
|
50
51
|
scope.unknown_pricing
|
|
51
|
-
.group(:model)
|
|
52
|
+
.group(:provider, :model)
|
|
52
53
|
.order(Arel.sql("COUNT(*) DESC"))
|
|
53
|
-
.select("model, COUNT(*) AS calls")
|
|
54
|
+
.select("provider, model, COUNT(*) AS calls")
|
|
54
55
|
.limit(10)
|
|
55
56
|
.map do |row|
|
|
56
57
|
calls = row.calls.to_i
|
|
57
|
-
UnknownPricingRow.new(
|
|
58
|
+
UnknownPricingRow.new(provider: row.provider, model: row.model, calls: calls,
|
|
59
|
+
share_percent: percentage(calls, total_calls))
|
|
58
60
|
end
|
|
59
61
|
end
|
|
60
62
|
|
|
@@ -125,6 +127,33 @@ module LlmCostTracker
|
|
|
125
127
|
index_costs_by_component(rows)
|
|
126
128
|
end
|
|
127
129
|
|
|
130
|
+
def streaming_health_rows(scope, total_streaming:)
|
|
131
|
+
return [] unless total_streaming.positive?
|
|
132
|
+
|
|
133
|
+
unknown_predicate = "usage_source = 'unknown' OR usage_source IS NULL"
|
|
134
|
+
rows = scope.unscope(:select, :order, :group)
|
|
135
|
+
.where(stream: true)
|
|
136
|
+
.group(:provider)
|
|
137
|
+
.order(Arel.sql("COUNT(*) DESC"), :provider)
|
|
138
|
+
.pluck(
|
|
139
|
+
:provider,
|
|
140
|
+
Arel.sql("COUNT(*)"),
|
|
141
|
+
Arel.sql("SUM(CASE WHEN #{unknown_predicate} THEN 1 ELSE 0 END)")
|
|
142
|
+
)
|
|
143
|
+
|
|
144
|
+
rows.map do |provider, streams, unknown|
|
|
145
|
+
streams_count = streams.to_i
|
|
146
|
+
unknown_count = unknown.to_i
|
|
147
|
+
StreamingHealthRow.new(
|
|
148
|
+
provider: provider,
|
|
149
|
+
streams: streams_count,
|
|
150
|
+
with_usage: streams_count - unknown_count,
|
|
151
|
+
unknown: unknown_count,
|
|
152
|
+
unknown_share: percentage(unknown_count, streams_count)
|
|
153
|
+
)
|
|
154
|
+
end
|
|
155
|
+
end
|
|
156
|
+
|
|
128
157
|
def hidden_output_summary(stats)
|
|
129
158
|
output_tokens = stats.output_tokens.to_i
|
|
130
159
|
return unless output_tokens.positive?
|
|
@@ -35,11 +35,13 @@ module LlmCostTracker
|
|
|
35
35
|
to_date = Dashboard::DateRange.parse(params, :to)
|
|
36
36
|
Dashboard::DateRange.validate!(from: from_date, to: to_date)
|
|
37
37
|
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
38
|
+
default_range = Dashboard::DateRange.call(params: params)
|
|
39
|
+
from_date ||= default_range.from
|
|
40
|
+
to_date ||= default_range.to
|
|
41
|
+
|
|
42
42
|
relation
|
|
43
|
+
.where(tracked_at: from_date.beginning_of_day..)
|
|
44
|
+
.where(tracked_at: ..to_date.end_of_day)
|
|
43
45
|
end
|
|
44
46
|
|
|
45
47
|
def apply_exact_filter(relation, key)
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "llm_cost_tracker/ledger/schema/calls"
|
|
4
|
+
require "llm_cost_tracker/ledger/schema/call_line_items"
|
|
5
|
+
require "llm_cost_tracker/ledger/schema/call_tags"
|
|
6
|
+
require "llm_cost_tracker/ledger/schema/call_rollups"
|
|
7
|
+
|
|
8
|
+
module LlmCostTracker
|
|
9
|
+
module Dashboard
|
|
10
|
+
module SetupState
|
|
11
|
+
SetupRequired = Data.define(:message, :details)
|
|
12
|
+
DOCS_HINT = "See docs/upgrading.md for the migration path."
|
|
13
|
+
MUTEX = Mutex.new
|
|
14
|
+
|
|
15
|
+
CORE_SCHEMA_CHECKS = [
|
|
16
|
+
[
|
|
17
|
+
LlmCostTracker::Ledger::Schema::Calls,
|
|
18
|
+
"The llm_cost_tracker_calls table does not match the current LLM Cost Tracker schema."
|
|
19
|
+
],
|
|
20
|
+
[
|
|
21
|
+
LlmCostTracker::Ledger::Schema::CallLineItems,
|
|
22
|
+
"The llm_cost_tracker_call_line_items table does not match the current LLM Cost Tracker schema."
|
|
23
|
+
],
|
|
24
|
+
[
|
|
25
|
+
LlmCostTracker::Ledger::Schema::CallTags,
|
|
26
|
+
"The llm_cost_tracker_call_tags table does not match the current LLM Cost Tracker schema."
|
|
27
|
+
]
|
|
28
|
+
].freeze
|
|
29
|
+
|
|
30
|
+
OPTIONAL_CALL_ROLLUPS_CHECK = [
|
|
31
|
+
LlmCostTracker::Ledger::Schema::CallRollups,
|
|
32
|
+
"The llm_cost_tracker_call_rollups table does not match the current LLM Cost Tracker schema."
|
|
33
|
+
].freeze
|
|
34
|
+
|
|
35
|
+
private_constant :MUTEX, :CORE_SCHEMA_CHECKS, :OPTIONAL_CALL_ROLLUPS_CHECK, :DOCS_HINT
|
|
36
|
+
|
|
37
|
+
class << self
|
|
38
|
+
def current
|
|
39
|
+
return @cached if defined?(@cached)
|
|
40
|
+
|
|
41
|
+
MUTEX.synchronize do
|
|
42
|
+
@cached = compute unless defined?(@cached)
|
|
43
|
+
end
|
|
44
|
+
@cached
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def reset!
|
|
48
|
+
MUTEX.synchronize do
|
|
49
|
+
remove_instance_variable(:@cached) if defined?(@cached)
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
private
|
|
54
|
+
|
|
55
|
+
def compute
|
|
56
|
+
LlmCostTracker::Logging.debug("Dashboard::SetupState recomputing")
|
|
57
|
+
return calls_table_missing unless LlmCostTracker::Call.table_exists?
|
|
58
|
+
|
|
59
|
+
core_drift = drift_in(schema_checks_for_current_config)
|
|
60
|
+
return core_drift if core_drift
|
|
61
|
+
return nil unless LlmCostTracker.reconciliation_enabled?
|
|
62
|
+
|
|
63
|
+
reconciliation_drift
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def schema_checks_for_current_config
|
|
67
|
+
return CORE_SCHEMA_CHECKS unless LlmCostTracker.configuration.cache_rollups
|
|
68
|
+
|
|
69
|
+
CORE_SCHEMA_CHECKS + [OPTIONAL_CALL_ROLLUPS_CHECK]
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def drift_in(checks)
|
|
73
|
+
checks.each do |schema, message|
|
|
74
|
+
errors = schema.current_schema_errors
|
|
75
|
+
next if errors.empty?
|
|
76
|
+
|
|
77
|
+
return SetupRequired.new(message: message, details: errors + [DOCS_HINT])
|
|
78
|
+
end
|
|
79
|
+
nil
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
def reconciliation_drift
|
|
83
|
+
connection = ActiveRecord::Base.connection
|
|
84
|
+
LlmCostTracker::Reconciliation::SCHEMA_TABLES.each do |schema, table|
|
|
85
|
+
unless connection.data_source_exists?(table)
|
|
86
|
+
return SetupRequired.new(
|
|
87
|
+
message: "The #{table} table is required when reconciliation is enabled.",
|
|
88
|
+
details: ["run bin/rails generate llm_cost_tracker:reconciliation && bin/rails db:migrate", DOCS_HINT]
|
|
89
|
+
)
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
errors = schema.current_schema_errors
|
|
93
|
+
next if errors.empty?
|
|
94
|
+
|
|
95
|
+
message = "The #{table} table does not match the current LLM Cost Tracker schema."
|
|
96
|
+
return SetupRequired.new(message: message, details: errors + [DOCS_HINT])
|
|
97
|
+
end
|
|
98
|
+
nil
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
def calls_table_missing
|
|
102
|
+
SetupRequired.new(
|
|
103
|
+
message: "The llm_cost_tracker_calls table is not available yet.",
|
|
104
|
+
details: nil
|
|
105
|
+
)
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
end
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
<% body = capture { yield } %>
|
|
1
2
|
<!DOCTYPE html>
|
|
2
3
|
<html lang="en">
|
|
3
4
|
<head>
|
|
@@ -5,6 +6,7 @@
|
|
|
5
6
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
6
7
|
<title>LLM Cost Tracker</title>
|
|
7
8
|
<%= stylesheet_link_tag stylesheet_path %>
|
|
9
|
+
<%= inline_style_block %>
|
|
8
10
|
</head>
|
|
9
11
|
<body class="lct-body">
|
|
10
12
|
<div class="lct-app">
|
|
@@ -19,10 +21,13 @@
|
|
|
19
21
|
<%= link_to "Calls", calls_path, class: ("lct-active" if request.path.start_with?(calls_path)) %>
|
|
20
22
|
<%= link_to "Tags", tags_path, class: ("lct-active" if request.path.start_with?(tags_path)) %>
|
|
21
23
|
<%= link_to "Data Quality", data_quality_path, class: ("lct-active" if request.path.start_with?(data_quality_path)) %>
|
|
24
|
+
<% if LlmCostTracker.reconciliation_enabled? %>
|
|
25
|
+
<%= link_to "Reconciliation", reconciliation_path, class: ("lct-active" if request.path.start_with?(reconciliation_path)) %>
|
|
26
|
+
<% end %>
|
|
22
27
|
</nav>
|
|
23
28
|
</header>
|
|
24
29
|
|
|
25
|
-
<%=
|
|
30
|
+
<%= body %>
|
|
26
31
|
</main>
|
|
27
32
|
</div>
|
|
28
33
|
</body>
|
|
@@ -14,15 +14,19 @@ end %>
|
|
|
14
14
|
<% end %>
|
|
15
15
|
|
|
16
16
|
<section class="lct-panel">
|
|
17
|
-
<
|
|
17
|
+
<nav class="lct-breadcrumb" aria-label="Breadcrumb">
|
|
18
|
+
<%= link_to "Calls", calls_path, class: "lct-breadcrumb-link" %>
|
|
19
|
+
<span class="lct-breadcrumb-sep" aria-hidden="true">›</span>
|
|
20
|
+
<span class="lct-breadcrumb-current">#<%= @call.id %></span>
|
|
21
|
+
</nav>
|
|
18
22
|
<div class="lct-call-hero">
|
|
19
23
|
<div>
|
|
20
|
-
<h2 class="lct-section-title lct-call-title">
|
|
24
|
+
<h2 class="lct-section-title lct-call-title">
|
|
25
|
+
<code class="lct-code"><%= @call.model %></code>
|
|
26
|
+
</h2>
|
|
21
27
|
<p class="lct-call-subtitle">
|
|
22
28
|
<code class="lct-code"><%= @call.provider %></code>
|
|
23
29
|
<span>·</span>
|
|
24
|
-
<code class="lct-code"><%= @call.model %></code>
|
|
25
|
-
<span>·</span>
|
|
26
30
|
<span><%= format_date(@call.tracked_at) %></span>
|
|
27
31
|
</p>
|
|
28
32
|
</div>
|
|
@@ -65,45 +69,26 @@ end %>
|
|
|
65
69
|
|
|
66
70
|
<div class="lct-detail-grid">
|
|
67
71
|
<dl class="lct-dl">
|
|
68
|
-
<dt>Tracked At</dt>
|
|
69
|
-
<dd><%= format_date(@call.tracked_at) %></dd>
|
|
70
|
-
|
|
71
|
-
<dt>Provider</dt>
|
|
72
|
-
<dd><%= @call.provider %></dd>
|
|
73
|
-
|
|
74
|
-
<dt>Model</dt>
|
|
75
|
-
<dd><%= @call.model %></dd>
|
|
76
|
-
|
|
77
|
-
<dt>Pricing Status</dt>
|
|
78
|
-
<dd><%= pricing_status(@call) %></dd>
|
|
79
|
-
|
|
80
72
|
<dt>Cost Status</dt>
|
|
81
73
|
<dd><%= @call.cost_status.presence || "n/a" %></dd>
|
|
82
74
|
|
|
83
|
-
<dt>
|
|
84
|
-
<dd><%= @call.
|
|
85
|
-
|
|
86
|
-
<dt>Provider Project ID</dt>
|
|
87
|
-
<dd><%= @call.provider_project_id.presence || "n/a" %></dd>
|
|
88
|
-
|
|
89
|
-
<dt>Provider API Key ID</dt>
|
|
90
|
-
<dd><%= @call.provider_api_key_id.presence || "n/a" %></dd>
|
|
91
|
-
|
|
92
|
-
<dt>Provider Workspace ID</dt>
|
|
93
|
-
<dd><%= @call.provider_workspace_id.presence || "n/a" %></dd>
|
|
75
|
+
<dt>Latency</dt>
|
|
76
|
+
<dd><%= @call.latency_ms ? "#{optional_number(@call.latency_ms)}ms" : "n/a" %></dd>
|
|
94
77
|
|
|
95
78
|
<dt>Batch</dt>
|
|
96
79
|
<dd><%= @call.batch? ? "yes" : "no" %></dd>
|
|
97
80
|
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
<dd><%= format_date(@call.created_at) %></dd>
|
|
101
|
-
<% end %>
|
|
81
|
+
<dt>Response ID</dt>
|
|
82
|
+
<dd><%= @call.provider_response_id.presence || "n/a" %></dd>
|
|
102
83
|
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
84
|
+
<dt>Project ID</dt>
|
|
85
|
+
<dd><%= @call.provider_project_id.present? ? LlmCostTracker::Masking.mask_value(:provider_project_id, @call.provider_project_id) : "n/a" %></dd>
|
|
86
|
+
|
|
87
|
+
<dt>API Key ID</dt>
|
|
88
|
+
<dd><%= @call.provider_api_key_id.present? ? LlmCostTracker::Masking.mask_value(:provider_api_key_id, @call.provider_api_key_id) : "n/a" %></dd>
|
|
89
|
+
|
|
90
|
+
<dt>Workspace ID</dt>
|
|
91
|
+
<dd><%= @call.provider_workspace_id.present? ? LlmCostTracker::Masking.mask_value(:provider_workspace_id, @call.provider_workspace_id) : "n/a" %></dd>
|
|
107
92
|
</dl>
|
|
108
93
|
|
|
109
94
|
<dl class="lct-dl">
|
|
@@ -114,7 +99,9 @@ end %>
|
|
|
114
99
|
|
|
115
100
|
<dt>Total Tokens</dt>
|
|
116
101
|
<dd><%= number(@call.total_tokens) %></dd>
|
|
102
|
+
</dl>
|
|
117
103
|
|
|
104
|
+
<dl class="lct-dl">
|
|
118
105
|
<% priced_components.each do |component| %>
|
|
119
106
|
<dt><%= component.fetch(:label).titleize %> Cost</dt>
|
|
120
107
|
<dd><%= optional_money(line_item_costs_by_component[component.fetch(:price_key)]) %></dd>
|
|
@@ -122,14 +109,11 @@ end %>
|
|
|
122
109
|
|
|
123
110
|
<dt>Total Cost</dt>
|
|
124
111
|
<dd><%= optional_money(@call.total_cost) %></dd>
|
|
125
|
-
|
|
126
|
-
<dt>Latency</dt>
|
|
127
|
-
<dd><%= @call.latency_ms ? "#{optional_number(@call.latency_ms)}ms" : "n/a" %></dd>
|
|
128
112
|
</dl>
|
|
129
113
|
</div>
|
|
130
114
|
</section>
|
|
131
115
|
|
|
132
|
-
<% service_line_items = @call.line_items.
|
|
116
|
+
<% service_line_items = @call.line_items.reject { |li| li.unit == "token" }.sort_by(&:position) %>
|
|
133
117
|
<% if service_line_items.any? %>
|
|
134
118
|
<section class="lct-panel">
|
|
135
119
|
<h2 class="lct-section-title">Service Charges</h2>
|
|
@@ -152,7 +136,8 @@ end %>
|
|
|
152
136
|
<td><%= line_item.unit %></td>
|
|
153
137
|
<td class="lct-num"><%= line_item.quantity %></td>
|
|
154
138
|
<td class="lct-num"><%= line_item.rate_amount ? "#{optional_money(line_item.rate_amount)} / #{line_item.rate_quantity}" : "n/a" %></td>
|
|
155
|
-
|
|
139
|
+
<% unknown_cost = line_item.cost.nil? || line_item.cost_status.to_s == LlmCostTracker::Billing::CostStatus::UNKNOWN %>
|
|
140
|
+
<td class="lct-num<%= ' lct-num-muted' if unknown_cost %>"><%= unknown_cost ? "n/a" : money(line_item.cost) %></td>
|
|
156
141
|
<td><%= line_item.cost_status %></td>
|
|
157
142
|
</tr>
|
|
158
143
|
<% end %>
|
|
@@ -177,6 +162,6 @@ end %>
|
|
|
177
162
|
<% if @call.has_attribute?("metadata") %>
|
|
178
163
|
<section class="lct-panel">
|
|
179
164
|
<h2 class="lct-section-title">Metadata</h2>
|
|
180
|
-
<pre class="lct-pre"><%= safe_json(@call.read_attribute("metadata")) %></pre>
|
|
165
|
+
<pre class="lct-pre"><%= safe_json(LlmCostTracker::Masking.mask_hash(masked_metadata_hash(@call.read_attribute("metadata")))) %></pre>
|
|
181
166
|
</section>
|
|
182
167
|
<% end %>
|
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
<h2 class="lct-state-title">No LLM calls yet</h2>
|
|
15
15
|
<p class="lct-state-copy">Tracked requests will appear here after your application records its first LLM API call.</p>
|
|
16
16
|
<div class="lct-state-actions">
|
|
17
|
-
<%= link_to "
|
|
17
|
+
<%= link_to "Calls", calls_path, class: "lct-button lct-button-secondary" %>
|
|
18
18
|
</div>
|
|
19
19
|
</section>
|
|
20
20
|
<% else %>
|
|
@@ -46,7 +46,7 @@
|
|
|
46
46
|
<% end %>
|
|
47
47
|
</p>
|
|
48
48
|
</div>
|
|
49
|
-
<%= link_to "
|
|
49
|
+
<%= link_to "Calls",
|
|
50
50
|
calls_path(current_query(provider: @spend_anomaly.fetch(:provider), model: @spend_anomaly.fetch(:model), from: @to_date.iso8601, to: @to_date.iso8601, page: nil, per: nil, format: nil)),
|
|
51
51
|
class: "lct-button lct-button-secondary" %>
|
|
52
52
|
</aside>
|
|
@@ -64,6 +64,11 @@
|
|
|
64
64
|
|
|
65
65
|
<div class="lct-hero-side">
|
|
66
66
|
<div class="lct-stat-grid">
|
|
67
|
+
<article class="lct-stat">
|
|
68
|
+
<p class="lct-stat-label">Avg cost / call</p>
|
|
69
|
+
<p class="lct-stat-value"><%= money(@stats.average_cost_per_call) %></p>
|
|
70
|
+
</article>
|
|
71
|
+
|
|
67
72
|
<article class="lct-stat">
|
|
68
73
|
<p class="lct-stat-label">Calls</p>
|
|
69
74
|
<p class="lct-stat-value"><%= number(@stats.total_calls) %></p>
|
|
@@ -71,11 +76,6 @@
|
|
|
71
76
|
<span class="<%= badge[:css_class] %>"><span class="lct-delta-dot" aria-hidden="true"></span><%= badge[:text] %></span>
|
|
72
77
|
</article>
|
|
73
78
|
|
|
74
|
-
<article class="lct-stat">
|
|
75
|
-
<p class="lct-stat-label">Avg cost / call</p>
|
|
76
|
-
<p class="lct-stat-value"><%= money(@stats.average_cost_per_call) %></p>
|
|
77
|
-
</article>
|
|
78
|
-
|
|
79
79
|
<% if @stats.average_latency_ms %>
|
|
80
80
|
<article class="lct-stat">
|
|
81
81
|
<p class="lct-stat-label">Avg latency</p>
|
|
@@ -103,9 +103,9 @@
|
|
|
103
103
|
<span class="lct-budget-percent"><%= percent(budget[:percent_used]) %></span>
|
|
104
104
|
</div>
|
|
105
105
|
<div class="lct-budget-track" role="progressbar" aria-valuemin="0" aria-valuemax="100" aria-valuenow="<%= budget[:progress_percent].round %>">
|
|
106
|
-
<div
|
|
106
|
+
<div data-lct-style="<%= inline_style("width: #{budget[:progress_percent]}%") %>" class="lct-budget-fill <%= budget[:fill_modifier] %>"></div>
|
|
107
107
|
<% if budget[:projected_spent].positive? %>
|
|
108
|
-
<span
|
|
108
|
+
<span data-lct-style="<%= inline_style("left: calc(#{budget[:projected_marker_percent]}% - 1px)") %>" class="lct-budget-marker" aria-hidden="true"></span>
|
|
109
109
|
<% end %>
|
|
110
110
|
</div>
|
|
111
111
|
<p class="lct-budget-projection">
|