llm_cost_tracker 0.5.1 → 0.5.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +43 -0
- data/README.md +18 -9
- data/app/controllers/llm_cost_tracker/calls_controller.rb +2 -1
- data/app/controllers/llm_cost_tracker/dashboard_controller.rb +3 -15
- data/app/controllers/llm_cost_tracker/tags_controller.rb +7 -6
- data/app/helpers/llm_cost_tracker/application_helper.rb +21 -6
- data/app/helpers/llm_cost_tracker/dashboard_filter_options_helper.rb +3 -1
- data/app/services/llm_cost_tracker/dashboard/date_range.rb +42 -0
- data/app/services/llm_cost_tracker/dashboard/filter.rb +6 -8
- data/app/services/llm_cost_tracker/dashboard/spend_anomaly.rb +6 -5
- data/app/services/llm_cost_tracker/dashboard/tag_breakdown.rb +74 -18
- data/app/services/llm_cost_tracker/dashboard/tag_key_explorer.rb +15 -4
- data/app/views/llm_cost_tracker/shared/_tag_chips.html.erb +1 -1
- data/app/views/llm_cost_tracker/tags/show.html.erb +4 -0
- data/docs/architecture.md +28 -0
- data/docs/budgets.md +45 -0
- data/docs/configuration.md +65 -0
- data/docs/cookbook.md +185 -0
- data/docs/dashboard-overview.png +0 -0
- data/docs/dashboard.md +38 -0
- data/docs/extending.md +32 -0
- data/docs/operations.md +44 -0
- data/docs/pricing.md +94 -0
- data/docs/querying.md +36 -0
- data/docs/streaming.md +70 -0
- data/docs/technical/README.md +10 -0
- data/docs/technical/data-flow.md +67 -0
- data/docs/technical/extension-points.md +111 -0
- data/docs/technical/module-map.md +197 -0
- data/docs/technical/operational-notes.md +77 -0
- data/docs/upgrading.md +46 -0
- data/lib/llm_cost_tracker/capture_verifier.rb +71 -0
- data/lib/llm_cost_tracker/configuration/instrumentation.rb +1 -1
- data/lib/llm_cost_tracker/configuration/storage_backend.rb +26 -0
- data/lib/llm_cost_tracker/configuration.rb +24 -17
- data/lib/llm_cost_tracker/doctor/capture_check.rb +39 -0
- data/lib/llm_cost_tracker/doctor.rb +6 -1
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/install_generator.rb +1 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/initializer.rb.erb +7 -1
- data/lib/llm_cost_tracker/integrations/anthropic.rb +51 -3
- data/lib/llm_cost_tracker/integrations/base.rb +77 -6
- data/lib/llm_cost_tracker/integrations/object_reader.rb +1 -1
- data/lib/llm_cost_tracker/integrations/openai.rb +78 -5
- data/lib/llm_cost_tracker/integrations/registry.rb +36 -4
- data/lib/llm_cost_tracker/integrations/ruby_llm.rb +171 -0
- data/lib/llm_cost_tracker/integrations/stream_tracker.rb +166 -0
- data/lib/llm_cost_tracker/llm_api_call.rb +2 -77
- data/lib/llm_cost_tracker/llm_api_call_metrics.rb +63 -0
- data/lib/llm_cost_tracker/middleware/faraday.rb +8 -4
- data/lib/llm_cost_tracker/parsers/gemini.rb +8 -1
- data/lib/llm_cost_tracker/parsers/openai_usage.rb +12 -3
- data/lib/llm_cost_tracker/price_registry.rb +3 -0
- data/lib/llm_cost_tracker/price_sync/fetcher.rb +41 -12
- data/lib/llm_cost_tracker/price_sync/registry_loader.rb +6 -0
- data/lib/llm_cost_tracker/pricing/effective_prices.rb +75 -0
- data/lib/llm_cost_tracker/pricing/explainer.rb +77 -0
- data/lib/llm_cost_tracker/pricing/lookup.rb +110 -0
- data/lib/llm_cost_tracker/pricing.rb +25 -108
- data/lib/llm_cost_tracker/report.rb +8 -1
- data/lib/llm_cost_tracker/report_data.rb +25 -9
- data/lib/llm_cost_tracker/retention.rb +33 -16
- data/lib/llm_cost_tracker/storage/active_record_backend.rb +115 -0
- data/lib/llm_cost_tracker/storage/active_record_rollups.rb +42 -0
- data/lib/llm_cost_tracker/storage/active_record_store.rb +26 -0
- data/lib/llm_cost_tracker/storage/custom_backend.rb +32 -0
- data/lib/llm_cost_tracker/storage/dispatcher.rb +11 -34
- data/lib/llm_cost_tracker/storage/log_backend.rb +38 -0
- data/lib/llm_cost_tracker/storage/registry.rb +63 -0
- data/lib/llm_cost_tracker/stream_capture.rb +7 -0
- data/lib/llm_cost_tracker/stream_collector.rb +25 -1
- data/lib/llm_cost_tracker/tag_sanitizer.rb +81 -0
- data/lib/llm_cost_tracker/tag_sql.rb +34 -0
- data/lib/llm_cost_tracker/tracker.rb +6 -2
- data/lib/llm_cost_tracker/version.rb +1 -1
- data/lib/llm_cost_tracker.rb +4 -0
- data/lib/tasks/llm_cost_tracker.rake +49 -0
- metadata +40 -6
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
# Extension Points
|
|
2
|
+
|
|
3
|
+
Extensions should plug into existing provider-agnostic boundaries. If a new feature needs a provider-specific branch outside ingestion code, revisit the design first.
|
|
4
|
+
|
|
5
|
+
## Custom Parsers
|
|
6
|
+
|
|
7
|
+
Use parser registration when a provider or gateway has a response shape the built-ins do not cover.
|
|
8
|
+
|
|
9
|
+
Expected parser contract:
|
|
10
|
+
|
|
11
|
+
- `match?(url)` detects supported request URLs.
|
|
12
|
+
- `parse(request_url, request_body, response_status, response_body)` returns `ParsedUsage` or `nil`.
|
|
13
|
+
- `parse_stream(request_url, request_body, response_status, events)` returns `ParsedUsage` or `nil`.
|
|
14
|
+
- `streaming_request?(request_url, request_body)` detects streaming requests when the provider does not use a simple `stream: true` field.
|
|
15
|
+
- `provider_names` returns provider names that can be used by `track_stream(provider: ...)`.
|
|
16
|
+
|
|
17
|
+
Use `Parsers::Base` helpers for URL matching and stream-event extraction. Use `Parsers::OpenaiUsage` only for OpenAI-shaped usage hashes.
|
|
18
|
+
|
|
19
|
+
## SDK Integrations
|
|
20
|
+
|
|
21
|
+
Use SDK integrations when a popular Ruby client does not expose a Faraday middleware stack but returns stable usage objects. RubyLLM and the official `openai` and `anthropic` gems qualify. Faraday-based clients that expose a middleware hook, such as `ruby-openai`'s constructor block, are covered by the Faraday middleware instead. Clients with no stable hook must use the explicit `track` / `track_stream` fallback until an integration exists.
|
|
22
|
+
|
|
23
|
+
Expected integration contract:
|
|
24
|
+
|
|
25
|
+
- no hard dependency on the provider SDK
|
|
26
|
+
- fail-fast boot when an explicitly enabled SDK is missing or below the minimum supported version
|
|
27
|
+
- install-time checks for the target classes and methods
|
|
28
|
+
- idempotent `Module#prepend` around narrow resource methods
|
|
29
|
+
- no tracking when the integration is not enabled in configuration
|
|
30
|
+
- canonical usage fields passed to `Tracker.record`
|
|
31
|
+
|
|
32
|
+
SDK integrations belong under `LlmCostTracker::Integrations`. Do not put SDK object-shape handling in parsers, storage, or pricing.
|
|
33
|
+
|
|
34
|
+
External integrations can register an adapter with
|
|
35
|
+
`LlmCostTracker::Integrations.register(:name, adapter)`. The adapter must
|
|
36
|
+
respond to `install` and `status`, and enabled names are still selected through
|
|
37
|
+
`config.instrument`.
|
|
38
|
+
|
|
39
|
+
## OpenAI-Compatible Gateways
|
|
40
|
+
|
|
41
|
+
Use `config.openai_compatible_providers` when a gateway speaks the OpenAI request and response shape.
|
|
42
|
+
|
|
43
|
+
This is for shape compatibility, not pricing. Gateway-specific model IDs or discounts belong in `prices_file` or `pricing_overrides`.
|
|
44
|
+
|
|
45
|
+
## Prices
|
|
46
|
+
|
|
47
|
+
Use `config.prices_file` for the app's source-controlled price snapshot.
|
|
48
|
+
|
|
49
|
+
Use `config.pricing_overrides` for urgent or environment-specific overrides that are easier to keep in Ruby.
|
|
50
|
+
|
|
51
|
+
Supported canonical keys:
|
|
52
|
+
|
|
53
|
+
- `input`
|
|
54
|
+
- `output`
|
|
55
|
+
- `cache_read_input`
|
|
56
|
+
- `cache_write_input`
|
|
57
|
+
- `batch_input`
|
|
58
|
+
- `batch_output`
|
|
59
|
+
- mode-prefixed keys such as `priority_input` or `batch_cache_read_input`
|
|
60
|
+
|
|
61
|
+
Provider-specific pricing details must be translated before they reach runtime pricing.
|
|
62
|
+
|
|
63
|
+
## Tags
|
|
64
|
+
|
|
65
|
+
Tags are the extension point for application attribution:
|
|
66
|
+
|
|
67
|
+
- tenant
|
|
68
|
+
- user
|
|
69
|
+
- feature
|
|
70
|
+
- trace
|
|
71
|
+
- job
|
|
72
|
+
- workflow
|
|
73
|
+
- agent session
|
|
74
|
+
|
|
75
|
+
Use `config.default_tags`, middleware `tags:`, explicit metadata, and `LlmCostTracker.with_tags`. Do not add first-class columns for app dimensions unless the ledger needs that field for provider-agnostic billing behavior.
|
|
76
|
+
|
|
77
|
+
## Storage
|
|
78
|
+
|
|
79
|
+
Use `storage_backend = :custom` only when the host app needs to own persistence completely.
|
|
80
|
+
|
|
81
|
+
Custom storage receives a canonical `Event`. Returning `false` tells the tracker not to run budget checks for that event.
|
|
82
|
+
|
|
83
|
+
ActiveRecord storage is the production path for dashboards and cross-process budgets.
|
|
84
|
+
|
|
85
|
+
Storage adapters can register with
|
|
86
|
+
`LlmCostTracker::Storage.register(:name, backend)`. A backend must respond to
|
|
87
|
+
`save(event)` and may expose `verify` for capture diagnostics.
|
|
88
|
+
|
|
89
|
+
## Dashboard
|
|
90
|
+
|
|
91
|
+
Dashboard additions should be read-only services under `app/services/llm_cost_tracker/dashboard`.
|
|
92
|
+
|
|
93
|
+
Keep controller actions thin:
|
|
94
|
+
|
|
95
|
+
- parse params
|
|
96
|
+
- build filtered scope
|
|
97
|
+
- call services
|
|
98
|
+
- render views
|
|
99
|
+
|
|
100
|
+
Keep view logic in helpers when it is reused across pages. Do not add JavaScript for dashboard behavior.
|
|
101
|
+
|
|
102
|
+
## Generators
|
|
103
|
+
|
|
104
|
+
Generators are installation contracts. New generator behavior should be:
|
|
105
|
+
|
|
106
|
+
- additive when possible
|
|
107
|
+
- idempotent where Rails generator APIs allow it
|
|
108
|
+
- explicit about destructive or table-rewriting operations
|
|
109
|
+
- covered by generator template specs
|
|
110
|
+
|
|
111
|
+
Fresh install templates and upgrade generators should stay aligned. If a fresh install gains a column or index, the upgrade path needs a generator unless the next release intentionally makes a breaking install path.
|
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
# Module Map
|
|
2
|
+
|
|
3
|
+
LLM Cost Tracker is organized around a small set of durable responsibilities. File layout does not need to mirror these modules perfectly, but new code should fit one of these boundaries.
|
|
4
|
+
|
|
5
|
+
## Public API and Configuration
|
|
6
|
+
|
|
7
|
+
Primary files:
|
|
8
|
+
|
|
9
|
+
- `lib/llm_cost_tracker.rb`
|
|
10
|
+
- `lib/llm_cost_tracker/configuration.rb`
|
|
11
|
+
- `lib/llm_cost_tracker/tag_context.rb`
|
|
12
|
+
- `lib/llm_cost_tracker/doctor.rb`
|
|
13
|
+
- `lib/llm_cost_tracker/logging.rb`
|
|
14
|
+
- `lib/llm_cost_tracker/errors.rb`
|
|
15
|
+
|
|
16
|
+
Responsibilities:
|
|
17
|
+
|
|
18
|
+
- Expose `configure`, `track`, `track_stream`, `with_tags`, and `enforce_budget!`.
|
|
19
|
+
- Keep configuration immutable after `configure` returns.
|
|
20
|
+
- Merge scoped tags and default tags without leaking state across threads.
|
|
21
|
+
- Report installation and pricing health through `llm_cost_tracker:doctor`.
|
|
22
|
+
|
|
23
|
+
This module should stay small. It can orchestrate other modules, but it should not contain provider parsing, SQL details, dashboard aggregation, or pricing-source logic.
|
|
24
|
+
|
|
25
|
+
## SDK Integrations
|
|
26
|
+
|
|
27
|
+
Primary files:
|
|
28
|
+
|
|
29
|
+
- `lib/llm_cost_tracker/integrations/*`
|
|
30
|
+
|
|
31
|
+
Responsibilities:
|
|
32
|
+
|
|
33
|
+
- Add optional instrumentation for Ruby SDKs without adding provider SDK dependencies.
|
|
34
|
+
- Install narrow, idempotent `Module#prepend` wrappers around stable SDK resource methods.
|
|
35
|
+
- Extract SDK response objects into canonical usage fields.
|
|
36
|
+
- Keep SDK-specific object handling out of `Tracker` and storage.
|
|
37
|
+
|
|
38
|
+
Integrations are for Ruby SDK object shapes. Parsers are for HTTP and stream payload shapes.
|
|
39
|
+
|
|
40
|
+
## Ingestion
|
|
41
|
+
|
|
42
|
+
Primary files:
|
|
43
|
+
|
|
44
|
+
- `lib/llm_cost_tracker/middleware/faraday.rb`
|
|
45
|
+
- `lib/llm_cost_tracker/stream_collector.rb`
|
|
46
|
+
- `lib/llm_cost_tracker/parsed_usage.rb`
|
|
47
|
+
- `lib/llm_cost_tracker/request_url.rb`
|
|
48
|
+
- `lib/llm_cost_tracker/parsers/*`
|
|
49
|
+
|
|
50
|
+
Responsibilities:
|
|
51
|
+
|
|
52
|
+
- Detect supported LLM HTTP requests.
|
|
53
|
+
- Parse provider responses and stream events into `ParsedUsage`.
|
|
54
|
+
- Translate provider-specific fields into canonical usage fields.
|
|
55
|
+
- Preserve app streaming behavior while teeing events for tracking.
|
|
56
|
+
|
|
57
|
+
Provider-specific code belongs here. The output boundary is `ParsedUsage`, not raw provider JSON.
|
|
58
|
+
|
|
59
|
+
## Canonical Ledger
|
|
60
|
+
|
|
61
|
+
Primary files:
|
|
62
|
+
|
|
63
|
+
- `lib/llm_cost_tracker/tracker.rb`
|
|
64
|
+
- `lib/llm_cost_tracker/event.rb`
|
|
65
|
+
- `lib/llm_cost_tracker/event_metadata.rb`
|
|
66
|
+
- `lib/llm_cost_tracker/usage_breakdown.rb`
|
|
67
|
+
- `lib/llm_cost_tracker/cost.rb`
|
|
68
|
+
- `lib/llm_cost_tracker/unknown_pricing.rb`
|
|
69
|
+
|
|
70
|
+
Responsibilities:
|
|
71
|
+
|
|
72
|
+
- Normalize provider, model, usage, tags, latency, streaming flags, and response IDs.
|
|
73
|
+
- Price canonical usage through `Pricing`.
|
|
74
|
+
- Emit `ActiveSupport::Notifications`.
|
|
75
|
+
- Persist the event through the configured storage backend.
|
|
76
|
+
- Run budget checks after successful storage.
|
|
77
|
+
|
|
78
|
+
This module must remain provider-agnostic. It should never branch on a specific provider model family.
|
|
79
|
+
|
|
80
|
+
## Pricing
|
|
81
|
+
|
|
82
|
+
Primary files:
|
|
83
|
+
|
|
84
|
+
- `lib/llm_cost_tracker/pricing.rb`
|
|
85
|
+
- `lib/llm_cost_tracker/price_registry.rb`
|
|
86
|
+
- `lib/llm_cost_tracker/price_freshness.rb`
|
|
87
|
+
- `lib/llm_cost_tracker/prices.json`
|
|
88
|
+
- `lib/llm_cost_tracker/price_sync/*`
|
|
89
|
+
- `lib/tasks/llm_cost_tracker.rake`
|
|
90
|
+
|
|
91
|
+
Responsibilities:
|
|
92
|
+
|
|
93
|
+
- Load bundled prices, local price snapshots, and Ruby overrides.
|
|
94
|
+
- Apply pricing precedence: `pricing_overrides`, `prices_file`, bundled prices.
|
|
95
|
+
- Calculate costs from canonical usage fields.
|
|
96
|
+
- Update local snapshots from the maintained LLM Cost Tracker price registry.
|
|
97
|
+
- Validate snapshot schema compatibility, gem-version compatibility, and price entry shape.
|
|
98
|
+
|
|
99
|
+
Pricing refresh must not perform boot-time or request-time network work. Runtime pricing uses bundled prices, local files, and in-memory caches.
|
|
100
|
+
|
|
101
|
+
## Storage
|
|
102
|
+
|
|
103
|
+
Primary files:
|
|
104
|
+
|
|
105
|
+
- `lib/llm_cost_tracker/llm_api_call.rb`
|
|
106
|
+
- `lib/llm_cost_tracker/period_total.rb`
|
|
107
|
+
- `lib/llm_cost_tracker/llm_api_call_metrics.rb`
|
|
108
|
+
- `lib/llm_cost_tracker/storage/active_record_store.rb`
|
|
109
|
+
- `lib/llm_cost_tracker/storage/active_record_rollups.rb`
|
|
110
|
+
- `lib/llm_cost_tracker/storage/registry.rb`
|
|
111
|
+
- `lib/llm_cost_tracker/tags_column.rb`
|
|
112
|
+
- `lib/llm_cost_tracker/tag_key.rb`
|
|
113
|
+
- `lib/llm_cost_tracker/tag_sql.rb`
|
|
114
|
+
- `lib/llm_cost_tracker/tag_query.rb`
|
|
115
|
+
- `lib/llm_cost_tracker/tag_accessors.rb`
|
|
116
|
+
- `lib/llm_cost_tracker/period_grouping.rb`
|
|
117
|
+
|
|
118
|
+
Responsibilities:
|
|
119
|
+
|
|
120
|
+
- Persist canonical events into ActiveRecord.
|
|
121
|
+
- Hide database-specific tag storage differences.
|
|
122
|
+
- Maintain period rollups for hot-path budget reads.
|
|
123
|
+
- Provide safe scopes for filters, periods, tags, unknown pricing, and reports.
|
|
124
|
+
|
|
125
|
+
Storage can know about database adapters and optional columns. It should not parse provider responses or fetch price data.
|
|
126
|
+
|
|
127
|
+
## Budgets and Retention
|
|
128
|
+
|
|
129
|
+
Primary files:
|
|
130
|
+
|
|
131
|
+
- `lib/llm_cost_tracker/budget.rb`
|
|
132
|
+
- `lib/llm_cost_tracker/retention.rb`
|
|
133
|
+
- `lib/llm_cost_tracker/storage/active_record_rollups.rb`
|
|
134
|
+
|
|
135
|
+
Responsibilities:
|
|
136
|
+
|
|
137
|
+
- Enforce monthly, daily, and per-call guardrails.
|
|
138
|
+
- Support preflight blocking where ActiveRecord rollups are available.
|
|
139
|
+
- Prune old ledger rows in batches.
|
|
140
|
+
- Keep budget checks bounded by maintained aggregates, not by full ledger scans.
|
|
141
|
+
|
|
142
|
+
Budget behavior is part of the hot path. Any change here must be measured against per-request overhead.
|
|
143
|
+
|
|
144
|
+
## Dashboard and Reporting
|
|
145
|
+
|
|
146
|
+
Primary files:
|
|
147
|
+
|
|
148
|
+
- `lib/llm_cost_tracker/report*.rb`
|
|
149
|
+
- `app/controllers/llm_cost_tracker/*`
|
|
150
|
+
- `app/services/llm_cost_tracker/dashboard/*`
|
|
151
|
+
- `app/helpers/llm_cost_tracker/*`
|
|
152
|
+
- `app/views/llm_cost_tracker/*`
|
|
153
|
+
- `app/assets/llm_cost_tracker/application.css`
|
|
154
|
+
|
|
155
|
+
Responsibilities:
|
|
156
|
+
|
|
157
|
+
- Render server-side dashboard pages.
|
|
158
|
+
- Aggregate spend, calls, providers, models, tags, latency, and data quality.
|
|
159
|
+
- Export filtered calls as CSV.
|
|
160
|
+
- Keep dashboard queries explicit and indexed.
|
|
161
|
+
|
|
162
|
+
Dashboard code may run grouped SQL because it is user-initiated reporting. It must stay server-rendered and must not introduce a JavaScript bundle.
|
|
163
|
+
|
|
164
|
+
## Rails Integration and Generators
|
|
165
|
+
|
|
166
|
+
Primary files:
|
|
167
|
+
|
|
168
|
+
- `lib/llm_cost_tracker/railtie.rb`
|
|
169
|
+
- `lib/llm_cost_tracker/engine.rb`
|
|
170
|
+
- `lib/llm_cost_tracker/assets.rb`
|
|
171
|
+
- `lib/llm_cost_tracker/generators/llm_cost_tracker/*`
|
|
172
|
+
- `config/routes.rb`
|
|
173
|
+
|
|
174
|
+
Responsibilities:
|
|
175
|
+
|
|
176
|
+
- Register rake tasks and Faraday middleware.
|
|
177
|
+
- Mount the isolated Rails engine.
|
|
178
|
+
- Generate migrations, initializer, dashboard route, and local price snapshots.
|
|
179
|
+
- Serve dashboard CSS as a fingerprinted engine asset.
|
|
180
|
+
|
|
181
|
+
Generator templates are public installation contracts. Treat them like API.
|
|
182
|
+
|
|
183
|
+
## Test Suites
|
|
184
|
+
|
|
185
|
+
Primary files:
|
|
186
|
+
|
|
187
|
+
- `spec/llm_cost_tracker/*`
|
|
188
|
+
- `spec/llm_cost_tracker/engine/*`
|
|
189
|
+
- `spec/llm_cost_tracker/dashboard/*`
|
|
190
|
+
- `spec/fixtures/pricing/*`
|
|
191
|
+
- `spec/support/*`
|
|
192
|
+
|
|
193
|
+
Responsibilities:
|
|
194
|
+
|
|
195
|
+
- Cover canonical behavior, parser boundaries, pricing precedence, storage rollups, dashboard rendering, generators, and concurrency.
|
|
196
|
+
- Keep request specs plain and stable.
|
|
197
|
+
- Run through `bin/check` before release work or commits that touch code.
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
# Operational Notes
|
|
2
|
+
|
|
3
|
+
This file describes runtime constraints that should shape implementation decisions.
|
|
4
|
+
|
|
5
|
+
## Hot Paths
|
|
6
|
+
|
|
7
|
+
Hot-path code includes:
|
|
8
|
+
|
|
9
|
+
- Faraday middleware request and response handling
|
|
10
|
+
- stream collection
|
|
11
|
+
- `Tracker.record`
|
|
12
|
+
- `Pricing.cost_for`
|
|
13
|
+
- ActiveRecord event persistence
|
|
14
|
+
- budget checks
|
|
15
|
+
|
|
16
|
+
Hot-path code must avoid:
|
|
17
|
+
|
|
18
|
+
- network calls
|
|
19
|
+
- per-event schema discovery beyond memoized checks
|
|
20
|
+
- full ledger aggregation
|
|
21
|
+
- unbounded stream buffers
|
|
22
|
+
- N+1 queries
|
|
23
|
+
- price-refresh work
|
|
24
|
+
|
|
25
|
+
## Pricing Freshness
|
|
26
|
+
|
|
27
|
+
Runtime pricing is local:
|
|
28
|
+
|
|
29
|
+
1. Ruby overrides
|
|
30
|
+
2. configured local price snapshot
|
|
31
|
+
3. bundled prices
|
|
32
|
+
|
|
33
|
+
Price update tasks are operational tooling. They can fetch the maintained LLM Cost Tracker price snapshot because the operator runs them intentionally. Request tracking must never depend on live provider pricing pages.
|
|
34
|
+
|
|
35
|
+
## Budget Reads
|
|
36
|
+
|
|
37
|
+
Monthly and daily budgets should read `llm_cost_tracker_period_totals` when the table exists. Falling back to summing `llm_api_calls` is an upgrade compatibility path, not the preferred production path.
|
|
38
|
+
|
|
39
|
+
Per-call budgets are checked from the current event only.
|
|
40
|
+
|
|
41
|
+
## Retention
|
|
42
|
+
|
|
43
|
+
Retention may delete old `llm_api_calls`. Period rollups are the durable budget aggregate. Any migration or refactor that changes rollups must preserve the meaning of retained totals or clearly document a breaking change.
|
|
44
|
+
|
|
45
|
+
## Optional Columns
|
|
46
|
+
|
|
47
|
+
The gem supports upgrade paths where older apps may not have every column yet. Optional column checks must be memoized and refreshed when ActiveRecord column information is reset.
|
|
48
|
+
|
|
49
|
+
Do not put table or column checks directly in loops that run for every event without caching.
|
|
50
|
+
|
|
51
|
+
## Dashboard Queries
|
|
52
|
+
|
|
53
|
+
Dashboard queries can aggregate because they are user-initiated. They should still use:
|
|
54
|
+
|
|
55
|
+
- filtered scopes
|
|
56
|
+
- bounded pagination
|
|
57
|
+
- database-side grouping
|
|
58
|
+
- indexes that match common filters
|
|
59
|
+
- single aggregate queries for related counters
|
|
60
|
+
|
|
61
|
+
Avoid loading ledger rows into Ruby just to count, sum, group, or sort.
|
|
62
|
+
|
|
63
|
+
## Streaming
|
|
64
|
+
|
|
65
|
+
Streaming capture must keep the host app's stream behavior intact.
|
|
66
|
+
|
|
67
|
+
The middleware should collect enough data to parse final usage while bounding memory. When usage never arrives or capture overflows, record an unknown-usage event so Data Quality can surface the gap.
|
|
68
|
+
|
|
69
|
+
## Release Checks
|
|
70
|
+
|
|
71
|
+
Run `bin/check` before committing code changes intended for release. It includes full RuboCop, full RSpec, project coverage, and patch coverage for the current diff.
|
|
72
|
+
|
|
73
|
+
Project coverage defaults to the Codecov target. Patch coverage defaults to 95% so local checks stay stricter than Codecov parser differences. Thresholds can be adjusted locally with `PROJECT_COVERAGE_MIN`, `PATCH_COVERAGE_MIN`, or `COVERAGE_BASE`.
|
|
74
|
+
|
|
75
|
+
For the closest match to the Codecov upload job, run `BUNDLE_GEMFILE=gemfiles/rails_8_1.gemfile bin/check`.
|
|
76
|
+
|
|
77
|
+
Docs-only changes do not require the full suite, but any code, generator, migration, parser, pricing, dashboard, or storage change does.
|
data/docs/upgrading.md
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
# Upgrading
|
|
2
|
+
|
|
3
|
+
LLM Cost Tracker is still moving quickly, so upgrades should be explicit:
|
|
4
|
+
inspect the changelog, run doctor, and apply only the generators your schema is
|
|
5
|
+
missing.
|
|
6
|
+
|
|
7
|
+
The version-by-version upgrade guide is moving here from the README.
|
|
8
|
+
|
|
9
|
+
## Canonical Sources
|
|
10
|
+
|
|
11
|
+
Until this page is expanded, use:
|
|
12
|
+
|
|
13
|
+
- [Changelog](../CHANGELOG.md)
|
|
14
|
+
- [Quickstart](../README.md#quickstart)
|
|
15
|
+
- [Operations](operations.md)
|
|
16
|
+
|
|
17
|
+
## Schema Generators
|
|
18
|
+
|
|
19
|
+
Existing installs can add newer optional columns through focused generators:
|
|
20
|
+
|
|
21
|
+
```bash
|
|
22
|
+
bin/rails generate llm_cost_tracker:add_period_totals
|
|
23
|
+
bin/rails generate llm_cost_tracker:add_streaming
|
|
24
|
+
bin/rails generate llm_cost_tracker:add_provider_response_id
|
|
25
|
+
bin/rails generate llm_cost_tracker:add_usage_breakdown
|
|
26
|
+
bin/rails generate llm_cost_tracker:upgrade_tags_to_jsonb
|
|
27
|
+
bin/rails generate llm_cost_tracker:upgrade_cost_precision
|
|
28
|
+
bin/rails generate llm_cost_tracker:add_latency_ms
|
|
29
|
+
bin/rails db:migrate
|
|
30
|
+
bin/rails llm_cost_tracker:doctor
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
On PostgreSQL, `upgrade_tags_to_jsonb` rewrites `llm_api_calls`. For large
|
|
34
|
+
tables, run it during a maintenance window or replace it with a two-phase
|
|
35
|
+
backfill.
|
|
36
|
+
|
|
37
|
+
## Upgrade Habit
|
|
38
|
+
|
|
39
|
+
Run:
|
|
40
|
+
|
|
41
|
+
```bash
|
|
42
|
+
bin/rails llm_cost_tracker:doctor
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
Doctor tells you which optional columns and production-hardening pieces are still
|
|
46
|
+
missing.
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "storage/dispatcher"
|
|
4
|
+
|
|
5
|
+
module LlmCostTracker
|
|
6
|
+
class CaptureVerifier
|
|
7
|
+
Check = Data.define(:status, :name, :message)
|
|
8
|
+
|
|
9
|
+
class << self
|
|
10
|
+
def call = new.checks
|
|
11
|
+
|
|
12
|
+
def report(checks = call)
|
|
13
|
+
(["LLM Cost Tracker capture verification"] + checks.map { |check| format_check(check) }).join("\n")
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def healthy?(checks = call)
|
|
17
|
+
checks.none? { |check| check.status == :error }
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
private
|
|
21
|
+
|
|
22
|
+
def format_check(check)
|
|
23
|
+
"[#{check.status}] #{check.name}: #{check.message}"
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def checks
|
|
28
|
+
[
|
|
29
|
+
enabled_check,
|
|
30
|
+
*integration_checks,
|
|
31
|
+
*storage_checks
|
|
32
|
+
].compact
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
private
|
|
36
|
+
|
|
37
|
+
def enabled_check
|
|
38
|
+
return Check.new(:ok, "tracking", "enabled") if LlmCostTracker.configuration.enabled
|
|
39
|
+
|
|
40
|
+
Check.new(:error, "tracking", "disabled; set config.enabled = true before verifying capture")
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def integration_checks
|
|
44
|
+
enabled = LlmCostTracker.configuration.instrumented_integrations
|
|
45
|
+
if enabled.empty?
|
|
46
|
+
return [
|
|
47
|
+
Check.new(:ok, "sdk integrations", "none enabled; Faraday middleware and manual capture remain available")
|
|
48
|
+
]
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
LlmCostTracker::Integrations.checks.map do |check|
|
|
52
|
+
Check.new(check.status, "sdk integration #{check.name}", check.message)
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def storage_checks
|
|
57
|
+
backend = LlmCostTracker::Storage::Registry.fetch(LlmCostTracker.configuration.storage_backend)
|
|
58
|
+
unless backend.respond_to?(:verify)
|
|
59
|
+
return [
|
|
60
|
+
Check.new(:warn, "storage", "#{LlmCostTracker.configuration.storage_backend} backend has no verifier")
|
|
61
|
+
]
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
backend.verify.map do |check|
|
|
65
|
+
Check.new(check.status, check.name, check.message)
|
|
66
|
+
end
|
|
67
|
+
rescue LlmCostTracker::Error => e
|
|
68
|
+
[Check.new(:error, "storage", e.message)]
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
end
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module LlmCostTracker
|
|
4
|
+
module ConfigurationStorageBackend
|
|
5
|
+
def storage_backend=(value)
|
|
6
|
+
ensure_shared_configuration_mutable!
|
|
7
|
+
@storage_backend = normalize_storage_backend(value)
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
private
|
|
11
|
+
|
|
12
|
+
def normalize_storage_backend(value)
|
|
13
|
+
value = :log if value.nil?
|
|
14
|
+
value = value.to_sym
|
|
15
|
+
return value if self.class::STORAGE_BACKENDS.include?(value)
|
|
16
|
+
return value if defined?(Storage::Registry) && Storage::Registry.registered?(value)
|
|
17
|
+
|
|
18
|
+
names = if defined?(Storage::Registry)
|
|
19
|
+
Storage::Registry.names
|
|
20
|
+
else
|
|
21
|
+
self.class::STORAGE_BACKENDS
|
|
22
|
+
end
|
|
23
|
+
raise Error, "Unknown storage_backend: #{value.inspect}. Use one of: #{names.join(', ')}"
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
end
|
|
@@ -1,38 +1,30 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require_relative "errors"
|
|
4
|
+
require_relative "tag_key"
|
|
4
5
|
require_relative "value_helpers"
|
|
5
6
|
require_relative "configuration/instrumentation"
|
|
7
|
+
require_relative "configuration/storage_backend"
|
|
6
8
|
|
|
7
9
|
module LlmCostTracker
|
|
8
10
|
class Configuration
|
|
9
11
|
include ConfigurationInstrumentation
|
|
12
|
+
include ConfigurationStorageBackend
|
|
10
13
|
|
|
11
|
-
OPENAI_COMPATIBLE_PROVIDERS = {
|
|
12
|
-
"openrouter.ai" => "openrouter",
|
|
13
|
-
"api.deepseek.com" => "deepseek"
|
|
14
|
-
}.freeze
|
|
14
|
+
OPENAI_COMPATIBLE_PROVIDERS = { "openrouter.ai" => "openrouter", "api.deepseek.com" => "deepseek" }.freeze
|
|
15
15
|
|
|
16
16
|
BUDGET_EXCEEDED_BEHAVIORS = %i[notify raise block_requests].freeze
|
|
17
17
|
STORAGE_ERROR_BEHAVIORS = %i[ignore warn raise].freeze
|
|
18
18
|
STORAGE_BACKENDS = %i[log active_record custom].freeze
|
|
19
19
|
UNKNOWN_PRICING_BEHAVIORS = %i[ignore warn raise].freeze
|
|
20
|
-
SHARED_SCALAR_ATTRIBUTES = %i[
|
|
21
|
-
|
|
22
|
-
custom_storage
|
|
23
|
-
on_budget_exceeded
|
|
24
|
-
monthly_budget
|
|
25
|
-
daily_budget
|
|
26
|
-
per_call_budget
|
|
27
|
-
log_level
|
|
28
|
-
prices_file
|
|
29
|
-
].freeze
|
|
20
|
+
SHARED_SCALAR_ATTRIBUTES = %i[enabled custom_storage on_budget_exceeded monthly_budget daily_budget per_call_budget
|
|
21
|
+
log_level prices_file max_tag_count max_tag_value_bytesize].freeze
|
|
30
22
|
SHARED_ENUM_ATTRIBUTES = {
|
|
31
|
-
storage_backend: [STORAGE_BACKENDS, :log],
|
|
32
23
|
budget_exceeded_behavior: [BUDGET_EXCEEDED_BEHAVIORS, :notify],
|
|
33
24
|
storage_error_behavior: [STORAGE_ERROR_BEHAVIORS, :warn],
|
|
34
25
|
unknown_pricing_behavior: [UNKNOWN_PRICING_BEHAVIORS, :warn]
|
|
35
26
|
}.freeze
|
|
27
|
+
DEFAULT_REDACTED_TAG_KEYS = %w[api_key access_token authorization credential password refresh_token secret].freeze
|
|
36
28
|
|
|
37
29
|
attr_reader(
|
|
38
30
|
*SHARED_SCALAR_ATTRIBUTES,
|
|
@@ -41,6 +33,7 @@ module LlmCostTracker
|
|
|
41
33
|
:pricing_overrides,
|
|
42
34
|
:instrumented_integrations,
|
|
43
35
|
:report_tag_breakdowns,
|
|
36
|
+
:redacted_tag_keys,
|
|
44
37
|
:storage_backend,
|
|
45
38
|
:storage_error_behavior,
|
|
46
39
|
:unknown_pricing_behavior,
|
|
@@ -61,9 +54,12 @@ module LlmCostTracker
|
|
|
61
54
|
self.unknown_pricing_behavior = :warn
|
|
62
55
|
@log_level = :info
|
|
63
56
|
@prices_file = nil
|
|
64
|
-
@
|
|
57
|
+
@max_tag_count = 50
|
|
58
|
+
@max_tag_value_bytesize = 1024
|
|
59
|
+
@pricing_overrides = {}
|
|
65
60
|
@instrumented_integrations = []
|
|
66
61
|
@report_tag_breakdowns = []
|
|
62
|
+
@redacted_tag_keys = DEFAULT_REDACTED_TAG_KEYS.dup
|
|
67
63
|
self.openai_compatible_providers = OPENAI_COMPATIBLE_PROVIDERS
|
|
68
64
|
@finalized = false
|
|
69
65
|
end
|
|
@@ -85,7 +81,12 @@ module LlmCostTracker
|
|
|
85
81
|
|
|
86
82
|
def report_tag_breakdowns=(value)
|
|
87
83
|
ensure_shared_configuration_mutable!
|
|
88
|
-
@report_tag_breakdowns = value
|
|
84
|
+
@report_tag_breakdowns = normalize_report_tag_breakdowns(value)
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
def redacted_tag_keys=(value)
|
|
88
|
+
ensure_shared_configuration_mutable!
|
|
89
|
+
@redacted_tag_keys = Array(value).map(&:to_s)
|
|
89
90
|
end
|
|
90
91
|
|
|
91
92
|
SHARED_SCALAR_ATTRIBUTES.each do |name|
|
|
@@ -107,6 +108,7 @@ module LlmCostTracker
|
|
|
107
108
|
@pricing_overrides = ValueHelpers.deep_freeze(@pricing_overrides || {})
|
|
108
109
|
@instrumented_integrations = ValueHelpers.deep_freeze(@instrumented_integrations || [])
|
|
109
110
|
@report_tag_breakdowns = ValueHelpers.deep_freeze(Array(@report_tag_breakdowns))
|
|
111
|
+
@redacted_tag_keys = ValueHelpers.deep_freeze(Array(@redacted_tag_keys))
|
|
110
112
|
@openai_compatible_providers = ValueHelpers.deep_freeze(@openai_compatible_providers || {})
|
|
111
113
|
@finalized = true
|
|
112
114
|
self
|
|
@@ -123,6 +125,7 @@ module LlmCostTracker
|
|
|
123
125
|
ValueHelpers.deep_dup(@instrumented_integrations || [])
|
|
124
126
|
)
|
|
125
127
|
copy.instance_variable_set(:@report_tag_breakdowns, ValueHelpers.deep_dup(@report_tag_breakdowns || []))
|
|
128
|
+
copy.instance_variable_set(:@redacted_tag_keys, ValueHelpers.deep_dup(@redacted_tag_keys || []))
|
|
126
129
|
copy.instance_variable_set(
|
|
127
130
|
:@openai_compatible_providers,
|
|
128
131
|
ValueHelpers.deep_dup(@openai_compatible_providers || {})
|
|
@@ -149,6 +152,10 @@ module LlmCostTracker
|
|
|
149
152
|
end
|
|
150
153
|
end
|
|
151
154
|
|
|
155
|
+
def normalize_report_tag_breakdowns(value)
|
|
156
|
+
Array(value).map { |key| TagKey.validate!(key, error_class: Error) }
|
|
157
|
+
end
|
|
158
|
+
|
|
152
159
|
def ensure_shared_configuration_mutable!
|
|
153
160
|
return unless finalized?
|
|
154
161
|
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module LlmCostTracker
|
|
4
|
+
class Doctor
|
|
5
|
+
class CaptureCheck
|
|
6
|
+
def self.call(check_class)
|
|
7
|
+
new(check_class).call
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
def initialize(check_class)
|
|
11
|
+
@check_class = check_class
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def call
|
|
15
|
+
config = LlmCostTracker.configuration
|
|
16
|
+
return disabled_check unless config.enabled
|
|
17
|
+
return integrations_check(config.instrumented_integrations) if config.instrumented_integrations.any?
|
|
18
|
+
|
|
19
|
+
check(:ok, "no SDK integrations enabled; Faraday middleware and manual capture remain available")
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
private
|
|
23
|
+
|
|
24
|
+
attr_reader :check_class
|
|
25
|
+
|
|
26
|
+
def disabled_check
|
|
27
|
+
check(:warn, "tracking is disabled; set config.enabled = true to record calls")
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def integrations_check(integrations)
|
|
31
|
+
check(:ok, "SDK integrations enabled: #{integrations.join(', ')}")
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def check(status, message)
|
|
35
|
+
check_class.new(status, "capture", message)
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
end
|