llm_cost_tracker 0.5.2 → 0.5.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +16 -0
- data/README.md +8 -3
- data/docs/architecture.md +28 -0
- data/docs/budgets.md +45 -0
- data/docs/configuration.md +65 -0
- data/docs/cookbook.md +185 -0
- data/docs/dashboard-overview.png +0 -0
- data/docs/dashboard.md +38 -0
- data/docs/extending.md +32 -0
- data/docs/operations.md +44 -0
- data/docs/pricing.md +94 -0
- data/docs/querying.md +36 -0
- data/docs/streaming.md +70 -0
- data/docs/technical/README.md +10 -0
- data/docs/technical/data-flow.md +67 -0
- data/docs/technical/extension-points.md +111 -0
- data/docs/technical/module-map.md +197 -0
- data/docs/technical/operational-notes.md +77 -0
- data/docs/upgrading.md +46 -0
- data/lib/llm_cost_tracker/capture_verifier.rb +71 -0
- data/lib/llm_cost_tracker/configuration/instrumentation.rb +1 -1
- data/lib/llm_cost_tracker/configuration/storage_backend.rb +26 -0
- data/lib/llm_cost_tracker/configuration.rb +2 -1
- data/lib/llm_cost_tracker/doctor/capture_check.rb +39 -0
- data/lib/llm_cost_tracker/doctor.rb +6 -1
- data/lib/llm_cost_tracker/integrations/anthropic.rb +41 -2
- data/lib/llm_cost_tracker/integrations/openai.rb +66 -2
- data/lib/llm_cost_tracker/integrations/registry.rb +33 -3
- data/lib/llm_cost_tracker/integrations/stream_tracker.rb +166 -0
- data/lib/llm_cost_tracker/llm_api_call.rb +2 -78
- data/lib/llm_cost_tracker/llm_api_call_metrics.rb +63 -0
- data/lib/llm_cost_tracker/parsers/openai_usage.rb +1 -1
- data/lib/llm_cost_tracker/pricing/effective_prices.rb +75 -0
- data/lib/llm_cost_tracker/pricing/explainer.rb +77 -0
- data/lib/llm_cost_tracker/pricing/lookup.rb +110 -0
- data/lib/llm_cost_tracker/pricing.rb +25 -108
- data/lib/llm_cost_tracker/retention.rb +3 -9
- data/lib/llm_cost_tracker/storage/active_record_backend.rb +115 -0
- data/lib/llm_cost_tracker/storage/active_record_rollups.rb +42 -0
- data/lib/llm_cost_tracker/storage/active_record_store.rb +26 -0
- data/lib/llm_cost_tracker/storage/custom_backend.rb +32 -0
- data/lib/llm_cost_tracker/storage/dispatcher.rb +11 -34
- data/lib/llm_cost_tracker/storage/log_backend.rb +38 -0
- data/lib/llm_cost_tracker/storage/registry.rb +63 -0
- data/lib/llm_cost_tracker/tag_sql.rb +34 -0
- data/lib/llm_cost_tracker/version.rb +1 -1
- data/lib/llm_cost_tracker.rb +3 -0
- data/lib/tasks/llm_cost_tracker.rake +49 -0
- metadata +32 -2
data/docs/streaming.md
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
# Streaming Capture
|
|
2
|
+
|
|
3
|
+
Streaming calls should appear in the ledger instead of disappearing into a live
|
|
4
|
+
callback. LLM Cost Tracker records them when the provider emits final usage or
|
|
5
|
+
when the app supplies explicit totals.
|
|
6
|
+
|
|
7
|
+
The full streaming reference is moving here from the README: Faraday streaming,
|
|
8
|
+
`track_stream`, provider response IDs, final usage events, and data-quality
|
|
9
|
+
states.
|
|
10
|
+
|
|
11
|
+
## Canonical Sources
|
|
12
|
+
|
|
13
|
+
Until this page is expanded, use:
|
|
14
|
+
|
|
15
|
+
- [Capturing calls](../README.md#capturing-calls)
|
|
16
|
+
- [Known limitations](../README.md#known-limitations)
|
|
17
|
+
- [Cookbook](cookbook.md)
|
|
18
|
+
|
|
19
|
+
## Faraday Path
|
|
20
|
+
|
|
21
|
+
The middleware tees Faraday's `on_data` callback, keeps chunks flowing to the
|
|
22
|
+
caller, and records usage when the response completes.
|
|
23
|
+
|
|
24
|
+
OpenAI streams need final usage:
|
|
25
|
+
|
|
26
|
+
```ruby
|
|
27
|
+
stream_options: { include_usage: true }
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
Anthropic and Gemini are parsed from their provider stream event shapes when
|
|
31
|
+
usage is present.
|
|
32
|
+
|
|
33
|
+
## SDK Path
|
|
34
|
+
|
|
35
|
+
Official OpenAI and Anthropic SDK streams are captured when `config.instrument`
|
|
36
|
+
is enabled for the provider. The returned stream object is preserved, and usage
|
|
37
|
+
is recorded after the stream is consumed.
|
|
38
|
+
|
|
39
|
+
```ruby
|
|
40
|
+
config.instrument :openai
|
|
41
|
+
config.instrument :anthropic
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
Captured SDK helpers:
|
|
45
|
+
|
|
46
|
+
- OpenAI `responses.stream`, `responses.stream_raw`, `responses.retrieve_streaming`, and `chat.completions.stream_raw`.
|
|
47
|
+
- Anthropic `messages.stream` and `messages.stream_raw`.
|
|
48
|
+
|
|
49
|
+
OpenAI Chat Completions streams need final usage:
|
|
50
|
+
|
|
51
|
+
```ruby
|
|
52
|
+
stream_options: { include_usage: true }
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
## Manual Path
|
|
56
|
+
|
|
57
|
+
```ruby
|
|
58
|
+
LlmCostTracker.track_stream(provider: "openai", model: "gpt-4o") do |stream|
|
|
59
|
+
my_client.stream(...) { |event| stream.event(event.to_h) }
|
|
60
|
+
end
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
If the client already knows totals, skip provider event parsing:
|
|
64
|
+
|
|
65
|
+
```ruby
|
|
66
|
+
stream.usage(input_tokens: 120, output_tokens: 45)
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
Missing final usage is stored with `usage_source: "unknown"` so the Data Quality
|
|
70
|
+
page can surface it.
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
# Technical Documentation
|
|
2
|
+
|
|
3
|
+
These files describe the internal module boundaries for LLM Cost Tracker.
|
|
4
|
+
|
|
5
|
+
- [Module map](module-map.md)
|
|
6
|
+
- [Data flow](data-flow.md)
|
|
7
|
+
- [Extension points](extension-points.md)
|
|
8
|
+
- [Operational notes](operational-notes.md)
|
|
9
|
+
|
|
10
|
+
The main rule is simple: provider-specific API shapes stop at ingestion boundaries. The ledger, storage, budgets, dashboard, and reports work with canonical billing concepts.
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
# Data Flow
|
|
2
|
+
|
|
3
|
+
This is the normal path from an application LLM call to stored ledger data.
|
|
4
|
+
|
|
5
|
+
## Faraday Requests
|
|
6
|
+
|
|
7
|
+
1. The host app sends an HTTP request through Faraday.
|
|
8
|
+
2. `LlmCostTracker::Middleware::Faraday` checks whether a parser matches the request URL.
|
|
9
|
+
3. For non-streaming responses, the middleware passes request and response data to the parser.
|
|
10
|
+
4. For streaming responses, the middleware tees `on_data`, collects stream events, and parses final usage when the stream completes.
|
|
11
|
+
5. The parser returns `ParsedUsage` with canonical fields.
|
|
12
|
+
6. `Tracker.record` prices and persists the event.
|
|
13
|
+
|
|
14
|
+
## SDK Integrations
|
|
15
|
+
|
|
16
|
+
1. The host app enables an integration with `config.instrument`.
|
|
17
|
+
2. `LlmCostTracker::Integrations` checks the SDK version, target classes, and target methods once at install time.
|
|
18
|
+
3. `LlmCostTracker::Integrations` prepends a narrow wrapper to supported SDK resource methods.
|
|
19
|
+
4. The host app keeps calling the provider SDK normally.
|
|
20
|
+
5. The wrapper measures latency, extracts usage from the SDK response object, and sends canonical fields to `Tracker.record`.
|
|
21
|
+
6. If an explicitly enabled SDK is not loaded or does not satisfy the install contract, boot raises before the app silently misses usage.
|
|
22
|
+
|
|
23
|
+
## Explicit Tracking
|
|
24
|
+
|
|
25
|
+
1. The host app calls `LlmCostTracker.track` with known usage totals, or `LlmCostTracker.track_stream` with stream events.
|
|
26
|
+
2. `track` sends manual totals directly to `Tracker.record`.
|
|
27
|
+
3. `track_stream` uses `StreamCollector`, then parser lookup by provider when events need parsing.
|
|
28
|
+
4. `Tracker.record` prices and persists the event.
|
|
29
|
+
|
|
30
|
+
## Canonical Event Build
|
|
31
|
+
|
|
32
|
+
`Tracker.record` performs the central normalization step:
|
|
33
|
+
|
|
34
|
+
1. Blank model identifiers become `unknown`.
|
|
35
|
+
2. Input, output, cache-read, cache-write, hidden-output, and pricing-mode values are extracted from metadata.
|
|
36
|
+
3. `Pricing.cost_for` calculates a `Cost` object or returns `nil` for unknown pricing.
|
|
37
|
+
4. Tags are merged from `with_tags`, `default_tags`, middleware tags, and explicit metadata.
|
|
38
|
+
5. An `Event` is created and emitted through `ActiveSupport::Notifications`.
|
|
39
|
+
6. The configured storage backend receives the event.
|
|
40
|
+
7. Budget checks run unless storage explicitly returns `false`.
|
|
41
|
+
|
|
42
|
+
## ActiveRecord Storage
|
|
43
|
+
|
|
44
|
+
1. `Storage::ActiveRecordStore.save` converts tags for JSON or text storage.
|
|
45
|
+
2. Optional fields are written only when their columns exist.
|
|
46
|
+
3. The call row and period rollup updates happen in one transaction.
|
|
47
|
+
4. `ActiveRecordRollups.increment!` updates daily and monthly totals atomically.
|
|
48
|
+
5. Budget reads use period totals when available.
|
|
49
|
+
|
|
50
|
+
## Dashboard Reads
|
|
51
|
+
|
|
52
|
+
1. Controllers build a filtered `LlmApiCall` scope.
|
|
53
|
+
2. Dashboard services run targeted aggregate queries.
|
|
54
|
+
3. Helpers render filters, charts, pagination, CSV links, and numeric formatting.
|
|
55
|
+
4. Views render plain ERB with the engine CSS asset.
|
|
56
|
+
|
|
57
|
+
Dashboard reads do not mutate ledger state. They can be heavier than request-time code, but they still need explicit grouping and indexes.
|
|
58
|
+
|
|
59
|
+
## Pricing Refresh
|
|
60
|
+
|
|
61
|
+
1. `llm_cost_tracker:prices:refresh` chooses `ENV["OUTPUT"]`, then `config.prices_file`, then `config/llm_cost_tracker_prices.yml`.
|
|
62
|
+
2. `PriceSync::Fetcher` fetches the maintained LLM Cost Tracker price snapshot.
|
|
63
|
+
3. `PriceSync` validates schema compatibility, gem-version compatibility, and model price shape.
|
|
64
|
+
4. `RegistryWriter` writes a local JSON or YAML registry.
|
|
65
|
+
5. Runtime pricing reloads the local file when its mtime changes.
|
|
66
|
+
|
|
67
|
+
The gem never fetches pricing from the network during normal request tracking.
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
# Extension Points
|
|
2
|
+
|
|
3
|
+
Extensions should plug into existing provider-agnostic boundaries. If a new feature needs a provider-specific branch outside ingestion code, revisit the design first.
|
|
4
|
+
|
|
5
|
+
## Custom Parsers
|
|
6
|
+
|
|
7
|
+
Use parser registration when a provider or gateway has a response shape the built-ins do not cover.
|
|
8
|
+
|
|
9
|
+
Expected parser contract:
|
|
10
|
+
|
|
11
|
+
- `match?(url)` detects supported request URLs.
|
|
12
|
+
- `parse(request_url, request_body, response_status, response_body)` returns `ParsedUsage` or `nil`.
|
|
13
|
+
- `parse_stream(request_url, request_body, response_status, events)` returns `ParsedUsage` or `nil`.
|
|
14
|
+
- `streaming_request?(request_url, request_body)` detects streaming requests when the provider does not use a simple `stream: true` field.
|
|
15
|
+
- `provider_names` returns provider names that can be used by `track_stream(provider: ...)`.
|
|
16
|
+
|
|
17
|
+
Use `Parsers::Base` helpers for URL matching and stream-event extraction. Use `Parsers::OpenaiUsage` only for OpenAI-shaped usage hashes.
|
|
18
|
+
|
|
19
|
+
## SDK Integrations
|
|
20
|
+
|
|
21
|
+
Use SDK integrations when a popular Ruby client does not expose a Faraday middleware stack but returns stable usage objects. RubyLLM and the official `openai` and `anthropic` gems qualify. Faraday-based clients that expose a middleware hook, such as `ruby-openai`'s constructor block, are covered by the Faraday middleware instead. Clients with no stable hook must use the explicit `track` / `track_stream` fallback until an integration exists.
|
|
22
|
+
|
|
23
|
+
Expected integration contract:
|
|
24
|
+
|
|
25
|
+
- no hard dependency on the provider SDK
|
|
26
|
+
- fail-fast boot when an explicitly enabled SDK is missing or below the minimum supported version
|
|
27
|
+
- install-time checks for the target classes and methods
|
|
28
|
+
- idempotent `Module#prepend` around narrow resource methods
|
|
29
|
+
- no tracking when the integration is not enabled in configuration
|
|
30
|
+
- canonical usage fields passed to `Tracker.record`
|
|
31
|
+
|
|
32
|
+
SDK integrations belong under `LlmCostTracker::Integrations`. Do not put SDK object-shape handling in parsers, storage, or pricing.
|
|
33
|
+
|
|
34
|
+
External integrations can register an adapter with
|
|
35
|
+
`LlmCostTracker::Integrations.register(:name, adapter)`. The adapter must
|
|
36
|
+
respond to `install` and `status`, and enabled names are still selected through
|
|
37
|
+
`config.instrument`.
|
|
38
|
+
|
|
39
|
+
## OpenAI-Compatible Gateways
|
|
40
|
+
|
|
41
|
+
Use `config.openai_compatible_providers` when a gateway speaks the OpenAI request and response shape.
|
|
42
|
+
|
|
43
|
+
This is for shape compatibility, not pricing. Gateway-specific model IDs or discounts belong in `prices_file` or `pricing_overrides`.
|
|
44
|
+
|
|
45
|
+
## Prices
|
|
46
|
+
|
|
47
|
+
Use `config.prices_file` for the app's source-controlled price snapshot.
|
|
48
|
+
|
|
49
|
+
Use `config.pricing_overrides` for urgent or environment-specific overrides that are easier to keep in Ruby.
|
|
50
|
+
|
|
51
|
+
Supported canonical keys:
|
|
52
|
+
|
|
53
|
+
- `input`
|
|
54
|
+
- `output`
|
|
55
|
+
- `cache_read_input`
|
|
56
|
+
- `cache_write_input`
|
|
57
|
+
- `batch_input`
|
|
58
|
+
- `batch_output`
|
|
59
|
+
- mode-prefixed keys such as `priority_input` or `batch_cache_read_input`
|
|
60
|
+
|
|
61
|
+
Provider-specific pricing details must be translated before they reach runtime pricing.
|
|
62
|
+
|
|
63
|
+
## Tags
|
|
64
|
+
|
|
65
|
+
Tags are the extension point for application attribution:
|
|
66
|
+
|
|
67
|
+
- tenant
|
|
68
|
+
- user
|
|
69
|
+
- feature
|
|
70
|
+
- trace
|
|
71
|
+
- job
|
|
72
|
+
- workflow
|
|
73
|
+
- agent session
|
|
74
|
+
|
|
75
|
+
Use `config.default_tags`, middleware `tags:`, explicit metadata, and `LlmCostTracker.with_tags`. Do not add first-class columns for app dimensions unless the ledger needs that field for provider-agnostic billing behavior.
|
|
76
|
+
|
|
77
|
+
## Storage
|
|
78
|
+
|
|
79
|
+
Use `storage_backend = :custom` only when the host app needs to own persistence completely.
|
|
80
|
+
|
|
81
|
+
Custom storage receives a canonical `Event`. Returning `false` tells the tracker not to run budget checks for that event.
|
|
82
|
+
|
|
83
|
+
ActiveRecord storage is the production path for dashboards and cross-process budgets.
|
|
84
|
+
|
|
85
|
+
Storage adapters can register with
|
|
86
|
+
`LlmCostTracker::Storage.register(:name, backend)`. A backend must respond to
|
|
87
|
+
`save(event)` and may expose `verify` for capture diagnostics.
|
|
88
|
+
|
|
89
|
+
## Dashboard
|
|
90
|
+
|
|
91
|
+
Dashboard additions should be read-only services under `app/services/llm_cost_tracker/dashboard`.
|
|
92
|
+
|
|
93
|
+
Keep controller actions thin:
|
|
94
|
+
|
|
95
|
+
- parse params
|
|
96
|
+
- build filtered scope
|
|
97
|
+
- call services
|
|
98
|
+
- render views
|
|
99
|
+
|
|
100
|
+
Keep view logic in helpers when it is reused across pages. Do not add JavaScript for dashboard behavior.
|
|
101
|
+
|
|
102
|
+
## Generators
|
|
103
|
+
|
|
104
|
+
Generators are installation contracts. New generator behavior should be:
|
|
105
|
+
|
|
106
|
+
- additive when possible
|
|
107
|
+
- idempotent where Rails generator APIs allow it
|
|
108
|
+
- explicit about destructive or table-rewriting operations
|
|
109
|
+
- covered by generator template specs
|
|
110
|
+
|
|
111
|
+
Fresh install templates and upgrade generators should stay aligned. If a fresh install gains a column or index, the upgrade path needs a generator unless the next release intentionally makes a breaking install path.
|
|
@@ -0,0 +1,197 @@
|
|
|
1
|
+
# Module Map
|
|
2
|
+
|
|
3
|
+
LLM Cost Tracker is organized around a small set of durable responsibilities. File layout does not need to mirror these modules perfectly, but new code should fit one of these boundaries.
|
|
4
|
+
|
|
5
|
+
## Public API and Configuration
|
|
6
|
+
|
|
7
|
+
Primary files:
|
|
8
|
+
|
|
9
|
+
- `lib/llm_cost_tracker.rb`
|
|
10
|
+
- `lib/llm_cost_tracker/configuration.rb`
|
|
11
|
+
- `lib/llm_cost_tracker/tag_context.rb`
|
|
12
|
+
- `lib/llm_cost_tracker/doctor.rb`
|
|
13
|
+
- `lib/llm_cost_tracker/logging.rb`
|
|
14
|
+
- `lib/llm_cost_tracker/errors.rb`
|
|
15
|
+
|
|
16
|
+
Responsibilities:
|
|
17
|
+
|
|
18
|
+
- Expose `configure`, `track`, `track_stream`, `with_tags`, and `enforce_budget!`.
|
|
19
|
+
- Keep configuration immutable after `configure` returns.
|
|
20
|
+
- Merge scoped tags and default tags without leaking state across threads.
|
|
21
|
+
- Report installation and pricing health through `llm_cost_tracker:doctor`.
|
|
22
|
+
|
|
23
|
+
This module should stay small. It can orchestrate other modules, but it should not contain provider parsing, SQL details, dashboard aggregation, or pricing-source logic.
|
|
24
|
+
|
|
25
|
+
## SDK Integrations
|
|
26
|
+
|
|
27
|
+
Primary files:
|
|
28
|
+
|
|
29
|
+
- `lib/llm_cost_tracker/integrations/*`
|
|
30
|
+
|
|
31
|
+
Responsibilities:
|
|
32
|
+
|
|
33
|
+
- Add optional instrumentation for Ruby SDKs without adding provider SDK dependencies.
|
|
34
|
+
- Install narrow, idempotent `Module#prepend` wrappers around stable SDK resource methods.
|
|
35
|
+
- Extract SDK response objects into canonical usage fields.
|
|
36
|
+
- Keep SDK-specific object handling out of `Tracker` and storage.
|
|
37
|
+
|
|
38
|
+
Integrations are for Ruby SDK object shapes. Parsers are for HTTP and stream payload shapes.
|
|
39
|
+
|
|
40
|
+
## Ingestion
|
|
41
|
+
|
|
42
|
+
Primary files:
|
|
43
|
+
|
|
44
|
+
- `lib/llm_cost_tracker/middleware/faraday.rb`
|
|
45
|
+
- `lib/llm_cost_tracker/stream_collector.rb`
|
|
46
|
+
- `lib/llm_cost_tracker/parsed_usage.rb`
|
|
47
|
+
- `lib/llm_cost_tracker/request_url.rb`
|
|
48
|
+
- `lib/llm_cost_tracker/parsers/*`
|
|
49
|
+
|
|
50
|
+
Responsibilities:
|
|
51
|
+
|
|
52
|
+
- Detect supported LLM HTTP requests.
|
|
53
|
+
- Parse provider responses and stream events into `ParsedUsage`.
|
|
54
|
+
- Translate provider-specific fields into canonical usage fields.
|
|
55
|
+
- Preserve app streaming behavior while teeing events for tracking.
|
|
56
|
+
|
|
57
|
+
Provider-specific code belongs here. The output boundary is `ParsedUsage`, not raw provider JSON.
|
|
58
|
+
|
|
59
|
+
## Canonical Ledger
|
|
60
|
+
|
|
61
|
+
Primary files:
|
|
62
|
+
|
|
63
|
+
- `lib/llm_cost_tracker/tracker.rb`
|
|
64
|
+
- `lib/llm_cost_tracker/event.rb`
|
|
65
|
+
- `lib/llm_cost_tracker/event_metadata.rb`
|
|
66
|
+
- `lib/llm_cost_tracker/usage_breakdown.rb`
|
|
67
|
+
- `lib/llm_cost_tracker/cost.rb`
|
|
68
|
+
- `lib/llm_cost_tracker/unknown_pricing.rb`
|
|
69
|
+
|
|
70
|
+
Responsibilities:
|
|
71
|
+
|
|
72
|
+
- Normalize provider, model, usage, tags, latency, streaming flags, and response IDs.
|
|
73
|
+
- Price canonical usage through `Pricing`.
|
|
74
|
+
- Emit `ActiveSupport::Notifications`.
|
|
75
|
+
- Persist the event through the configured storage backend.
|
|
76
|
+
- Run budget checks after successful storage.
|
|
77
|
+
|
|
78
|
+
This module must remain provider-agnostic. It should never branch on a specific provider model family.
|
|
79
|
+
|
|
80
|
+
## Pricing
|
|
81
|
+
|
|
82
|
+
Primary files:
|
|
83
|
+
|
|
84
|
+
- `lib/llm_cost_tracker/pricing.rb`
|
|
85
|
+
- `lib/llm_cost_tracker/price_registry.rb`
|
|
86
|
+
- `lib/llm_cost_tracker/price_freshness.rb`
|
|
87
|
+
- `lib/llm_cost_tracker/prices.json`
|
|
88
|
+
- `lib/llm_cost_tracker/price_sync/*`
|
|
89
|
+
- `lib/tasks/llm_cost_tracker.rake`
|
|
90
|
+
|
|
91
|
+
Responsibilities:
|
|
92
|
+
|
|
93
|
+
- Load bundled prices, local price snapshots, and Ruby overrides.
|
|
94
|
+
- Apply pricing precedence: `pricing_overrides`, `prices_file`, bundled prices.
|
|
95
|
+
- Calculate costs from canonical usage fields.
|
|
96
|
+
- Update local snapshots from the maintained LLM Cost Tracker price registry.
|
|
97
|
+
- Validate snapshot schema compatibility, gem-version compatibility, and price entry shape.
|
|
98
|
+
|
|
99
|
+
Pricing refresh must not perform boot-time or request-time network work. Runtime pricing uses bundled prices, local files, and in-memory caches.
|
|
100
|
+
|
|
101
|
+
## Storage
|
|
102
|
+
|
|
103
|
+
Primary files:
|
|
104
|
+
|
|
105
|
+
- `lib/llm_cost_tracker/llm_api_call.rb`
|
|
106
|
+
- `lib/llm_cost_tracker/period_total.rb`
|
|
107
|
+
- `lib/llm_cost_tracker/llm_api_call_metrics.rb`
|
|
108
|
+
- `lib/llm_cost_tracker/storage/active_record_store.rb`
|
|
109
|
+
- `lib/llm_cost_tracker/storage/active_record_rollups.rb`
|
|
110
|
+
- `lib/llm_cost_tracker/storage/registry.rb`
|
|
111
|
+
- `lib/llm_cost_tracker/tags_column.rb`
|
|
112
|
+
- `lib/llm_cost_tracker/tag_key.rb`
|
|
113
|
+
- `lib/llm_cost_tracker/tag_sql.rb`
|
|
114
|
+
- `lib/llm_cost_tracker/tag_query.rb`
|
|
115
|
+
- `lib/llm_cost_tracker/tag_accessors.rb`
|
|
116
|
+
- `lib/llm_cost_tracker/period_grouping.rb`
|
|
117
|
+
|
|
118
|
+
Responsibilities:
|
|
119
|
+
|
|
120
|
+
- Persist canonical events into ActiveRecord.
|
|
121
|
+
- Hide database-specific tag storage differences.
|
|
122
|
+
- Maintain period rollups for hot-path budget reads.
|
|
123
|
+
- Provide safe scopes for filters, periods, tags, unknown pricing, and reports.
|
|
124
|
+
|
|
125
|
+
Storage can know about database adapters and optional columns. It should not parse provider responses or fetch price data.
|
|
126
|
+
|
|
127
|
+
## Budgets and Retention
|
|
128
|
+
|
|
129
|
+
Primary files:
|
|
130
|
+
|
|
131
|
+
- `lib/llm_cost_tracker/budget.rb`
|
|
132
|
+
- `lib/llm_cost_tracker/retention.rb`
|
|
133
|
+
- `lib/llm_cost_tracker/storage/active_record_rollups.rb`
|
|
134
|
+
|
|
135
|
+
Responsibilities:
|
|
136
|
+
|
|
137
|
+
- Enforce monthly, daily, and per-call guardrails.
|
|
138
|
+
- Support preflight blocking where ActiveRecord rollups are available.
|
|
139
|
+
- Prune old ledger rows in batches.
|
|
140
|
+
- Keep budget checks bounded by maintained aggregates, not by full ledger scans.
|
|
141
|
+
|
|
142
|
+
Budget behavior is part of the hot path. Any change here must be measured against per-request overhead.
|
|
143
|
+
|
|
144
|
+
## Dashboard and Reporting
|
|
145
|
+
|
|
146
|
+
Primary files:
|
|
147
|
+
|
|
148
|
+
- `lib/llm_cost_tracker/report*.rb`
|
|
149
|
+
- `app/controllers/llm_cost_tracker/*`
|
|
150
|
+
- `app/services/llm_cost_tracker/dashboard/*`
|
|
151
|
+
- `app/helpers/llm_cost_tracker/*`
|
|
152
|
+
- `app/views/llm_cost_tracker/*`
|
|
153
|
+
- `app/assets/llm_cost_tracker/application.css`
|
|
154
|
+
|
|
155
|
+
Responsibilities:
|
|
156
|
+
|
|
157
|
+
- Render server-side dashboard pages.
|
|
158
|
+
- Aggregate spend, calls, providers, models, tags, latency, and data quality.
|
|
159
|
+
- Export filtered calls as CSV.
|
|
160
|
+
- Keep dashboard queries explicit and indexed.
|
|
161
|
+
|
|
162
|
+
Dashboard code may run grouped SQL because it is user-initiated reporting. It must stay server-rendered and must not introduce a JavaScript bundle.
|
|
163
|
+
|
|
164
|
+
## Rails Integration and Generators
|
|
165
|
+
|
|
166
|
+
Primary files:
|
|
167
|
+
|
|
168
|
+
- `lib/llm_cost_tracker/railtie.rb`
|
|
169
|
+
- `lib/llm_cost_tracker/engine.rb`
|
|
170
|
+
- `lib/llm_cost_tracker/assets.rb`
|
|
171
|
+
- `lib/llm_cost_tracker/generators/llm_cost_tracker/*`
|
|
172
|
+
- `config/routes.rb`
|
|
173
|
+
|
|
174
|
+
Responsibilities:
|
|
175
|
+
|
|
176
|
+
- Register rake tasks and Faraday middleware.
|
|
177
|
+
- Mount the isolated Rails engine.
|
|
178
|
+
- Generate migrations, initializer, dashboard route, and local price snapshots.
|
|
179
|
+
- Serve dashboard CSS as a fingerprinted engine asset.
|
|
180
|
+
|
|
181
|
+
Generator templates are public installation contracts. Treat them like API.
|
|
182
|
+
|
|
183
|
+
## Test Suites
|
|
184
|
+
|
|
185
|
+
Primary files:
|
|
186
|
+
|
|
187
|
+
- `spec/llm_cost_tracker/*`
|
|
188
|
+
- `spec/llm_cost_tracker/engine/*`
|
|
189
|
+
- `spec/llm_cost_tracker/dashboard/*`
|
|
190
|
+
- `spec/fixtures/pricing/*`
|
|
191
|
+
- `spec/support/*`
|
|
192
|
+
|
|
193
|
+
Responsibilities:
|
|
194
|
+
|
|
195
|
+
- Cover canonical behavior, parser boundaries, pricing precedence, storage rollups, dashboard rendering, generators, and concurrency.
|
|
196
|
+
- Keep request specs plain and stable.
|
|
197
|
+
- Run through `bin/check` before release work or commits that touch code.
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
# Operational Notes
|
|
2
|
+
|
|
3
|
+
This file describes runtime constraints that should shape implementation decisions.
|
|
4
|
+
|
|
5
|
+
## Hot Paths
|
|
6
|
+
|
|
7
|
+
Hot-path code includes:
|
|
8
|
+
|
|
9
|
+
- Faraday middleware request and response handling
|
|
10
|
+
- stream collection
|
|
11
|
+
- `Tracker.record`
|
|
12
|
+
- `Pricing.cost_for`
|
|
13
|
+
- ActiveRecord event persistence
|
|
14
|
+
- budget checks
|
|
15
|
+
|
|
16
|
+
Hot-path code must avoid:
|
|
17
|
+
|
|
18
|
+
- network calls
|
|
19
|
+
- per-event schema discovery beyond memoized checks
|
|
20
|
+
- full ledger aggregation
|
|
21
|
+
- unbounded stream buffers
|
|
22
|
+
- N+1 queries
|
|
23
|
+
- price-refresh work
|
|
24
|
+
|
|
25
|
+
## Pricing Freshness
|
|
26
|
+
|
|
27
|
+
Runtime pricing is local:
|
|
28
|
+
|
|
29
|
+
1. Ruby overrides
|
|
30
|
+
2. configured local price snapshot
|
|
31
|
+
3. bundled prices
|
|
32
|
+
|
|
33
|
+
Price update tasks are operational tooling. They can fetch the maintained LLM Cost Tracker price snapshot because the operator runs them intentionally. Request tracking must never depend on live provider pricing pages.
|
|
34
|
+
|
|
35
|
+
## Budget Reads
|
|
36
|
+
|
|
37
|
+
Monthly and daily budgets should read `llm_cost_tracker_period_totals` when the table exists. Falling back to summing `llm_api_calls` is an upgrade compatibility path, not the preferred production path.
|
|
38
|
+
|
|
39
|
+
Per-call budgets are checked from the current event only.
|
|
40
|
+
|
|
41
|
+
## Retention
|
|
42
|
+
|
|
43
|
+
Retention may delete old `llm_api_calls`. Period rollups are the durable budget aggregate. Any migration or refactor that changes rollups must preserve the meaning of retained totals or clearly document a breaking change.
|
|
44
|
+
|
|
45
|
+
## Optional Columns
|
|
46
|
+
|
|
47
|
+
The gem supports upgrade paths where older apps may not have every column yet. Optional column checks must be memoized and refreshed when ActiveRecord column information is reset.
|
|
48
|
+
|
|
49
|
+
Do not put table or column checks directly in loops that run for every event without caching.
|
|
50
|
+
|
|
51
|
+
## Dashboard Queries
|
|
52
|
+
|
|
53
|
+
Dashboard queries can aggregate because they are user-initiated. They should still use:
|
|
54
|
+
|
|
55
|
+
- filtered scopes
|
|
56
|
+
- bounded pagination
|
|
57
|
+
- database-side grouping
|
|
58
|
+
- indexes that match common filters
|
|
59
|
+
- single aggregate queries for related counters
|
|
60
|
+
|
|
61
|
+
Avoid loading ledger rows into Ruby just to count, sum, group, or sort.
|
|
62
|
+
|
|
63
|
+
## Streaming
|
|
64
|
+
|
|
65
|
+
Streaming capture must keep the host app's stream behavior intact.
|
|
66
|
+
|
|
67
|
+
The middleware should collect enough data to parse final usage while bounding memory. When usage never arrives or capture overflows, record an unknown-usage event so Data Quality can surface the gap.
|
|
68
|
+
|
|
69
|
+
## Release Checks
|
|
70
|
+
|
|
71
|
+
Run `bin/check` before committing code changes intended for release. It includes full RuboCop, full RSpec, project coverage, and patch coverage for the current diff.
|
|
72
|
+
|
|
73
|
+
Project coverage defaults to the Codecov target. Patch coverage defaults to 95% so local checks stay stricter than Codecov parser differences. Thresholds can be adjusted locally with `PROJECT_COVERAGE_MIN`, `PATCH_COVERAGE_MIN`, or `COVERAGE_BASE`.
|
|
74
|
+
|
|
75
|
+
For the closest match to the Codecov upload job, run `BUNDLE_GEMFILE=gemfiles/rails_8_1.gemfile bin/check`.
|
|
76
|
+
|
|
77
|
+
Docs-only changes do not require the full suite, but any code, generator, migration, parser, pricing, dashboard, or storage change does.
|
data/docs/upgrading.md
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
# Upgrading
|
|
2
|
+
|
|
3
|
+
LLM Cost Tracker is still moving quickly, so upgrades should be explicit:
|
|
4
|
+
inspect the changelog, run doctor, and apply only the generators your schema is
|
|
5
|
+
missing.
|
|
6
|
+
|
|
7
|
+
The version-by-version upgrade guide is moving here from the README.
|
|
8
|
+
|
|
9
|
+
## Canonical Sources
|
|
10
|
+
|
|
11
|
+
Until this page is expanded, use:
|
|
12
|
+
|
|
13
|
+
- [Changelog](../CHANGELOG.md)
|
|
14
|
+
- [Quickstart](../README.md#quickstart)
|
|
15
|
+
- [Operations](operations.md)
|
|
16
|
+
|
|
17
|
+
## Schema Generators
|
|
18
|
+
|
|
19
|
+
Existing installs can add newer optional columns through focused generators:
|
|
20
|
+
|
|
21
|
+
```bash
|
|
22
|
+
bin/rails generate llm_cost_tracker:add_period_totals
|
|
23
|
+
bin/rails generate llm_cost_tracker:add_streaming
|
|
24
|
+
bin/rails generate llm_cost_tracker:add_provider_response_id
|
|
25
|
+
bin/rails generate llm_cost_tracker:add_usage_breakdown
|
|
26
|
+
bin/rails generate llm_cost_tracker:upgrade_tags_to_jsonb
|
|
27
|
+
bin/rails generate llm_cost_tracker:upgrade_cost_precision
|
|
28
|
+
bin/rails generate llm_cost_tracker:add_latency_ms
|
|
29
|
+
bin/rails db:migrate
|
|
30
|
+
bin/rails llm_cost_tracker:doctor
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
On PostgreSQL, `upgrade_tags_to_jsonb` rewrites `llm_api_calls`. For large
|
|
34
|
+
tables, run it during a maintenance window or replace it with a two-phase
|
|
35
|
+
backfill.
|
|
36
|
+
|
|
37
|
+
## Upgrade Habit
|
|
38
|
+
|
|
39
|
+
Run:
|
|
40
|
+
|
|
41
|
+
```bash
|
|
42
|
+
bin/rails llm_cost_tracker:doctor
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
Doctor tells you which optional columns and production-hardening pieces are still
|
|
46
|
+
missing.
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "storage/dispatcher"
|
|
4
|
+
|
|
5
|
+
module LlmCostTracker
|
|
6
|
+
class CaptureVerifier
|
|
7
|
+
Check = Data.define(:status, :name, :message)
|
|
8
|
+
|
|
9
|
+
class << self
|
|
10
|
+
def call = new.checks
|
|
11
|
+
|
|
12
|
+
def report(checks = call)
|
|
13
|
+
(["LLM Cost Tracker capture verification"] + checks.map { |check| format_check(check) }).join("\n")
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def healthy?(checks = call)
|
|
17
|
+
checks.none? { |check| check.status == :error }
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
private
|
|
21
|
+
|
|
22
|
+
def format_check(check)
|
|
23
|
+
"[#{check.status}] #{check.name}: #{check.message}"
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def checks
|
|
28
|
+
[
|
|
29
|
+
enabled_check,
|
|
30
|
+
*integration_checks,
|
|
31
|
+
*storage_checks
|
|
32
|
+
].compact
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
private
|
|
36
|
+
|
|
37
|
+
def enabled_check
|
|
38
|
+
return Check.new(:ok, "tracking", "enabled") if LlmCostTracker.configuration.enabled
|
|
39
|
+
|
|
40
|
+
Check.new(:error, "tracking", "disabled; set config.enabled = true before verifying capture")
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def integration_checks
|
|
44
|
+
enabled = LlmCostTracker.configuration.instrumented_integrations
|
|
45
|
+
if enabled.empty?
|
|
46
|
+
return [
|
|
47
|
+
Check.new(:ok, "sdk integrations", "none enabled; Faraday middleware and manual capture remain available")
|
|
48
|
+
]
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
LlmCostTracker::Integrations.checks.map do |check|
|
|
52
|
+
Check.new(check.status, "sdk integration #{check.name}", check.message)
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def storage_checks
|
|
57
|
+
backend = LlmCostTracker::Storage::Registry.fetch(LlmCostTracker.configuration.storage_backend)
|
|
58
|
+
unless backend.respond_to?(:verify)
|
|
59
|
+
return [
|
|
60
|
+
Check.new(:warn, "storage", "#{LlmCostTracker.configuration.storage_backend} backend has no verifier")
|
|
61
|
+
]
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
backend.verify.map do |check|
|
|
65
|
+
Check.new(check.status, check.name, check.message)
|
|
66
|
+
end
|
|
67
|
+
rescue LlmCostTracker::Error => e
|
|
68
|
+
[Check.new(:error, "storage", e.message)]
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
end
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module LlmCostTracker
|
|
4
|
+
module ConfigurationStorageBackend
|
|
5
|
+
def storage_backend=(value)
|
|
6
|
+
ensure_shared_configuration_mutable!
|
|
7
|
+
@storage_backend = normalize_storage_backend(value)
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
private
|
|
11
|
+
|
|
12
|
+
def normalize_storage_backend(value)
|
|
13
|
+
value = :log if value.nil?
|
|
14
|
+
value = value.to_sym
|
|
15
|
+
return value if self.class::STORAGE_BACKENDS.include?(value)
|
|
16
|
+
return value if defined?(Storage::Registry) && Storage::Registry.registered?(value)
|
|
17
|
+
|
|
18
|
+
names = if defined?(Storage::Registry)
|
|
19
|
+
Storage::Registry.names
|
|
20
|
+
else
|
|
21
|
+
self.class::STORAGE_BACKENDS
|
|
22
|
+
end
|
|
23
|
+
raise Error, "Unknown storage_backend: #{value.inspect}. Use one of: #{names.join(', ')}"
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
end
|