llm_cost_tracker 0.8.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +136 -0
- data/README.md +14 -6
- data/app/assets/llm_cost_tracker/application.css +65 -5
- data/app/controllers/llm_cost_tracker/application_controller.rb +25 -33
- data/app/controllers/llm_cost_tracker/assets_controller.rb +1 -1
- data/app/controllers/llm_cost_tracker/calls_controller.rb +21 -11
- data/app/controllers/llm_cost_tracker/data_quality_controller.rb +4 -0
- data/app/controllers/llm_cost_tracker/reconciliation_controller.rb +106 -0
- data/app/controllers/llm_cost_tracker/tags_controller.rb +15 -1
- data/app/helpers/llm_cost_tracker/application_helper.rb +11 -1
- data/app/helpers/llm_cost_tracker/inline_style_helper.rb +28 -0
- data/app/helpers/llm_cost_tracker/reconciliation_helper.rb +13 -0
- data/app/helpers/llm_cost_tracker/token_usage_helper.rb +5 -1
- data/app/models/llm_cost_tracker/call.rb +0 -3
- data/app/models/llm_cost_tracker/call_line_item.rb +1 -5
- data/app/models/llm_cost_tracker/call_rollup.rb +0 -3
- data/app/models/llm_cost_tracker/call_tag.rb +0 -4
- data/app/models/llm_cost_tracker/ingestion/inbox_entry.rb +0 -4
- data/app/models/llm_cost_tracker/ingestion/lease.rb +0 -3
- data/app/models/llm_cost_tracker/provider_invoice.rb +7 -3
- data/app/models/llm_cost_tracker/provider_invoice_import.rb +29 -0
- data/app/services/llm_cost_tracker/dashboard/data_quality.rb +33 -4
- data/app/services/llm_cost_tracker/dashboard/filter.rb +6 -4
- data/app/services/llm_cost_tracker/dashboard/setup_state.rb +110 -0
- data/app/views/layouts/llm_cost_tracker/application.html.erb +6 -1
- data/app/views/llm_cost_tracker/calls/show.html.erb +26 -41
- data/app/views/llm_cost_tracker/dashboard/index.html.erb +9 -9
- data/app/views/llm_cost_tracker/data_quality/index.html.erb +92 -53
- data/app/views/llm_cost_tracker/reconciliation/index.html.erb +183 -0
- data/app/views/llm_cost_tracker/shared/_bar.html.erb +1 -1
- data/app/views/llm_cost_tracker/shared/_filters.html.erb +3 -0
- data/app/views/llm_cost_tracker/shared/_metric_stack.html.erb +1 -1
- data/app/views/llm_cost_tracker/tags/show.html.erb +60 -0
- data/config/routes.rb +3 -2
- data/lib/llm_cost_tracker/billing/components.rb +45 -3
- data/lib/llm_cost_tracker/billing/components.yml +71 -0
- data/lib/llm_cost_tracker/billing/cost_status.rb +21 -25
- data/lib/llm_cost_tracker/billing/line_item.rb +16 -50
- data/lib/llm_cost_tracker/budget.rb +31 -7
- data/lib/llm_cost_tracker/capture/stream_collector.rb +113 -34
- data/lib/llm_cost_tracker/capture/stream_tracker.rb +40 -5
- data/lib/llm_cost_tracker/configuration.rb +72 -17
- data/lib/llm_cost_tracker/doctor/capture_verifier.rb +1 -1
- data/lib/llm_cost_tracker/doctor/cost_drift_check.rb +2 -0
- data/lib/llm_cost_tracker/doctor/ingestion_check.rb +30 -4
- data/lib/llm_cost_tracker/doctor/invoice_reconciliation_check.rb +164 -0
- data/lib/llm_cost_tracker/doctor/legacy_audit_check.rb +0 -2
- data/lib/llm_cost_tracker/doctor/legacy_billing_status_check.rb +0 -2
- data/lib/llm_cost_tracker/doctor/schema_check.rb +5 -2
- data/lib/llm_cost_tracker/doctor.rb +72 -14
- data/lib/llm_cost_tracker/engine.rb +8 -0
- data/lib/llm_cost_tracker/errors.rb +3 -2
- data/lib/llm_cost_tracker/event.rb +48 -1
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/async_ingestion_generator.rb +43 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/call_rollups_generator.rb +43 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/install_generator.rb +17 -26
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/reconciliation_generator.rb +34 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/create_llm_cost_tracker_async_ingestion.rb.erb +29 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/create_llm_cost_tracker_call_rollups.rb.erb +15 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/create_llm_cost_tracker_calls.rb.erb +5 -58
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/create_llm_cost_tracker_reconciliation.rb.erb +60 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/initializer.rb.erb +35 -25
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_call_rollups_provider.rb.erb +35 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_call_tags_key_value_index.rb.erb +32 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_image_tokens.rb.erb +18 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_provider_invoice_imports_provider.rb.erb +32 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/templates/upgrade_provider_invoices_metadata_index.rb.erb +25 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/upgrade_call_rollups_provider_generator.rb +29 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/upgrade_call_tags_key_value_index_generator.rb +30 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/upgrade_image_tokens_generator.rb +29 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/upgrade_provider_invoice_imports_provider_generator.rb +31 -0
- data/lib/llm_cost_tracker/generators/llm_cost_tracker/upgrade_provider_invoices_metadata_index_generator.rb +31 -0
- data/lib/llm_cost_tracker/ingestion/batch.rb +5 -2
- data/lib/llm_cost_tracker/ingestion/inbox.rb +3 -25
- data/lib/llm_cost_tracker/ingestion/pool.rb +44 -0
- data/lib/llm_cost_tracker/ingestion/worker.rb +28 -34
- data/lib/llm_cost_tracker/ingestion.rb +48 -11
- data/lib/llm_cost_tracker/integrations/anthropic.rb +31 -26
- data/lib/llm_cost_tracker/integrations/base.rb +35 -15
- data/lib/llm_cost_tracker/integrations/openai.rb +345 -84
- data/lib/llm_cost_tracker/integrations/ruby_llm.rb +111 -14
- data/lib/llm_cost_tracker/integrations.rb +33 -14
- data/lib/llm_cost_tracker/ledger/period/totals.rb +25 -7
- data/lib/llm_cost_tracker/ledger/rollups.rb +22 -17
- data/lib/llm_cost_tracker/ledger/schema/call_line_items.rb +41 -1
- data/lib/llm_cost_tracker/ledger/schema/call_rollups.rb +16 -6
- data/lib/llm_cost_tracker/ledger/schema/call_tags.rb +28 -2
- data/lib/llm_cost_tracker/ledger/schema/calls.rb +2 -4
- data/lib/llm_cost_tracker/ledger/schema/ingestion_inbox_entries.rb +57 -0
- data/lib/llm_cost_tracker/ledger/schema/ingestion_leases.rb +52 -0
- data/lib/llm_cost_tracker/ledger/schema/provider_invoice_imports.rb +56 -0
- data/lib/llm_cost_tracker/ledger/schema/provider_invoices.rb +28 -13
- data/lib/llm_cost_tracker/ledger/store.rb +34 -31
- data/lib/llm_cost_tracker/ledger/tags/encoding.rb +37 -0
- data/lib/llm_cost_tracker/ledger/tags/query.rb +2 -2
- data/lib/llm_cost_tracker/ledger.rb +2 -1
- data/lib/llm_cost_tracker/logging.rb +0 -4
- data/lib/llm_cost_tracker/masking.rb +39 -0
- data/lib/llm_cost_tracker/middleware/faraday.rb +120 -33
- data/lib/llm_cost_tracker/parsers/anthropic.rb +36 -28
- data/lib/llm_cost_tracker/parsers/azure.rb +46 -0
- data/lib/llm_cost_tracker/parsers/base.rb +53 -43
- data/lib/llm_cost_tracker/parsers/gemini.rb +24 -22
- data/lib/llm_cost_tracker/parsers/openai.rb +20 -38
- data/lib/llm_cost_tracker/parsers/openai_compatible.rb +26 -39
- data/lib/llm_cost_tracker/parsers/openai_service_charges.rb +81 -13
- data/lib/llm_cost_tracker/parsers/openai_usage.rb +126 -59
- data/lib/llm_cost_tracker/parsers.rb +31 -4
- data/lib/llm_cost_tracker/prices.json +572 -493
- data/lib/llm_cost_tracker/pricing/backfill.rb +140 -0
- data/lib/llm_cost_tracker/pricing/effective_prices.rb +7 -40
- data/lib/llm_cost_tracker/pricing/estimator.rb +33 -0
- data/lib/llm_cost_tracker/pricing/explainer.rb +4 -1
- data/lib/llm_cost_tracker/pricing/lookup.rb +73 -5
- data/lib/llm_cost_tracker/pricing/mode.rb +76 -0
- data/lib/llm_cost_tracker/pricing/registry.rb +3 -8
- data/lib/llm_cost_tracker/pricing/service_charges.rb +14 -12
- data/lib/llm_cost_tracker/pricing/{sync_change_printer.rb → sync/change_printer.rb} +3 -3
- data/lib/llm_cost_tracker/pricing/sync/registry_writer.rb +62 -1
- data/lib/llm_cost_tracker/pricing/sync.rb +4 -10
- data/lib/llm_cost_tracker/pricing/unknown.rb +5 -2
- data/lib/llm_cost_tracker/pricing.rb +117 -44
- data/lib/llm_cost_tracker/providers/anthropic/tier_classification.rb +22 -0
- data/lib/llm_cost_tracker/providers/azure/hosts.rb +17 -0
- data/lib/llm_cost_tracker/providers/gemini/model_families.rb +17 -0
- data/lib/llm_cost_tracker/providers/openai/hosts.rb +35 -0
- data/lib/llm_cost_tracker/providers/openai/model_families.rb +51 -0
- data/lib/llm_cost_tracker/railtie.rb +8 -0
- data/lib/llm_cost_tracker/reconcile_tasks.rb +134 -0
- data/lib/llm_cost_tracker/reconciliation/diff.rb +409 -0
- data/lib/llm_cost_tracker/reconciliation/diff_result.rb +44 -0
- data/lib/llm_cost_tracker/reconciliation/import_result.rb +19 -0
- data/lib/llm_cost_tracker/reconciliation/importer.rb +254 -0
- data/lib/llm_cost_tracker/reconciliation/sources/anthropic_usage.rb +172 -0
- data/lib/llm_cost_tracker/reconciliation/sources/fingerprint.rb +20 -0
- data/lib/llm_cost_tracker/reconciliation/sources/openai_usage.rb +142 -0
- data/lib/llm_cost_tracker/reconciliation.rb +118 -0
- data/lib/llm_cost_tracker/report/data.rb +4 -1
- data/lib/llm_cost_tracker/report.rb +0 -4
- data/lib/llm_cost_tracker/retention.rb +31 -6
- data/lib/llm_cost_tracker/tags/context.rb +3 -4
- data/lib/llm_cost_tracker/tags/sanitizer.rb +73 -21
- data/lib/llm_cost_tracker/token_usage.rb +14 -2
- data/lib/llm_cost_tracker/tracker.rb +41 -55
- data/lib/llm_cost_tracker/version.rb +1 -1
- data/lib/llm_cost_tracker.rb +19 -14
- data/lib/tasks/llm_cost_tracker.rake +41 -4
- metadata +49 -3
- data/lib/llm_cost_tracker/usage_capture.rb +0 -58
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require_relative "base"
|
|
4
|
+
require_relative "../providers/anthropic/tier_classification"
|
|
4
5
|
|
|
5
6
|
module LlmCostTracker
|
|
6
7
|
module Integrations
|
|
@@ -25,7 +26,7 @@ module LlmCostTracker
|
|
|
25
26
|
patch_target(
|
|
26
27
|
"RubyLLM::Provider",
|
|
27
28
|
with: ProviderPatch,
|
|
28
|
-
methods: %i[slug complete embed transcribe]
|
|
29
|
+
methods: %i[slug complete embed transcribe paint moderate]
|
|
29
30
|
)
|
|
30
31
|
]
|
|
31
32
|
end
|
|
@@ -65,6 +66,69 @@ module LlmCostTracker
|
|
|
65
66
|
)
|
|
66
67
|
end
|
|
67
68
|
|
|
69
|
+
def record_image(provider, response, request:, latency_ms:)
|
|
70
|
+
usage = object_value(response, :usage)
|
|
71
|
+
usage = {} unless usage.is_a?(Hash)
|
|
72
|
+
raw_input = (usage[:input_tokens] || usage["input_tokens"]).to_i
|
|
73
|
+
raw_output = (usage[:output_tokens] || usage["output_tokens"]).to_i
|
|
74
|
+
image_input = image_token_detail(usage, :input)
|
|
75
|
+
image_output = image_token_detail(usage, :output)
|
|
76
|
+
text_input = [raw_input - image_input, 0].max
|
|
77
|
+
text_output = [raw_output - image_output, 0].max
|
|
78
|
+
record_passthrough(
|
|
79
|
+
provider: provider_slug(provider),
|
|
80
|
+
model: response_model_id(response) || model_id(request[:model]),
|
|
81
|
+
response: response,
|
|
82
|
+
latency_ms: latency_ms,
|
|
83
|
+
input_tokens: text_input,
|
|
84
|
+
image_input_tokens: image_input,
|
|
85
|
+
output_tokens: text_output,
|
|
86
|
+
image_output_tokens: image_output
|
|
87
|
+
)
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
def record_moderation(provider, response, request:, latency_ms:)
|
|
91
|
+
record_passthrough(
|
|
92
|
+
provider: provider_slug(provider),
|
|
93
|
+
model: response_model_id(response) || model_id(request[:model]),
|
|
94
|
+
response: response,
|
|
95
|
+
latency_ms: latency_ms,
|
|
96
|
+
input_tokens: 0,
|
|
97
|
+
output_tokens: 0
|
|
98
|
+
)
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
def image_token_detail(usage, direction)
|
|
102
|
+
container_key = direction == :input ? :input_tokens_details : :output_tokens_details
|
|
103
|
+
details = usage[container_key] || usage[container_key.to_s] || {}
|
|
104
|
+
return 0 unless details.is_a?(Hash)
|
|
105
|
+
|
|
106
|
+
(details[:image_tokens] || details["image_tokens"]).to_i
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
def record_passthrough(provider:, model:, response:, latency_ms:, input_tokens:, output_tokens:,
|
|
110
|
+
image_input_tokens: 0, image_output_tokens: 0)
|
|
111
|
+
return unless active?
|
|
112
|
+
|
|
113
|
+
record_safely do
|
|
114
|
+
LlmCostTracker::Tracker.record(
|
|
115
|
+
event: Event.build(
|
|
116
|
+
provider: provider,
|
|
117
|
+
model: model,
|
|
118
|
+
token_usage: TokenUsage.build(
|
|
119
|
+
input_tokens: input_tokens,
|
|
120
|
+
output_tokens: output_tokens,
|
|
121
|
+
image_input_tokens: image_input_tokens,
|
|
122
|
+
image_output_tokens: image_output_tokens
|
|
123
|
+
),
|
|
124
|
+
usage_source: :sdk_response,
|
|
125
|
+
provider_response_id: provider_response_id(response)
|
|
126
|
+
),
|
|
127
|
+
latency_ms: latency_ms
|
|
128
|
+
)
|
|
129
|
+
end
|
|
130
|
+
end
|
|
131
|
+
|
|
68
132
|
def record_usage(provider:, model:, response:, latency_ms:, stream:, output_tokens: nil)
|
|
69
133
|
return unless active?
|
|
70
134
|
|
|
@@ -77,10 +141,10 @@ module LlmCostTracker
|
|
|
77
141
|
hidden_output = object_value(response, :thinking_tokens, :reasoning_tokens).to_i
|
|
78
142
|
|
|
79
143
|
LlmCostTracker::Tracker.record(
|
|
80
|
-
|
|
144
|
+
event: Event.build(
|
|
81
145
|
provider: provider,
|
|
82
146
|
model: model,
|
|
83
|
-
pricing_mode: pricing_mode(response),
|
|
147
|
+
pricing_mode: pricing_mode(provider: provider, response: response),
|
|
84
148
|
token_usage: TokenUsage.build(
|
|
85
149
|
input_tokens: regular_input_tokens(input_tokens, cache_read),
|
|
86
150
|
output_tokens: output_tokens.to_i,
|
|
@@ -119,13 +183,16 @@ module LlmCostTracker
|
|
|
119
183
|
end
|
|
120
184
|
|
|
121
185
|
def provider_response_id(response)
|
|
122
|
-
object_value(response, :id, :provider_response_id)
|
|
186
|
+
object_value(response, :id, :provider_response_id)
|
|
123
187
|
end
|
|
124
188
|
|
|
125
|
-
def pricing_mode(response)
|
|
126
|
-
object_value(response, :pricing_mode, :service_tier)
|
|
127
|
-
|
|
128
|
-
|
|
189
|
+
def pricing_mode(provider:, response:)
|
|
190
|
+
raw = object_value(response, :pricing_mode, :service_tier)
|
|
191
|
+
if provider == "anthropic" && LlmCostTracker::Providers::Anthropic::TierClassification.standard_equivalent_tier?(raw)
|
|
192
|
+
return nil
|
|
193
|
+
end
|
|
194
|
+
|
|
195
|
+
raw
|
|
129
196
|
end
|
|
130
197
|
end
|
|
131
198
|
|
|
@@ -133,14 +200,14 @@ module LlmCostTracker
|
|
|
133
200
|
def complete(*args, **kwargs, &)
|
|
134
201
|
integration = LlmCostTracker::Integrations::RubyLlm
|
|
135
202
|
request = integration.request_params(args, kwargs)
|
|
136
|
-
integration.enforce_budget!
|
|
203
|
+
integration.enforce_budget!(request: request)
|
|
137
204
|
started_at = LlmCostTracker::Timing.now_monotonic
|
|
138
205
|
response = super
|
|
139
206
|
integration.record_completion(
|
|
140
207
|
self,
|
|
141
208
|
response,
|
|
142
209
|
request: request,
|
|
143
|
-
latency_ms:
|
|
210
|
+
latency_ms: LlmCostTracker::Timing.elapsed_ms(started_at),
|
|
144
211
|
stream: integration.streaming_request?(request, has_block: block_given?)
|
|
145
212
|
)
|
|
146
213
|
response
|
|
@@ -149,14 +216,14 @@ module LlmCostTracker
|
|
|
149
216
|
def embed(*args, **kwargs)
|
|
150
217
|
integration = LlmCostTracker::Integrations::RubyLlm
|
|
151
218
|
request = integration.request_params(args, kwargs)
|
|
152
|
-
integration.enforce_budget!
|
|
219
|
+
integration.enforce_budget!(request: request)
|
|
153
220
|
started_at = LlmCostTracker::Timing.now_monotonic
|
|
154
221
|
response = super
|
|
155
222
|
integration.record_embedding(
|
|
156
223
|
self,
|
|
157
224
|
response,
|
|
158
225
|
request: request,
|
|
159
|
-
latency_ms:
|
|
226
|
+
latency_ms: LlmCostTracker::Timing.elapsed_ms(started_at)
|
|
160
227
|
)
|
|
161
228
|
response
|
|
162
229
|
end
|
|
@@ -164,14 +231,44 @@ module LlmCostTracker
|
|
|
164
231
|
def transcribe(*args, **kwargs)
|
|
165
232
|
integration = LlmCostTracker::Integrations::RubyLlm
|
|
166
233
|
request = integration.request_params(args, kwargs)
|
|
167
|
-
integration.enforce_budget!
|
|
234
|
+
integration.enforce_budget!(request: request)
|
|
168
235
|
started_at = LlmCostTracker::Timing.now_monotonic
|
|
169
236
|
response = super
|
|
170
237
|
integration.record_transcription(
|
|
171
238
|
self,
|
|
172
239
|
response,
|
|
173
240
|
request: request,
|
|
174
|
-
latency_ms:
|
|
241
|
+
latency_ms: LlmCostTracker::Timing.elapsed_ms(started_at)
|
|
242
|
+
)
|
|
243
|
+
response
|
|
244
|
+
end
|
|
245
|
+
|
|
246
|
+
def paint(*args, **kwargs)
|
|
247
|
+
integration = LlmCostTracker::Integrations::RubyLlm
|
|
248
|
+
request = integration.request_params(args, kwargs)
|
|
249
|
+
integration.enforce_budget!(request: request)
|
|
250
|
+
started_at = LlmCostTracker::Timing.now_monotonic
|
|
251
|
+
response = super
|
|
252
|
+
integration.record_image(
|
|
253
|
+
self,
|
|
254
|
+
response,
|
|
255
|
+
request: request,
|
|
256
|
+
latency_ms: LlmCostTracker::Timing.elapsed_ms(started_at)
|
|
257
|
+
)
|
|
258
|
+
response
|
|
259
|
+
end
|
|
260
|
+
|
|
261
|
+
def moderate(*args, **kwargs)
|
|
262
|
+
integration = LlmCostTracker::Integrations::RubyLlm
|
|
263
|
+
request = integration.request_params(args, kwargs)
|
|
264
|
+
integration.enforce_budget!(request: request)
|
|
265
|
+
started_at = LlmCostTracker::Timing.now_monotonic
|
|
266
|
+
response = super
|
|
267
|
+
integration.record_moderation(
|
|
268
|
+
self,
|
|
269
|
+
response,
|
|
270
|
+
request: request,
|
|
271
|
+
latency_ms: LlmCostTracker::Timing.elapsed_ms(started_at)
|
|
175
272
|
)
|
|
176
273
|
response
|
|
177
274
|
end
|
|
@@ -1,26 +1,29 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require_relative "errors"
|
|
4
|
-
require_relative "
|
|
5
|
-
require_relative "integrations/anthropic"
|
|
6
|
-
require_relative "integrations/ruby_llm"
|
|
4
|
+
require_relative "logging"
|
|
7
5
|
|
|
8
6
|
module LlmCostTracker
|
|
9
7
|
module Integrations
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
8
|
+
autoload :Base, "llm_cost_tracker/integrations/base"
|
|
9
|
+
autoload :Openai, "llm_cost_tracker/integrations/openai"
|
|
10
|
+
autoload :Anthropic, "llm_cost_tracker/integrations/anthropic"
|
|
11
|
+
autoload :RubyLlm, "llm_cost_tracker/integrations/ruby_llm"
|
|
12
|
+
|
|
13
|
+
INTEGRATION_CONSTANTS = { openai: :Openai, anthropic: :Anthropic, ruby_llm: :RubyLlm }.freeze
|
|
14
|
+
DOUBLE_INSTRUMENTATION_OVERLAPS = %i[openai anthropic].freeze
|
|
15
|
+
private_constant :DOUBLE_INSTRUMENTATION_OVERLAPS
|
|
15
16
|
|
|
16
17
|
module_function
|
|
17
18
|
|
|
18
19
|
def install!(names = LlmCostTracker.configuration.instrumented_integrations)
|
|
19
|
-
normalize(names)
|
|
20
|
+
normalized = normalize(names)
|
|
21
|
+
warn_double_instrumentation(normalized)
|
|
22
|
+
normalized.each { |name| fetch(name).install }
|
|
20
23
|
end
|
|
21
24
|
|
|
22
25
|
def checks(names = LlmCostTracker.configuration.instrumented_integrations)
|
|
23
|
-
return [Base::Result.new(:
|
|
26
|
+
return [Base::Result.new(:ok, "integrations", "no SDK integrations enabled")] if names.empty?
|
|
24
27
|
|
|
25
28
|
normalize(names).map { |name| fetch(name).status }
|
|
26
29
|
end
|
|
@@ -29,15 +32,31 @@ module LlmCostTracker
|
|
|
29
32
|
Array(names).flatten.uniq
|
|
30
33
|
end
|
|
31
34
|
|
|
35
|
+
def warn_double_instrumentation(names)
|
|
36
|
+
return unless names.include?(:ruby_llm)
|
|
37
|
+
|
|
38
|
+
overlapping = names & DOUBLE_INSTRUMENTATION_OVERLAPS
|
|
39
|
+
return if overlapping.empty?
|
|
40
|
+
|
|
41
|
+
Logging.warn(
|
|
42
|
+
":ruby_llm is enabled together with #{overlapping.map(&:inspect).join(', ')}. " \
|
|
43
|
+
"RubyLLM uses HTTP underneath, so calls routed to those providers may be recorded twice " \
|
|
44
|
+
"(once via the SDK patch, once via the Faraday parser). Pick one path per provider."
|
|
45
|
+
)
|
|
46
|
+
end
|
|
47
|
+
|
|
32
48
|
def fetch(name)
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
raise LlmCostTracker::Error,
|
|
49
|
+
const_name = INTEGRATION_CONSTANTS[name.to_sym]
|
|
50
|
+
unless const_name
|
|
51
|
+
raise LlmCostTracker::Error,
|
|
52
|
+
"Unknown integration: #{name.inspect}. Use one of: #{names.join(', ')}"
|
|
36
53
|
end
|
|
54
|
+
|
|
55
|
+
const_get(const_name)
|
|
37
56
|
end
|
|
38
57
|
|
|
39
58
|
def names
|
|
40
|
-
|
|
59
|
+
INTEGRATION_CONSTANTS.keys
|
|
41
60
|
end
|
|
42
61
|
end
|
|
43
62
|
end
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require "bigdecimal"
|
|
4
|
+
|
|
3
5
|
require_relative "../period"
|
|
4
6
|
|
|
5
7
|
module LlmCostTracker
|
|
@@ -26,28 +28,44 @@ module LlmCostTracker
|
|
|
26
28
|
attr_reader :periods, :time
|
|
27
29
|
|
|
28
30
|
def snapshot_totals
|
|
29
|
-
values = periods.to_h { |period| [period, 0
|
|
31
|
+
values = periods.to_h { |period| [period, BigDecimal("0")] }
|
|
30
32
|
period_by_name = periods.to_h { |period| [period.name, period] }
|
|
31
33
|
sql = periods.map { |period| snapshot_select(period) }.join(" UNION ALL ")
|
|
32
34
|
LlmCostTracker::Call.find_by_sql(sql).each do |row|
|
|
33
35
|
period = period_by_name.fetch(row.period_key)
|
|
34
|
-
values[period] = row.total_cost.
|
|
36
|
+
values[period] = BigDecimal(row.total_cost.to_s)
|
|
35
37
|
end
|
|
36
38
|
values
|
|
37
39
|
end
|
|
38
40
|
|
|
39
41
|
def snapshot_select(period)
|
|
40
42
|
start = Period.range_start(period, time)
|
|
43
|
+
components = [period_total_sql(period, start)]
|
|
44
|
+
components << pending_total_sql(start) if Ingestion.async?
|
|
41
45
|
"SELECT #{connection.quote(period.name)} AS period_key, " \
|
|
42
|
-
"(#{
|
|
46
|
+
"(#{components.join(') + (')}) AS total_cost"
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def period_total_sql(period, start)
|
|
50
|
+
if LlmCostTracker.configuration.cache_rollups
|
|
51
|
+
"GREATEST(COALESCE(#{rollup_sum_sql(period)}, 0), COALESCE(#{calls_sum_sql(start)}, 0))"
|
|
52
|
+
else
|
|
53
|
+
"COALESCE(#{calls_sum_sql(start)}, 0)"
|
|
54
|
+
end
|
|
43
55
|
end
|
|
44
56
|
|
|
45
|
-
def
|
|
57
|
+
def rollup_sum_sql(period)
|
|
46
58
|
table = connection.quote_table_name("llm_cost_tracker_call_rollups")
|
|
47
|
-
"
|
|
59
|
+
"(SELECT SUM(total_cost) FROM #{table} " \
|
|
48
60
|
"WHERE period = #{connection.quote(Period::PERIODS.fetch(period))} " \
|
|
49
|
-
"AND period_start = #{connection.quote(Period.bucket(period, time))}
|
|
50
|
-
|
|
61
|
+
"AND period_start = #{connection.quote(Period.bucket(period, time))})"
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def calls_sum_sql(start)
|
|
65
|
+
table = connection.quote_table_name("llm_cost_tracker_calls")
|
|
66
|
+
tracked_at = connection.quote_column_name("tracked_at")
|
|
67
|
+
"(SELECT SUM(total_cost) FROM #{table} " \
|
|
68
|
+
"WHERE #{tracked_at} BETWEEN #{connection.quote(start)} AND #{connection.quote(time)})"
|
|
51
69
|
end
|
|
52
70
|
|
|
53
71
|
def pending_total_sql(start)
|
|
@@ -34,23 +34,26 @@ module LlmCostTracker
|
|
|
34
34
|
private
|
|
35
35
|
|
|
36
36
|
def period_rows(event)
|
|
37
|
-
currency =
|
|
37
|
+
currency = currency_from_snapshot(event.pricing_snapshot)
|
|
38
|
+
provider = event.provider.to_s
|
|
38
39
|
Period::PERIODS.map do |period, name|
|
|
39
40
|
{
|
|
40
41
|
period: name,
|
|
41
42
|
period_start: Period.bucket(period, event.tracked_at),
|
|
42
43
|
currency: currency,
|
|
44
|
+
provider: provider,
|
|
43
45
|
total_cost: event.total_cost
|
|
44
46
|
}
|
|
45
47
|
end
|
|
46
48
|
end
|
|
47
49
|
|
|
48
50
|
def period_rows_for_events(events)
|
|
49
|
-
call_rollups(events).map do |(period, period_start, currency), total_cost|
|
|
51
|
+
call_rollups(events).map do |(period, period_start, currency, provider), total_cost|
|
|
50
52
|
{
|
|
51
53
|
period: period,
|
|
52
54
|
period_start: period_start,
|
|
53
55
|
currency: currency,
|
|
56
|
+
provider: provider,
|
|
54
57
|
total_cost: total_cost
|
|
55
58
|
}
|
|
56
59
|
end
|
|
@@ -58,30 +61,34 @@ module LlmCostTracker
|
|
|
58
61
|
|
|
59
62
|
def call_rollups(events)
|
|
60
63
|
events.each_with_object(Hash.new { |totals, key| totals[key] = BigDecimal("0") }) do |event, totals|
|
|
61
|
-
currency =
|
|
64
|
+
currency = currency_from_snapshot(event.pricing_snapshot)
|
|
65
|
+
provider = event.provider.to_s
|
|
62
66
|
Period::PERIODS.each do |period, name|
|
|
63
|
-
|
|
67
|
+
key = [name, Period.bucket(period, event.tracked_at), currency, provider]
|
|
68
|
+
totals[key] += BigDecimal(event.total_cost.to_s)
|
|
64
69
|
end
|
|
65
70
|
end
|
|
66
71
|
end
|
|
67
72
|
|
|
68
73
|
def period_decrement_totals(call_rows)
|
|
69
74
|
call_rows.each_with_object(Hash.new { |totals, key| totals[key] = BigDecimal("0") }) do |row, totals|
|
|
70
|
-
_id, tracked_at, total_cost, pricing_snapshot = row
|
|
75
|
+
_id, tracked_at, total_cost, pricing_snapshot, provider = row
|
|
71
76
|
next unless total_cost
|
|
72
77
|
|
|
73
78
|
currency = currency_from_snapshot(pricing_snapshot)
|
|
79
|
+
provider_key = provider.to_s
|
|
74
80
|
Period::PERIODS.each_key do |period|
|
|
75
|
-
totals[[period, Period.bucket(period, tracked_at), currency]] += total_cost
|
|
81
|
+
totals[[period, Period.bucket(period, tracked_at), currency, provider_key]] += total_cost
|
|
76
82
|
end
|
|
77
83
|
end
|
|
78
84
|
end
|
|
79
85
|
|
|
80
86
|
def apply_decrements(totals)
|
|
81
87
|
now = Time.now.utc
|
|
82
|
-
buckets_by_period = totals.each_with_object({}) do |(
|
|
83
|
-
|
|
84
|
-
grouped[[period, currency]]
|
|
88
|
+
buckets_by_period = totals.each_with_object({}) do |(key, amount), grouped|
|
|
89
|
+
period, period_start, currency, provider = key
|
|
90
|
+
grouped[[period, currency, provider]] ||= {}
|
|
91
|
+
grouped[[period, currency, provider]][period_start] = amount
|
|
85
92
|
end
|
|
86
93
|
|
|
87
94
|
conn = LlmCostTracker::CallRollup.connection
|
|
@@ -89,10 +96,11 @@ module LlmCostTracker
|
|
|
89
96
|
period_col = conn.quote_column_name("period")
|
|
90
97
|
start_col = conn.quote_column_name("period_start")
|
|
91
98
|
currency_col = conn.quote_column_name("currency")
|
|
99
|
+
provider_col = conn.quote_column_name("provider")
|
|
92
100
|
total_col = conn.quote_column_name("total_cost")
|
|
93
101
|
updated_col = conn.quote_column_name("updated_at")
|
|
94
102
|
|
|
95
|
-
buckets_by_period.each do |(period, currency), by_start|
|
|
103
|
+
buckets_by_period.each do |(period, currency, provider), by_start|
|
|
96
104
|
case_clauses = by_start.map do |period_start, amount|
|
|
97
105
|
"WHEN #{start_col} = #{conn.quote(period_start)} THEN #{conn.quote(amount)}"
|
|
98
106
|
end.join(" ")
|
|
@@ -104,18 +112,15 @@ module LlmCostTracker
|
|
|
104
112
|
"#{updated_col} = #{conn.quote(now)} " \
|
|
105
113
|
"WHERE #{period_col} = #{conn.quote(Period::PERIODS.fetch(period))} " \
|
|
106
114
|
"AND #{currency_col} = #{conn.quote(currency)} " \
|
|
115
|
+
"AND #{provider_col} = #{conn.quote(provider)} " \
|
|
107
116
|
"AND #{start_col} IN (#{starts})"
|
|
108
117
|
)
|
|
109
118
|
end
|
|
110
119
|
end
|
|
111
120
|
|
|
112
|
-
def currency_for(event)
|
|
113
|
-
snapshot = event.respond_to?(:pricing_snapshot) ? event.pricing_snapshot : nil
|
|
114
|
-
currency_from_snapshot(snapshot)
|
|
115
|
-
end
|
|
116
|
-
|
|
117
121
|
def currency_from_snapshot(snapshot)
|
|
118
|
-
(snapshot.is_a?(Hash) && (snapshot["currency"] || snapshot[:currency])) || DEFAULT_CURRENCY
|
|
122
|
+
value = (snapshot.is_a?(Hash) && (snapshot["currency"] || snapshot[:currency])) || DEFAULT_CURRENCY
|
|
123
|
+
value.to_s.upcase
|
|
119
124
|
end
|
|
120
125
|
|
|
121
126
|
def upsert_call_rollups(rows)
|
|
@@ -130,7 +135,7 @@ module LlmCostTracker
|
|
|
130
135
|
def call_rollups_unique_by
|
|
131
136
|
return unless LlmCostTracker::CallRollup.connection.supports_insert_conflict_target?
|
|
132
137
|
|
|
133
|
-
%i[period period_start currency]
|
|
138
|
+
%i[period period_start currency provider]
|
|
134
139
|
end
|
|
135
140
|
end
|
|
136
141
|
end
|
|
@@ -27,6 +27,11 @@ module LlmCostTracker
|
|
|
27
27
|
provider_field
|
|
28
28
|
provider_item_id
|
|
29
29
|
details
|
|
30
|
+
created_at
|
|
31
|
+
].freeze
|
|
32
|
+
|
|
33
|
+
REQUIRED_INDEX_COLUMNS = [
|
|
34
|
+
%w[llm_cost_tracker_call_id position]
|
|
30
35
|
].freeze
|
|
31
36
|
|
|
32
37
|
class << self
|
|
@@ -37,11 +42,46 @@ module LlmCostTracker
|
|
|
37
42
|
return ["#{table_name} table is missing"] unless connection.data_source_exists?(table_name)
|
|
38
43
|
|
|
39
44
|
columns = LlmCostTracker::CallLineItem.columns_hash
|
|
45
|
+
cache = @schema_capabilities
|
|
46
|
+
return cache.fetch(:errors) if cache && cache.fetch(:columns).equal?(columns)
|
|
47
|
+
|
|
48
|
+
errors = compute_errors(connection, table_name, columns)
|
|
49
|
+
@schema_capabilities = { columns: columns, errors: errors }
|
|
50
|
+
errors
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
private
|
|
54
|
+
|
|
55
|
+
def compute_errors(connection, table_name, columns)
|
|
40
56
|
errors = []
|
|
41
57
|
missing = REQUIRED_COLUMNS - columns.keys
|
|
42
58
|
errors << "missing columns: #{missing.join(', ')}" if missing.any?
|
|
43
59
|
errors.concat(Adapter.json_column_errors(columns["details"], connection, "details"))
|
|
44
|
-
errors
|
|
60
|
+
errors.concat(missing_index_errors(connection, table_name))
|
|
61
|
+
errors << missing_fk_error(connection, table_name) if missing_fk?(connection, table_name)
|
|
62
|
+
errors.compact
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def missing_index_errors(connection, table_name)
|
|
66
|
+
existing = connection.indexes(table_name).map { |index| Array(index.columns).map(&:to_s) }
|
|
67
|
+
REQUIRED_INDEX_COLUMNS.filter_map do |required|
|
|
68
|
+
next if existing.any? { |columns| columns == required }
|
|
69
|
+
|
|
70
|
+
"missing index on (#{required.join(', ')})"
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def missing_fk?(connection, table_name)
|
|
75
|
+
connection.foreign_keys(table_name).none? do |fk|
|
|
76
|
+
fk.column.to_s == "llm_cost_tracker_call_id" &&
|
|
77
|
+
fk.to_table.to_s == "llm_cost_tracker_calls"
|
|
78
|
+
end
|
|
79
|
+
rescue NotImplementedError, NoMethodError
|
|
80
|
+
false
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def missing_fk_error(_connection, _table_name)
|
|
84
|
+
"missing foreign key on llm_cost_tracker_call_id referencing llm_cost_tracker_calls"
|
|
45
85
|
end
|
|
46
86
|
end
|
|
47
87
|
end
|
|
@@ -6,8 +6,8 @@ module LlmCostTracker
|
|
|
6
6
|
module Ledger
|
|
7
7
|
module Schema
|
|
8
8
|
module CallRollups
|
|
9
|
-
REQUIRED_COLUMNS = %w[period period_start currency total_cost].freeze
|
|
10
|
-
UNIQUE_COLUMNS = %i[period period_start currency].freeze
|
|
9
|
+
REQUIRED_COLUMNS = %w[period period_start currency provider total_cost created_at updated_at].freeze
|
|
10
|
+
UNIQUE_COLUMNS = %i[period period_start currency provider].freeze
|
|
11
11
|
|
|
12
12
|
class << self
|
|
13
13
|
def current_schema_errors
|
|
@@ -16,17 +16,27 @@ module LlmCostTracker
|
|
|
16
16
|
table_name = LlmCostTracker::CallRollup.table_name
|
|
17
17
|
return ["#{table_name} table is missing"] unless connection.data_source_exists?(table_name)
|
|
18
18
|
|
|
19
|
+
columns = LlmCostTracker::CallRollup.columns_hash
|
|
20
|
+
cache = @schema_capabilities
|
|
21
|
+
return cache.fetch(:errors) if cache && cache.fetch(:columns).equal?(columns)
|
|
22
|
+
|
|
23
|
+
errors = compute_errors(connection, table_name, columns)
|
|
24
|
+
@schema_capabilities = { columns: columns, errors: errors }
|
|
25
|
+
errors
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
private
|
|
29
|
+
|
|
30
|
+
def compute_errors(connection, table_name, columns)
|
|
19
31
|
errors = []
|
|
20
|
-
missing = REQUIRED_COLUMNS -
|
|
32
|
+
missing = REQUIRED_COLUMNS - columns.keys
|
|
21
33
|
errors << "missing columns: #{missing.join(', ')}" if missing.any?
|
|
22
34
|
unless unique_period_index?(connection, table_name)
|
|
23
|
-
errors << "missing unique index: period, period_start, currency"
|
|
35
|
+
errors << "missing unique index: period, period_start, currency, provider"
|
|
24
36
|
end
|
|
25
37
|
errors
|
|
26
38
|
end
|
|
27
39
|
|
|
28
|
-
private
|
|
29
|
-
|
|
30
40
|
def unique_period_index?(connection, table_name)
|
|
31
41
|
connection.index_exists?(table_name, UNIQUE_COLUMNS, unique: true)
|
|
32
42
|
end
|
|
@@ -6,6 +6,11 @@ module LlmCostTracker
|
|
|
6
6
|
module CallTags
|
|
7
7
|
REQUIRED_COLUMNS = %w[llm_cost_tracker_call_id key value].freeze
|
|
8
8
|
|
|
9
|
+
REQUIRED_INDEX_COLUMNS = [
|
|
10
|
+
%w[key value],
|
|
11
|
+
%w[llm_cost_tracker_call_id]
|
|
12
|
+
].freeze
|
|
13
|
+
|
|
9
14
|
class << self
|
|
10
15
|
def current_schema_errors
|
|
11
16
|
connection = LlmCostTracker::Call.connection
|
|
@@ -14,10 +19,31 @@ module LlmCostTracker
|
|
|
14
19
|
return ["#{table_name} table is missing"] unless connection.data_source_exists?(table_name)
|
|
15
20
|
|
|
16
21
|
columns = LlmCostTracker::CallTag.columns_hash
|
|
22
|
+
cache = @schema_capabilities
|
|
23
|
+
return cache.fetch(:errors) if cache && cache.fetch(:columns).equal?(columns)
|
|
24
|
+
|
|
25
|
+
errors = compute_errors(connection, table_name, columns)
|
|
26
|
+
@schema_capabilities = { columns: columns, errors: errors }
|
|
27
|
+
errors
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
private
|
|
31
|
+
|
|
32
|
+
def compute_errors(connection, table_name, columns)
|
|
33
|
+
errors = []
|
|
17
34
|
missing = REQUIRED_COLUMNS - columns.keys
|
|
18
|
-
|
|
35
|
+
errors << "missing columns: #{missing.join(', ')}" if missing.any?
|
|
36
|
+
errors.concat(missing_index_errors(connection, table_name))
|
|
37
|
+
errors
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def missing_index_errors(connection, table_name)
|
|
41
|
+
existing = connection.indexes(table_name).map { |index| Array(index.columns).map(&:to_s) }
|
|
42
|
+
REQUIRED_INDEX_COLUMNS.filter_map do |required|
|
|
43
|
+
next if existing.any? { |columns| (required - columns).empty? }
|
|
19
44
|
|
|
20
|
-
|
|
45
|
+
"missing index on (#{required.join(', ')})"
|
|
46
|
+
end
|
|
21
47
|
end
|
|
22
48
|
end
|
|
23
49
|
end
|
|
@@ -18,6 +18,8 @@ module LlmCostTracker
|
|
|
18
18
|
cache_write_extended_input_tokens
|
|
19
19
|
audio_input_tokens
|
|
20
20
|
audio_output_tokens
|
|
21
|
+
image_input_tokens
|
|
22
|
+
image_output_tokens
|
|
21
23
|
hidden_output_tokens
|
|
22
24
|
total_cost
|
|
23
25
|
latency_ms
|
|
@@ -45,10 +47,6 @@ module LlmCostTracker
|
|
|
45
47
|
private_constant :REQUIRED_INDEXES
|
|
46
48
|
|
|
47
49
|
class << self
|
|
48
|
-
def current_schema?
|
|
49
|
-
current_schema_errors.empty?
|
|
50
|
-
end
|
|
51
|
-
|
|
52
50
|
def current_schema_errors
|
|
53
51
|
schema_capabilities.fetch(:current_schema_errors)
|
|
54
52
|
end
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "adapter"
|
|
4
|
+
|
|
5
|
+
module LlmCostTracker
|
|
6
|
+
module Ledger
|
|
7
|
+
module Schema
|
|
8
|
+
module IngestionInboxEntries
|
|
9
|
+
REQUIRED_COLUMNS = %w[
|
|
10
|
+
event_id
|
|
11
|
+
total_cost
|
|
12
|
+
tracked_at
|
|
13
|
+
payload
|
|
14
|
+
locked_at
|
|
15
|
+
locked_by
|
|
16
|
+
attempts
|
|
17
|
+
last_error
|
|
18
|
+
created_at
|
|
19
|
+
updated_at
|
|
20
|
+
].freeze
|
|
21
|
+
|
|
22
|
+
UNIQUE_COLUMNS = %i[event_id].freeze
|
|
23
|
+
|
|
24
|
+
class << self
|
|
25
|
+
def current_schema_errors
|
|
26
|
+
connection = LlmCostTracker::Ingestion::InboxEntry.connection
|
|
27
|
+
Adapter.ensure_supported!(connection)
|
|
28
|
+
table_name = LlmCostTracker::Ingestion::InboxEntry.table_name
|
|
29
|
+
return ["#{table_name} table is missing"] unless connection.data_source_exists?(table_name)
|
|
30
|
+
|
|
31
|
+
columns = LlmCostTracker::Ingestion::InboxEntry.columns_hash
|
|
32
|
+
cache = @schema_capabilities
|
|
33
|
+
return cache.fetch(:errors) if cache && cache.fetch(:columns).equal?(columns)
|
|
34
|
+
|
|
35
|
+
errors = compute_errors(connection, table_name, columns)
|
|
36
|
+
@schema_capabilities = { columns: columns, errors: errors }
|
|
37
|
+
errors
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
private
|
|
41
|
+
|
|
42
|
+
def compute_errors(connection, table_name, columns)
|
|
43
|
+
errors = []
|
|
44
|
+
missing = REQUIRED_COLUMNS - columns.keys
|
|
45
|
+
errors << "missing columns: #{missing.join(', ')}" if missing.any?
|
|
46
|
+
errors << "missing unique index: event_id" unless event_id_unique_index?(connection, table_name)
|
|
47
|
+
errors
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def event_id_unique_index?(connection, table_name)
|
|
51
|
+
connection.index_exists?(table_name, UNIQUE_COLUMNS, unique: true)
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
end
|