dspy 0.16.0 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +3 -3
- data/lib/dspy/chain_of_thought.rb +13 -16
- data/lib/dspy/code_act.rb +31 -37
- data/lib/dspy/context.rb +67 -0
- data/lib/dspy/evaluate.rb +20 -17
- data/lib/dspy/lm.rb +72 -37
- data/lib/dspy/memory/memory_compactor.rb +5 -6
- data/lib/dspy/memory/memory_manager.rb +5 -4
- data/lib/dspy/observability.rb +109 -0
- data/lib/dspy/predict.rb +18 -6
- data/lib/dspy/propose/grounded_proposer.rb +13 -12
- data/lib/dspy/re_act.rb +34 -41
- data/lib/dspy/registry/registry_manager.rb +8 -10
- data/lib/dspy/registry/signature_registry.rb +40 -52
- data/lib/dspy/storage/program_storage.rb +28 -37
- data/lib/dspy/storage/storage_manager.rb +3 -4
- data/lib/dspy/teleprompt/teleprompter.rb +11 -12
- data/lib/dspy/teleprompt/utils.rb +24 -22
- data/lib/dspy/version.rb +1 -1
- data/lib/dspy.rb +42 -82
- metadata +33 -26
- data/lib/dspy/instrumentation/event_payload_factory.rb +0 -282
- data/lib/dspy/instrumentation/event_payloads.rb +0 -476
- data/lib/dspy/instrumentation/token_tracker.rb +0 -70
- data/lib/dspy/instrumentation.rb +0 -341
- data/lib/dspy/mixins/instrumentation_helpers.rb +0 -120
- data/lib/dspy/subscribers/langfuse_subscriber.rb +0 -669
- data/lib/dspy/subscribers/logger_subscriber.rb +0 -480
- data/lib/dspy/subscribers/newrelic_subscriber.rb +0 -686
- data/lib/dspy/subscribers/otel_subscriber.rb +0 -537
|
@@ -1,476 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
require 'sorbet-runtime'
|
|
4
|
-
|
|
5
|
-
module DSPy
|
|
6
|
-
module Instrumentation
|
|
7
|
-
# Type-safe event payload structures for DSPy instrumentation
|
|
8
|
-
# Each event is a complete T::Struct (no inheritance due to T::Struct limitations)
|
|
9
|
-
|
|
10
|
-
# LM Request event payload
|
|
11
|
-
class LMRequestEvent < T::Struct
|
|
12
|
-
extend T::Sig
|
|
13
|
-
|
|
14
|
-
# Common fields
|
|
15
|
-
const :timestamp, String
|
|
16
|
-
const :duration_ms, Float
|
|
17
|
-
const :cpu_time_ms, Float
|
|
18
|
-
const :status, String
|
|
19
|
-
|
|
20
|
-
# LM-specific fields
|
|
21
|
-
const :gen_ai_operation_name, String
|
|
22
|
-
const :gen_ai_system, String
|
|
23
|
-
const :gen_ai_request_model, String
|
|
24
|
-
const :signature_class, T.nilable(String), default: nil
|
|
25
|
-
const :provider, String
|
|
26
|
-
const :adapter_class, String
|
|
27
|
-
const :input_size, Integer
|
|
28
|
-
|
|
29
|
-
# Error fields (optional)
|
|
30
|
-
const :error_type, T.nilable(String), default: nil
|
|
31
|
-
const :error_message, T.nilable(String), default: nil
|
|
32
|
-
|
|
33
|
-
sig { returns(T::Hash[Symbol, T.untyped]) }
|
|
34
|
-
def to_h
|
|
35
|
-
hash = {
|
|
36
|
-
timestamp: timestamp,
|
|
37
|
-
duration_ms: duration_ms,
|
|
38
|
-
cpu_time_ms: cpu_time_ms,
|
|
39
|
-
status: status,
|
|
40
|
-
gen_ai_operation_name: gen_ai_operation_name,
|
|
41
|
-
gen_ai_system: gen_ai_system,
|
|
42
|
-
gen_ai_request_model: gen_ai_request_model,
|
|
43
|
-
provider: provider,
|
|
44
|
-
adapter_class: adapter_class,
|
|
45
|
-
input_size: input_size
|
|
46
|
-
}
|
|
47
|
-
hash[:signature_class] = signature_class if signature_class
|
|
48
|
-
hash[:error_type] = error_type if error_type
|
|
49
|
-
hash[:error_message] = error_message if error_message
|
|
50
|
-
hash
|
|
51
|
-
end
|
|
52
|
-
end
|
|
53
|
-
|
|
54
|
-
# Token usage event payload
|
|
55
|
-
class LMTokensEvent < T::Struct
|
|
56
|
-
extend T::Sig
|
|
57
|
-
|
|
58
|
-
# Common fields
|
|
59
|
-
const :timestamp, String
|
|
60
|
-
const :status, String
|
|
61
|
-
|
|
62
|
-
# Token-specific fields
|
|
63
|
-
const :input_tokens, Integer
|
|
64
|
-
const :output_tokens, Integer
|
|
65
|
-
const :total_tokens, Integer
|
|
66
|
-
const :gen_ai_system, String
|
|
67
|
-
const :gen_ai_request_model, String
|
|
68
|
-
const :signature_class, T.nilable(String), default: nil
|
|
69
|
-
|
|
70
|
-
sig { returns(T::Hash[Symbol, T.untyped]) }
|
|
71
|
-
def to_h
|
|
72
|
-
hash = {
|
|
73
|
-
timestamp: timestamp,
|
|
74
|
-
status: status,
|
|
75
|
-
input_tokens: input_tokens,
|
|
76
|
-
output_tokens: output_tokens,
|
|
77
|
-
total_tokens: total_tokens,
|
|
78
|
-
gen_ai_system: gen_ai_system,
|
|
79
|
-
gen_ai_request_model: gen_ai_request_model
|
|
80
|
-
}
|
|
81
|
-
hash[:signature_class] = signature_class if signature_class
|
|
82
|
-
hash
|
|
83
|
-
end
|
|
84
|
-
end
|
|
85
|
-
|
|
86
|
-
# LM Response parsed event payload
|
|
87
|
-
class LMResponseParsedEvent < T::Struct
|
|
88
|
-
extend T::Sig
|
|
89
|
-
|
|
90
|
-
# Common fields
|
|
91
|
-
const :timestamp, String
|
|
92
|
-
const :duration_ms, Float
|
|
93
|
-
const :cpu_time_ms, Float
|
|
94
|
-
const :status, String
|
|
95
|
-
|
|
96
|
-
# Response parsing fields
|
|
97
|
-
const :signature_class, String
|
|
98
|
-
const :provider, String
|
|
99
|
-
const :success, T::Boolean
|
|
100
|
-
const :response_length, Integer
|
|
101
|
-
const :parse_type, T.nilable(String), default: nil
|
|
102
|
-
|
|
103
|
-
# Error fields (optional)
|
|
104
|
-
const :error_type, T.nilable(String), default: nil
|
|
105
|
-
const :error_message, T.nilable(String), default: nil
|
|
106
|
-
|
|
107
|
-
sig { returns(T::Hash[Symbol, T.untyped]) }
|
|
108
|
-
def to_h
|
|
109
|
-
hash = {
|
|
110
|
-
timestamp: timestamp,
|
|
111
|
-
duration_ms: duration_ms,
|
|
112
|
-
cpu_time_ms: cpu_time_ms,
|
|
113
|
-
status: status,
|
|
114
|
-
signature_class: signature_class,
|
|
115
|
-
provider: provider,
|
|
116
|
-
success: success,
|
|
117
|
-
response_length: response_length
|
|
118
|
-
}
|
|
119
|
-
hash[:parse_type] = parse_type if parse_type
|
|
120
|
-
hash[:error_type] = error_type if error_type
|
|
121
|
-
hash[:error_message] = error_message if error_message
|
|
122
|
-
hash
|
|
123
|
-
end
|
|
124
|
-
end
|
|
125
|
-
|
|
126
|
-
# Predict event payload
|
|
127
|
-
class PredictEvent < T::Struct
|
|
128
|
-
extend T::Sig
|
|
129
|
-
|
|
130
|
-
# Common fields
|
|
131
|
-
const :timestamp, String
|
|
132
|
-
const :duration_ms, Float
|
|
133
|
-
const :cpu_time_ms, Float
|
|
134
|
-
const :status, String
|
|
135
|
-
|
|
136
|
-
# Predict-specific fields
|
|
137
|
-
const :signature_class, String
|
|
138
|
-
const :module_name, String
|
|
139
|
-
const :model, String
|
|
140
|
-
const :provider, String
|
|
141
|
-
const :input_fields, T::Array[String]
|
|
142
|
-
const :input_size, T.nilable(Integer), default: nil
|
|
143
|
-
const :output_size, T.nilable(Integer), default: nil
|
|
144
|
-
|
|
145
|
-
# Error fields (optional)
|
|
146
|
-
const :error_type, T.nilable(String), default: nil
|
|
147
|
-
const :error_message, T.nilable(String), default: nil
|
|
148
|
-
|
|
149
|
-
sig { returns(T::Hash[Symbol, T.untyped]) }
|
|
150
|
-
def to_h
|
|
151
|
-
hash = {
|
|
152
|
-
timestamp: timestamp,
|
|
153
|
-
duration_ms: duration_ms,
|
|
154
|
-
cpu_time_ms: cpu_time_ms,
|
|
155
|
-
status: status,
|
|
156
|
-
signature_class: signature_class,
|
|
157
|
-
module_name: module_name,
|
|
158
|
-
model: model,
|
|
159
|
-
provider: provider,
|
|
160
|
-
input_fields: input_fields
|
|
161
|
-
}
|
|
162
|
-
hash[:input_size] = input_size if input_size
|
|
163
|
-
hash[:output_size] = output_size if output_size
|
|
164
|
-
hash[:error_type] = error_type if error_type
|
|
165
|
-
hash[:error_message] = error_message if error_message
|
|
166
|
-
hash
|
|
167
|
-
end
|
|
168
|
-
end
|
|
169
|
-
|
|
170
|
-
# Chain of Thought event payload
|
|
171
|
-
class ChainOfThoughtEvent < T::Struct
|
|
172
|
-
extend T::Sig
|
|
173
|
-
|
|
174
|
-
# Common fields
|
|
175
|
-
const :timestamp, String
|
|
176
|
-
const :duration_ms, Float
|
|
177
|
-
const :cpu_time_ms, Float
|
|
178
|
-
const :status, String
|
|
179
|
-
|
|
180
|
-
# CoT-specific fields
|
|
181
|
-
const :signature_class, String
|
|
182
|
-
const :module_name, String
|
|
183
|
-
const :model, String
|
|
184
|
-
const :provider, String
|
|
185
|
-
const :reasoning_length, T.nilable(Integer), default: nil
|
|
186
|
-
const :answer_length, T.nilable(Integer), default: nil
|
|
187
|
-
|
|
188
|
-
# Error fields (optional)
|
|
189
|
-
const :error_type, T.nilable(String), default: nil
|
|
190
|
-
const :error_message, T.nilable(String), default: nil
|
|
191
|
-
|
|
192
|
-
sig { returns(T::Hash[Symbol, T.untyped]) }
|
|
193
|
-
def to_h
|
|
194
|
-
hash = {
|
|
195
|
-
timestamp: timestamp,
|
|
196
|
-
duration_ms: duration_ms,
|
|
197
|
-
cpu_time_ms: cpu_time_ms,
|
|
198
|
-
status: status,
|
|
199
|
-
signature_class: signature_class,
|
|
200
|
-
module_name: module_name,
|
|
201
|
-
model: model,
|
|
202
|
-
provider: provider
|
|
203
|
-
}
|
|
204
|
-
hash[:reasoning_length] = reasoning_length if reasoning_length
|
|
205
|
-
hash[:answer_length] = answer_length if answer_length
|
|
206
|
-
hash[:error_type] = error_type if error_type
|
|
207
|
-
hash[:error_message] = error_message if error_message
|
|
208
|
-
hash
|
|
209
|
-
end
|
|
210
|
-
end
|
|
211
|
-
|
|
212
|
-
# ReAct iteration event payload
|
|
213
|
-
class ReactIterationEvent < T::Struct
|
|
214
|
-
extend T::Sig
|
|
215
|
-
|
|
216
|
-
# Common fields
|
|
217
|
-
const :timestamp, String
|
|
218
|
-
const :duration_ms, Float
|
|
219
|
-
const :cpu_time_ms, Float
|
|
220
|
-
const :status, String
|
|
221
|
-
|
|
222
|
-
# ReAct-specific fields
|
|
223
|
-
const :iteration, Integer
|
|
224
|
-
const :max_iterations, Integer
|
|
225
|
-
const :history_length, Integer
|
|
226
|
-
const :tools_used_so_far, T::Array[String]
|
|
227
|
-
|
|
228
|
-
# Error fields (optional)
|
|
229
|
-
const :error_type, T.nilable(String), default: nil
|
|
230
|
-
const :error_message, T.nilable(String), default: nil
|
|
231
|
-
|
|
232
|
-
sig { returns(T::Hash[Symbol, T.untyped]) }
|
|
233
|
-
def to_h
|
|
234
|
-
hash = {
|
|
235
|
-
timestamp: timestamp,
|
|
236
|
-
duration_ms: duration_ms,
|
|
237
|
-
cpu_time_ms: cpu_time_ms,
|
|
238
|
-
status: status,
|
|
239
|
-
iteration: iteration,
|
|
240
|
-
max_iterations: max_iterations,
|
|
241
|
-
history_length: history_length,
|
|
242
|
-
tools_used_so_far: tools_used_so_far
|
|
243
|
-
}
|
|
244
|
-
hash[:error_type] = error_type if error_type
|
|
245
|
-
hash[:error_message] = error_message if error_message
|
|
246
|
-
hash
|
|
247
|
-
end
|
|
248
|
-
end
|
|
249
|
-
|
|
250
|
-
# ReAct tool call event payload
|
|
251
|
-
class ReactToolCallEvent < T::Struct
|
|
252
|
-
extend T::Sig
|
|
253
|
-
|
|
254
|
-
# Common fields
|
|
255
|
-
const :timestamp, String
|
|
256
|
-
const :duration_ms, Float
|
|
257
|
-
const :cpu_time_ms, Float
|
|
258
|
-
const :status, String
|
|
259
|
-
|
|
260
|
-
# Tool call fields
|
|
261
|
-
const :iteration, Integer
|
|
262
|
-
const :tool_name, String
|
|
263
|
-
const :tool_input, T.untyped
|
|
264
|
-
|
|
265
|
-
# Error fields (optional)
|
|
266
|
-
const :error_type, T.nilable(String), default: nil
|
|
267
|
-
const :error_message, T.nilable(String), default: nil
|
|
268
|
-
|
|
269
|
-
sig { returns(T::Hash[Symbol, T.untyped]) }
|
|
270
|
-
def to_h
|
|
271
|
-
hash = {
|
|
272
|
-
timestamp: timestamp,
|
|
273
|
-
duration_ms: duration_ms,
|
|
274
|
-
cpu_time_ms: cpu_time_ms,
|
|
275
|
-
status: status,
|
|
276
|
-
iteration: iteration,
|
|
277
|
-
tool_name: tool_name,
|
|
278
|
-
tool_input: tool_input
|
|
279
|
-
}
|
|
280
|
-
hash[:error_type] = error_type if error_type
|
|
281
|
-
hash[:error_message] = error_message if error_message
|
|
282
|
-
hash
|
|
283
|
-
end
|
|
284
|
-
end
|
|
285
|
-
|
|
286
|
-
# ReAct iteration complete event (emit, not instrument)
|
|
287
|
-
class ReactIterationCompleteEvent < T::Struct
|
|
288
|
-
extend T::Sig
|
|
289
|
-
|
|
290
|
-
# Common fields
|
|
291
|
-
const :timestamp, String
|
|
292
|
-
const :status, String
|
|
293
|
-
|
|
294
|
-
# Iteration complete fields
|
|
295
|
-
const :iteration, Integer
|
|
296
|
-
const :thought, String
|
|
297
|
-
const :action, String
|
|
298
|
-
const :action_input, T.untyped
|
|
299
|
-
const :observation, String
|
|
300
|
-
const :tools_used, T::Array[String]
|
|
301
|
-
|
|
302
|
-
sig { returns(T::Hash[Symbol, T.untyped]) }
|
|
303
|
-
def to_h
|
|
304
|
-
{
|
|
305
|
-
timestamp: timestamp,
|
|
306
|
-
status: status,
|
|
307
|
-
iteration: iteration,
|
|
308
|
-
thought: thought,
|
|
309
|
-
action: action,
|
|
310
|
-
action_input: action_input,
|
|
311
|
-
observation: observation,
|
|
312
|
-
tools_used: tools_used
|
|
313
|
-
}
|
|
314
|
-
end
|
|
315
|
-
end
|
|
316
|
-
|
|
317
|
-
# ReAct max iterations event (emit, not instrument)
|
|
318
|
-
class ReactMaxIterationsEvent < T::Struct
|
|
319
|
-
extend T::Sig
|
|
320
|
-
|
|
321
|
-
# Common fields
|
|
322
|
-
const :timestamp, String
|
|
323
|
-
const :status, String
|
|
324
|
-
|
|
325
|
-
# Max iterations fields
|
|
326
|
-
const :iteration_count, Integer
|
|
327
|
-
const :max_iterations, Integer
|
|
328
|
-
const :tools_used, T::Array[String]
|
|
329
|
-
const :final_history_length, Integer
|
|
330
|
-
|
|
331
|
-
sig { returns(T::Hash[Symbol, T.untyped]) }
|
|
332
|
-
def to_h
|
|
333
|
-
{
|
|
334
|
-
timestamp: timestamp,
|
|
335
|
-
status: status,
|
|
336
|
-
iteration_count: iteration_count,
|
|
337
|
-
max_iterations: max_iterations,
|
|
338
|
-
tools_used: tools_used,
|
|
339
|
-
final_history_length: final_history_length
|
|
340
|
-
}
|
|
341
|
-
end
|
|
342
|
-
end
|
|
343
|
-
|
|
344
|
-
# CodeAct iteration event payload
|
|
345
|
-
class CodeActIterationEvent < T::Struct
|
|
346
|
-
extend T::Sig
|
|
347
|
-
|
|
348
|
-
# Common fields
|
|
349
|
-
const :timestamp, String
|
|
350
|
-
const :duration_ms, Float
|
|
351
|
-
const :cpu_time_ms, Float
|
|
352
|
-
const :status, String
|
|
353
|
-
|
|
354
|
-
# CodeAct-specific fields
|
|
355
|
-
const :iteration, Integer
|
|
356
|
-
const :max_iterations, Integer
|
|
357
|
-
const :history_length, Integer
|
|
358
|
-
const :code_blocks_executed, Integer
|
|
359
|
-
|
|
360
|
-
# Error fields (optional)
|
|
361
|
-
const :error_type, T.nilable(String), default: nil
|
|
362
|
-
const :error_message, T.nilable(String), default: nil
|
|
363
|
-
|
|
364
|
-
sig { returns(T::Hash[Symbol, T.untyped]) }
|
|
365
|
-
def to_h
|
|
366
|
-
hash = {
|
|
367
|
-
timestamp: timestamp,
|
|
368
|
-
duration_ms: duration_ms,
|
|
369
|
-
cpu_time_ms: cpu_time_ms,
|
|
370
|
-
status: status,
|
|
371
|
-
iteration: iteration,
|
|
372
|
-
max_iterations: max_iterations,
|
|
373
|
-
history_length: history_length,
|
|
374
|
-
code_blocks_executed: code_blocks_executed
|
|
375
|
-
}
|
|
376
|
-
hash[:error_type] = error_type if error_type
|
|
377
|
-
hash[:error_message] = error_message if error_message
|
|
378
|
-
hash
|
|
379
|
-
end
|
|
380
|
-
end
|
|
381
|
-
|
|
382
|
-
# CodeAct code execution event payload
|
|
383
|
-
class CodeActCodeExecutionEvent < T::Struct
|
|
384
|
-
extend T::Sig
|
|
385
|
-
|
|
386
|
-
# Common fields
|
|
387
|
-
const :timestamp, String
|
|
388
|
-
const :duration_ms, Float
|
|
389
|
-
const :cpu_time_ms, Float
|
|
390
|
-
const :status, String
|
|
391
|
-
|
|
392
|
-
# Code execution fields
|
|
393
|
-
const :iteration, Integer
|
|
394
|
-
const :code_type, String
|
|
395
|
-
const :code_length, Integer
|
|
396
|
-
const :execution_success, T::Boolean
|
|
397
|
-
|
|
398
|
-
# Error fields (optional)
|
|
399
|
-
const :error_type, T.nilable(String), default: nil
|
|
400
|
-
const :error_message, T.nilable(String), default: nil
|
|
401
|
-
|
|
402
|
-
sig { returns(T::Hash[Symbol, T.untyped]) }
|
|
403
|
-
def to_h
|
|
404
|
-
hash = {
|
|
405
|
-
timestamp: timestamp,
|
|
406
|
-
duration_ms: duration_ms,
|
|
407
|
-
cpu_time_ms: cpu_time_ms,
|
|
408
|
-
status: status,
|
|
409
|
-
iteration: iteration,
|
|
410
|
-
code_type: code_type,
|
|
411
|
-
code_length: code_length,
|
|
412
|
-
execution_success: execution_success
|
|
413
|
-
}
|
|
414
|
-
hash[:error_type] = error_type if error_type
|
|
415
|
-
hash[:error_message] = error_message if error_message
|
|
416
|
-
hash
|
|
417
|
-
end
|
|
418
|
-
end
|
|
419
|
-
|
|
420
|
-
# Chain of thought reasoning complete event (emit, not instrument)
|
|
421
|
-
class ChainOfThoughtReasoningCompleteEvent < T::Struct
|
|
422
|
-
extend T::Sig
|
|
423
|
-
|
|
424
|
-
# Common fields
|
|
425
|
-
const :timestamp, String
|
|
426
|
-
const :status, String
|
|
427
|
-
|
|
428
|
-
# Reasoning complete fields
|
|
429
|
-
const :signature_class, String
|
|
430
|
-
const :module_name, String
|
|
431
|
-
const :reasoning_length, Integer
|
|
432
|
-
const :answer_present, T::Boolean
|
|
433
|
-
|
|
434
|
-
sig { returns(T::Hash[Symbol, T.untyped]) }
|
|
435
|
-
def to_h
|
|
436
|
-
{
|
|
437
|
-
timestamp: timestamp,
|
|
438
|
-
status: status,
|
|
439
|
-
signature_class: signature_class,
|
|
440
|
-
module_name: module_name,
|
|
441
|
-
reasoning_length: reasoning_length,
|
|
442
|
-
answer_present: answer_present
|
|
443
|
-
}
|
|
444
|
-
end
|
|
445
|
-
end
|
|
446
|
-
|
|
447
|
-
# Validation error event (emit, not instrument)
|
|
448
|
-
class PredictValidationErrorEvent < T::Struct
|
|
449
|
-
extend T::Sig
|
|
450
|
-
|
|
451
|
-
# Common fields
|
|
452
|
-
const :timestamp, String
|
|
453
|
-
const :status, String
|
|
454
|
-
|
|
455
|
-
# Validation error fields
|
|
456
|
-
const :signature_class, String
|
|
457
|
-
const :module_name, String
|
|
458
|
-
const :field_name, String
|
|
459
|
-
const :error_message, String
|
|
460
|
-
const :retry_count, Integer
|
|
461
|
-
|
|
462
|
-
sig { returns(T::Hash[Symbol, T.untyped]) }
|
|
463
|
-
def to_h
|
|
464
|
-
{
|
|
465
|
-
timestamp: timestamp,
|
|
466
|
-
status: status,
|
|
467
|
-
signature_class: signature_class,
|
|
468
|
-
module_name: module_name,
|
|
469
|
-
field_name: field_name,
|
|
470
|
-
error_message: error_message,
|
|
471
|
-
retry_count: retry_count
|
|
472
|
-
}
|
|
473
|
-
end
|
|
474
|
-
end
|
|
475
|
-
end
|
|
476
|
-
end
|
|
@@ -1,70 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
require_relative '../lm/usage'
|
|
4
|
-
|
|
5
|
-
module DSPy
|
|
6
|
-
module Instrumentation
|
|
7
|
-
# Utility for extracting token usage from different LM adapters
|
|
8
|
-
# Uses actual token counts from API responses for accuracy
|
|
9
|
-
module TokenTracker
|
|
10
|
-
extend self
|
|
11
|
-
|
|
12
|
-
# Extract actual token usage from API responses
|
|
13
|
-
def extract_token_usage(response, provider)
|
|
14
|
-
return {} unless response&.usage
|
|
15
|
-
|
|
16
|
-
# Handle Usage struct
|
|
17
|
-
if response.usage.is_a?(DSPy::LM::Usage) || response.usage.is_a?(DSPy::LM::OpenAIUsage)
|
|
18
|
-
return {
|
|
19
|
-
input_tokens: response.usage.input_tokens,
|
|
20
|
-
output_tokens: response.usage.output_tokens,
|
|
21
|
-
total_tokens: response.usage.total_tokens
|
|
22
|
-
}
|
|
23
|
-
end
|
|
24
|
-
|
|
25
|
-
# Fallback to legacy hash handling
|
|
26
|
-
case provider.to_s.downcase
|
|
27
|
-
when 'openai'
|
|
28
|
-
extract_openai_tokens(response)
|
|
29
|
-
when 'anthropic'
|
|
30
|
-
extract_anthropic_tokens(response)
|
|
31
|
-
else
|
|
32
|
-
{} # No token information for other providers
|
|
33
|
-
end
|
|
34
|
-
end
|
|
35
|
-
|
|
36
|
-
private
|
|
37
|
-
|
|
38
|
-
def extract_openai_tokens(response)
|
|
39
|
-
return {} unless response&.usage
|
|
40
|
-
|
|
41
|
-
usage = response.usage
|
|
42
|
-
return {} unless usage.is_a?(Hash)
|
|
43
|
-
|
|
44
|
-
# Handle both symbol and string keys for VCR compatibility
|
|
45
|
-
{
|
|
46
|
-
input_tokens: usage[:prompt_tokens] || usage['prompt_tokens'],
|
|
47
|
-
output_tokens: usage[:completion_tokens] || usage['completion_tokens'],
|
|
48
|
-
total_tokens: usage[:total_tokens] || usage['total_tokens']
|
|
49
|
-
}.compact # Remove nil values
|
|
50
|
-
end
|
|
51
|
-
|
|
52
|
-
def extract_anthropic_tokens(response)
|
|
53
|
-
return {} unless response&.usage
|
|
54
|
-
|
|
55
|
-
usage = response.usage
|
|
56
|
-
return {} unless usage.is_a?(Hash)
|
|
57
|
-
|
|
58
|
-
# Handle both symbol and string keys for VCR compatibility
|
|
59
|
-
input_tokens = usage[:input_tokens] || usage['input_tokens'] || 0
|
|
60
|
-
output_tokens = usage[:output_tokens] || usage['output_tokens'] || 0
|
|
61
|
-
|
|
62
|
-
{
|
|
63
|
-
input_tokens: input_tokens,
|
|
64
|
-
output_tokens: output_tokens,
|
|
65
|
-
total_tokens: input_tokens + output_tokens
|
|
66
|
-
}.compact # Remove nil values
|
|
67
|
-
end
|
|
68
|
-
end
|
|
69
|
-
end
|
|
70
|
-
end
|