dspy 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +357 -248
- data/lib/dspy/chain_of_thought.rb +151 -11
- data/lib/dspy/instrumentation/token_tracker.rb +54 -0
- data/lib/dspy/instrumentation.rb +100 -0
- data/lib/dspy/lm/adapter.rb +41 -0
- data/lib/dspy/lm/adapter_factory.rb +59 -0
- data/lib/dspy/lm/adapters/anthropic_adapter.rb +96 -0
- data/lib/dspy/lm/adapters/openai_adapter.rb +53 -0
- data/lib/dspy/lm/adapters/ruby_llm_adapter.rb +81 -0
- data/lib/dspy/lm/errors.rb +10 -0
- data/lib/dspy/lm/response.rb +28 -0
- data/lib/dspy/lm.rb +92 -40
- data/lib/dspy/module.rb +51 -6
- data/lib/dspy/predict.rb +135 -15
- data/lib/dspy/re_act.rb +366 -191
- data/lib/dspy/schema_adapters.rb +55 -0
- data/lib/dspy/signature.rb +282 -10
- data/lib/dspy/subscribers/logger_subscriber.rb +197 -0
- data/lib/dspy/tools/{sorbet_tool.rb → base.rb} +33 -33
- data/lib/dspy/tools.rb +1 -1
- data/lib/dspy.rb +19 -10
- metadata +60 -28
- data/lib/dspy/ext/dry_schema.rb +0 -94
- data/lib/dspy/sorbet_chain_of_thought.rb +0 -91
- data/lib/dspy/sorbet_module.rb +0 -47
- data/lib/dspy/sorbet_predict.rb +0 -180
- data/lib/dspy/sorbet_re_act.rb +0 -332
- data/lib/dspy/sorbet_signature.rb +0 -218
- data/lib/dspy/types.rb +0 -3
data/README.md
CHANGED
@@ -1,77 +1,90 @@
|
|
1
1
|
# DSPy.rb
|
2
2
|
|
3
|
-
|
3
|
+
**Build reliable LLM applications in Ruby using composable, type-safe modules.**
|
4
4
|
|
5
|
-
|
5
|
+
DSPy.rb brings structured LLM programming to Ruby developers.
|
6
|
+
Instead of wrestling with prompt strings and parsing responses,
|
7
|
+
you define typed signatures and compose them into pipelines that just work.
|
6
8
|
|
7
|
-
|
9
|
+
Traditional prompting is like writing code with string concatenation: it works until
|
10
|
+
it doesn't. DSPy.rb brings you the programming approach pioneered
|
11
|
+
by [dspy.ai](https://dspy.ai/): instead of crafting fragile prompts, you define
|
12
|
+
modular signatures and let the framework handle the messy details.
|
8
13
|
|
9
|
-
|
10
|
-
- **Predict**: Basic LLM completion with structured inputs and outputs
|
11
|
-
- **Chain of Thought**: Enhanced reasoning through step-by-step thinking
|
12
|
-
- **ReAct**: Compose multiple LLM calls in a structured workflow using tools.
|
13
|
-
- **RAG (Retrieval-Augmented Generation)**: Enriched responses with context from retrieval
|
14
|
-
- **Multi-stage Pipelines**: Compose multiple LLM calls in a structured workflow
|
14
|
+
The result? LLM applications that actually scale and don't break when you sneeze.
|
15
15
|
|
16
|
-
|
17
|
-
- OpenAI and Anthropic via [Ruby LLM](https://github.com/crmne/ruby_llm)
|
18
|
-
- JSON schema validation with [dry-schema](https://dry-rb.org/gems/dry-schema/)
|
16
|
+
## What You Get
|
19
17
|
|
20
|
-
|
18
|
+
**Core Building Blocks:**
|
19
|
+
- **Signatures** - Define input/output schemas using Sorbet types
|
20
|
+
- **Predict** - Basic LLM completion with structured data
|
21
|
+
- **Chain of Thought** - Step-by-step reasoning for complex problems
|
22
|
+
- **ReAct** - Tool-using agents that can actually get things done
|
23
|
+
- **RAG** - Context-enriched responses from your data
|
24
|
+
- **Multi-stage Pipelines** - Compose multiple LLM calls into workflows
|
25
|
+
- OpenAI and Anthropic support via [Ruby LLM](https://github.com/crmne/ruby_llm)
|
26
|
+
- Runtime type checking with [Sorbet](https://sorbet.org/)
|
27
|
+
- Type-safe tool definitions for ReAct agents
|
28
|
+
|
29
|
+
## Fair Warning
|
30
|
+
|
31
|
+
This is fresh off the oven and evolving fast.
|
32
|
+
I'm actively building this as a Ruby port of the [DSPy library](https://dspy.ai/).
|
33
|
+
If you hit bugs or want to contribute, just email me directly!
|
21
34
|
|
22
|
-
|
23
|
-
|
35
|
+
## What's Next
|
36
|
+
These are my goals to release v1.0.
|
24
37
|
|
38
|
+
- Solidify prompt optimization
|
39
|
+
- OTel Integration
|
40
|
+
- Ollama support
|
41
|
+
|
42
|
+
## Installation
|
43
|
+
|
44
|
+
Skip the gem for now - install straight from this repo while I prep the first release:
|
25
45
|
```ruby
|
26
46
|
gem 'dspy', github: 'vicentereig/dspy.rb'
|
27
47
|
```
|
28
48
|
|
29
49
|
## Usage Examples
|
30
50
|
|
31
|
-
###
|
51
|
+
### Simple Prediction
|
32
52
|
|
33
53
|
```ruby
|
34
54
|
# Define a signature for sentiment classification
|
35
55
|
class Classify < DSPy::Signature
|
36
56
|
description "Classify sentiment of a given sentence."
|
37
57
|
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
.meta(description: 'The sentiment classification')
|
45
|
-
required(:confidence).value(:float).meta(description: 'Confidence score')
|
58
|
+
class Sentiment < T::Enum
|
59
|
+
enums do
|
60
|
+
Positive = new('positive')
|
61
|
+
Negative = new('negative')
|
62
|
+
Neutral = new('neutral')
|
63
|
+
end
|
46
64
|
end
|
47
|
-
end
|
48
|
-
|
49
|
-
# Initialize the language model
|
50
|
-
class SentimentClassifierWithDescriptions < DSPy::Signature
|
51
|
-
description "Classify sentiment of a given sentence."
|
52
65
|
|
53
66
|
input do
|
54
|
-
|
55
|
-
.value(:string)
|
56
|
-
.meta(description: 'The sentence whose sentiment you are analyzing')
|
67
|
+
const :sentence, String
|
57
68
|
end
|
58
69
|
|
59
70
|
output do
|
60
|
-
|
61
|
-
|
62
|
-
.meta(description: 'The allowed values to classify sentences')
|
63
|
-
|
64
|
-
required(:confidence).value(:float)
|
65
|
-
.meta(description:'The confidence score for the classification')
|
71
|
+
const :sentiment, Sentiment
|
72
|
+
const :confidence, Float
|
66
73
|
end
|
67
74
|
end
|
75
|
+
|
76
|
+
# Configure DSPy with your LLM
|
68
77
|
DSPy.configure do |c|
|
69
78
|
c.lm = DSPy::LM.new('openai/gpt-4o-mini', api_key: ENV['OPENAI_API_KEY'])
|
70
79
|
end
|
80
|
+
|
71
81
|
# Create the predictor and run inference
|
72
82
|
classify = DSPy::Predict.new(Classify)
|
73
83
|
result = classify.call(sentence: "This book was super fun to read, though not the last chapter.")
|
74
|
-
|
84
|
+
|
85
|
+
# result is a properly typed T::Struct instance
|
86
|
+
puts result.sentiment # => #<Sentiment::Positive>
|
87
|
+
puts result.confidence # => 0.85
|
75
88
|
```
|
76
89
|
|
77
90
|
### Chain of Thought Reasoning
|
@@ -81,301 +94,397 @@ class AnswerPredictor < DSPy::Signature
|
|
81
94
|
description "Provides a concise answer to the question"
|
82
95
|
|
83
96
|
input do
|
84
|
-
|
97
|
+
const :question, String
|
85
98
|
end
|
86
99
|
|
87
100
|
output do
|
88
|
-
|
101
|
+
const :answer, String
|
89
102
|
end
|
90
103
|
end
|
91
104
|
|
92
|
-
|
93
|
-
c.lm = DSPy::LM.new('openai/gpt-4o-mini', api_key: ENV['OPENAI_API_KEY'])
|
94
|
-
end
|
95
|
-
|
105
|
+
# Chain of thought automatically adds a 'reasoning' field to the output
|
96
106
|
qa_cot = DSPy::ChainOfThought.new(AnswerPredictor)
|
97
|
-
|
98
|
-
|
99
|
-
#
|
107
|
+
result = qa_cot.call(question: "Two dice are tossed. What is the probability that the sum equals two?")
|
108
|
+
|
109
|
+
puts result.reasoning # => "There is only one way to get a sum of 2..."
|
110
|
+
puts result.answer # => "1/36"
|
100
111
|
```
|
101
112
|
|
102
|
-
###
|
113
|
+
### ReAct Agents with Tools
|
103
114
|
|
104
115
|
```ruby
|
105
|
-
|
106
|
-
|
107
|
-
|
116
|
+
|
117
|
+
class DeepQA < DSPy::Signature
|
118
|
+
description "Answer questions with consideration for the context"
|
119
|
+
|
108
120
|
input do
|
109
|
-
|
110
|
-
required(:question).filled(:string)
|
121
|
+
const :question, String
|
111
122
|
end
|
112
123
|
|
113
124
|
output do
|
114
|
-
|
125
|
+
const :answer, String
|
115
126
|
end
|
116
127
|
end
|
117
128
|
|
118
|
-
|
119
|
-
|
120
|
-
|
129
|
+
# Define tools for the agent
|
130
|
+
class CalculatorTool < DSPy::Tools::Base
|
131
|
+
|
132
|
+
tool_name 'calculator'
|
133
|
+
tool_description 'Performs basic arithmetic operations'
|
134
|
+
|
135
|
+
sig { params(operation: String, num1: Float, num2: Float).returns(T.any(Float, String)) }
|
136
|
+
def call(operation:, num1:, num2:)
|
137
|
+
case operation.downcase
|
138
|
+
when 'add' then num1 + num2
|
139
|
+
when 'subtract' then num1 - num2
|
140
|
+
when 'multiply' then num1 * num2
|
141
|
+
when 'divide'
|
142
|
+
return "Error: Cannot divide by zero" if num2 == 0
|
143
|
+
num1 / num2
|
144
|
+
else
|
145
|
+
"Error: Unknown operation '#{operation}'. Use add, subtract, multiply, or divide"
|
146
|
+
end
|
147
|
+
end
|
121
148
|
|
122
|
-
#
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
149
|
+
# Create ReAct agent with tools
|
150
|
+
agent = DSPy::ReAct.new(DeepQA, tools: [CalculatorTool.new])
|
151
|
+
|
152
|
+
# Run the agent
|
153
|
+
result = agent.forward(question: "What is 42 plus 58?")
|
154
|
+
puts result.answer # => "100"
|
155
|
+
puts result.history # => Array of reasoning steps and tool calls
|
127
156
|
```
|
128
157
|
|
129
|
-
### Multi-stage
|
158
|
+
### Multi-stage Pipelines
|
159
|
+
Outline the sections of an article and draft them out.
|
130
160
|
|
131
161
|
```ruby
|
132
|
-
|
162
|
+
|
163
|
+
# write an article!
|
164
|
+
drafter = ArticleDrafter.new
|
165
|
+
article = drafter.forward(topic: "The impact of AI on software development") # { title: '....', sections: [{content: '....'}]}
|
166
|
+
|
167
|
+
class Outline < DSPy::Signature
|
168
|
+
description "Outline a thorough overview of a topic."
|
169
|
+
|
170
|
+
input do
|
171
|
+
const :topic, String
|
172
|
+
end
|
173
|
+
|
174
|
+
output do
|
175
|
+
const :title, String
|
176
|
+
const :sections, T::Array[String]
|
177
|
+
end
|
178
|
+
end
|
179
|
+
|
180
|
+
class DraftSection < DSPy::Signature
|
181
|
+
description "Draft a section of an article"
|
182
|
+
|
183
|
+
input do
|
184
|
+
const :topic, String
|
185
|
+
const :title, String
|
186
|
+
const :section, String
|
187
|
+
end
|
188
|
+
|
189
|
+
output do
|
190
|
+
const :content, String
|
191
|
+
end
|
192
|
+
end
|
193
|
+
|
133
194
|
class ArticleDrafter < DSPy::Module
|
134
195
|
def initialize
|
135
196
|
@build_outline = DSPy::ChainOfThought.new(Outline)
|
136
197
|
@draft_section = DSPy::ChainOfThought.new(DraftSection)
|
137
198
|
end
|
138
199
|
|
139
|
-
def forward(topic)
|
140
|
-
# First build the outline
|
200
|
+
def forward(topic:)
|
141
201
|
outline = @build_outline.call(topic: topic)
|
142
202
|
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
section_heading: "## #{heading}",
|
149
|
-
section_subheadings: [subheadings].flatten.map { |sh| "### #{sh}" }
|
203
|
+
sections = outline.sections.map do |section|
|
204
|
+
@draft_section.call(
|
205
|
+
topic: topic,
|
206
|
+
title: outline.title,
|
207
|
+
section: section
|
150
208
|
)
|
151
|
-
sections << section
|
152
209
|
end
|
153
210
|
|
154
|
-
|
211
|
+
{
|
212
|
+
title: outline.title,
|
213
|
+
sections: sections.map(&:content)
|
214
|
+
}
|
155
215
|
end
|
156
216
|
end
|
157
217
|
|
158
|
-
DSPy.configure do |c|
|
159
|
-
c.lm = DSPy::LM.new('openai/gpt-4o-mini', api_key: ENV['OPENAI_API_KEY'])
|
160
|
-
end
|
161
|
-
# Usage
|
162
|
-
drafter = ArticleDrafter.new
|
163
|
-
article = drafter.call("World Cup 2002")
|
164
218
|
```
|
165
219
|
|
166
|
-
|
220
|
+
## Working with Complex Types
|
167
221
|
|
168
|
-
|
222
|
+
### Enums
|
169
223
|
|
170
|
-
|
224
|
+
```ruby
|
225
|
+
class Color < T::Enum
|
226
|
+
enums do
|
227
|
+
Red = new
|
228
|
+
Green = new
|
229
|
+
Blue = new
|
230
|
+
end
|
231
|
+
end
|
171
232
|
|
172
|
-
|
173
|
-
|
174
|
-
* **LLM**: The ReAct agent internally uses an LLM (configured via `DSPy.configure`) to generate thoughts and decide on actions.
|
233
|
+
class ColorSignature < DSPy::Signature
|
234
|
+
description "Identify the dominant color in a description"
|
175
235
|
|
176
|
-
|
236
|
+
input do
|
237
|
+
const :description, String,
|
238
|
+
description: 'Description of an object or scene'
|
239
|
+
end
|
240
|
+
|
241
|
+
output do
|
242
|
+
const :color, Color,
|
243
|
+
description: 'The dominant color (Red, Green, or Blue)'
|
244
|
+
end
|
245
|
+
end
|
246
|
+
|
247
|
+
predictor = DSPy::Predict.new(ColorSignature)
|
248
|
+
result = predictor.call(description: "A red apple on a wooden table")
|
249
|
+
puts result.color # => #<Color::Red>
|
250
|
+
```
|
177
251
|
|
178
|
-
|
252
|
+
### Optional Fields and Defaults
|
179
253
|
|
180
254
|
```ruby
|
181
|
-
|
182
|
-
|
183
|
-
description "Answers mathematical questions."
|
255
|
+
class AnalysisSignature < DSPy::Signature
|
256
|
+
description "Analyze text with optional metadata"
|
184
257
|
|
185
258
|
input do
|
186
|
-
|
259
|
+
const :text, String,
|
260
|
+
description: 'Text to analyze'
|
261
|
+
const :include_metadata, T::Boolean,
|
262
|
+
description: 'Whether to include metadata in analysis',
|
263
|
+
default: false
|
187
264
|
end
|
188
265
|
|
189
266
|
output do
|
190
|
-
|
267
|
+
const :summary, String,
|
268
|
+
description: 'Summary of the text'
|
269
|
+
const :word_count, Integer,
|
270
|
+
description: 'Number of words (optional)',
|
271
|
+
default: 0
|
191
272
|
end
|
192
273
|
end
|
274
|
+
```
|
193
275
|
|
194
|
-
|
195
|
-
class CalculatorTool < DSPy::Tools::Tool
|
196
|
-
def initialize
|
197
|
-
super('calculator', 'Calculates the result of a simple arithmetic expression (e.g., "5 + 7"). Input must be a string representing the expression.')
|
198
|
-
end
|
276
|
+
## Advanced Usage Patterns
|
199
277
|
|
200
|
-
|
201
|
-
|
202
|
-
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
278
|
+
### Multi-stage Pipelines
|
279
|
+
|
280
|
+
```ruby
|
281
|
+
class TopicSignature < DSPy::Signature
|
282
|
+
description "Extract main topic from text"
|
283
|
+
|
284
|
+
input do
|
285
|
+
const :content, String,
|
286
|
+
description: 'Text content to analyze'
|
287
|
+
end
|
288
|
+
|
289
|
+
output do
|
290
|
+
const :topic, String,
|
291
|
+
description: 'Main topic of the content'
|
212
292
|
end
|
213
293
|
end
|
214
294
|
|
215
|
-
|
216
|
-
|
217
|
-
|
295
|
+
class SummarySignature < DSPy::Signature
|
296
|
+
description "Create summary focusing on specific topic"
|
297
|
+
|
298
|
+
input do
|
299
|
+
const :content, String,
|
300
|
+
description: 'Original text content'
|
301
|
+
const :topic, String,
|
302
|
+
description: 'Topic to focus on'
|
303
|
+
end
|
304
|
+
|
305
|
+
output do
|
306
|
+
const :summary, String,
|
307
|
+
description: 'Topic-focused summary'
|
308
|
+
end
|
218
309
|
end
|
219
310
|
|
220
|
-
|
221
|
-
|
222
|
-
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
|
227
|
-
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
|
233
|
-
|
234
|
-
|
235
|
-
|
236
|
-
|
237
|
-
|
311
|
+
class ArticlePipeline < DSPy::Signature
|
312
|
+
extend T::Sig
|
313
|
+
|
314
|
+
def initialize
|
315
|
+
@topic_extractor = DSPy::Predict.new(TopicSignature)
|
316
|
+
@summarizer = DSPy::ChainOfThought.new(SummarySignature)
|
317
|
+
end
|
318
|
+
|
319
|
+
sig { params(content: String).returns(T.untyped) }
|
320
|
+
def forward(content:)
|
321
|
+
# Extract topic
|
322
|
+
topic_result = @topic_extractor.call(content: content)
|
323
|
+
|
324
|
+
# Create focused summary
|
325
|
+
summary_result = @summarizer.call(
|
326
|
+
content: content,
|
327
|
+
topic: topic_result.topic
|
328
|
+
)
|
329
|
+
|
330
|
+
{
|
331
|
+
topic: topic_result.topic,
|
332
|
+
summary: summary_result.summary,
|
333
|
+
reasoning: summary_result.reasoning
|
334
|
+
}
|
335
|
+
end
|
238
336
|
end
|
239
|
-
# Expected output (will vary based on LLM's reasoning):
|
240
|
-
# Question: What is 5 plus 7?
|
241
|
-
# Answer: 12
|
242
|
-
# Iterations: 2
|
243
|
-
# History:
|
244
|
-
# Step 1:
|
245
|
-
# Thought: I need to calculate 5 plus 7. I have a calculator tool that can do this.
|
246
|
-
# Action: calculator
|
247
|
-
# Action Input: 5 + 7
|
248
|
-
# Observation: 12
|
249
|
-
# Step 2:
|
250
|
-
# Thought: The calculator returned 12, which is the answer to "5 plus 7?". I can now finish.
|
251
|
-
# Action: finish
|
252
|
-
# Action Input: 12
|
253
|
-
```
|
254
|
-
|
255
|
-
**Example 2: Web Search with Serper.dev**
|
256
337
|
|
257
|
-
|
338
|
+
# Usage
|
339
|
+
pipeline = ArticlePipeline.new
|
340
|
+
result = pipeline.call(content: "Long article content...")
|
341
|
+
```
|
258
342
|
|
259
|
-
|
343
|
+
### Retrieval Augmented Generation
|
260
344
|
|
261
345
|
```ruby
|
262
|
-
|
263
|
-
|
264
|
-
|
265
|
-
|
266
|
-
# Define a signature for web-based QA
|
267
|
-
class WebQuestionAnswer < DSPy::Signature
|
268
|
-
description "Answers questions that may require web searches."
|
269
|
-
|
346
|
+
class ContextualQA < DSPy::Signature
|
347
|
+
description "Answer questions using relevant context"
|
348
|
+
|
270
349
|
input do
|
271
|
-
|
350
|
+
const :question, String,
|
351
|
+
description: 'The question to answer'
|
352
|
+
const :context, T::Array[String],
|
353
|
+
description: 'Relevant context passages'
|
272
354
|
end
|
273
355
|
|
274
356
|
output do
|
275
|
-
|
357
|
+
const :answer, String,
|
358
|
+
description: 'Answer based on the provided context'
|
359
|
+
const :confidence, Float,
|
360
|
+
description: 'Confidence in the answer (0.0 to 1.0)'
|
276
361
|
end
|
277
362
|
end
|
278
363
|
|
279
|
-
#
|
280
|
-
|
281
|
-
|
282
|
-
super('web_search', 'Searches the web for a given query and returns the first organic result snippet. Useful for finding current information or answers to general knowledge questions.')
|
283
|
-
end
|
364
|
+
# Usage with retriever
|
365
|
+
retriever = YourRetrieverClass.new
|
366
|
+
qa = DSPy::ChainOfThought.new(ContextualQA)
|
284
367
|
|
285
|
-
|
286
|
-
|
287
|
-
unless api_key
|
288
|
-
return "Error: SERPER_API_KEY environment variable not set."
|
289
|
-
end
|
368
|
+
question = "What is the capital of France?"
|
369
|
+
context = retriever.retrieve(question) # Returns array of strings
|
290
370
|
|
291
|
-
|
292
|
-
|
293
|
-
|
294
|
-
|
295
|
-
|
296
|
-
|
297
|
-
begin
|
298
|
-
response = Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == 'https') do |http|
|
299
|
-
http.request(request)
|
300
|
-
end
|
301
|
-
|
302
|
-
if response.is_a?(Net::HTTPSuccess)
|
303
|
-
results = JSON.parse(response.body)
|
304
|
-
first_organic_result = results['organic']&.first
|
305
|
-
if first_organic_result && first_organic_result['snippet']
|
306
|
-
return "Source: #{first_organic_result['link']}\nSnippet: #{first_organic_result['snippet']}"
|
307
|
-
elsif first_organic_result && first_organic_result['title']
|
308
|
-
return "Source: #{first_organic_result['link']}\nTitle: #{first_organic_result['title']}"
|
309
|
-
else
|
310
|
-
return "No relevant snippet found in the first result."
|
311
|
-
end
|
312
|
-
else
|
313
|
-
return "Error: Serper API request failed with status #{response.code} - #{response.body}"
|
314
|
-
end
|
315
|
-
rescue StandardError => e
|
316
|
-
return "Error performing web search: #{e.message}"
|
317
|
-
end
|
318
|
-
end
|
319
|
-
end
|
371
|
+
result = qa.call(question: question, context: context)
|
372
|
+
puts result.reasoning # Step-by-step reasoning
|
373
|
+
puts result.answer # "Paris"
|
374
|
+
puts result.confidence # 0.95
|
375
|
+
```
|
320
376
|
|
321
|
-
|
322
|
-
DSPy.configure do |c|
|
323
|
-
c.lm = DSPy::LM.new('openai/gpt-4o-mini', api_key: ENV['OPENAI_API_KEY']) # Ensure your LM is configured
|
324
|
-
end
|
377
|
+
## Instrumentation & Observability
|
325
378
|
|
326
|
-
|
327
|
-
|
328
|
-
|
329
|
-
|
330
|
-
|
331
|
-
|
332
|
-
|
333
|
-
|
334
|
-
|
335
|
-
|
336
|
-
|
337
|
-
puts "History (summary):"
|
338
|
-
result.history.each_with_index do |entry, index|
|
339
|
-
puts " Step #{entry[:step]}: Action: #{entry[:action]}, Input: #{entry[:action_input]&.slice(0, 50)}..."
|
340
|
-
# For brevity, not printing full thought/observation here.
|
379
|
+
DSPy.rb includes built-in instrumentation that captures detailed events and
|
380
|
+
performance metrics from your LLM operations. Perfect for monitoring your
|
381
|
+
applications and integrating with observability tools.
|
382
|
+
|
383
|
+
### Quick Setup
|
384
|
+
|
385
|
+
Enable instrumentation to start capturing events:
|
386
|
+
|
387
|
+
```ruby
|
388
|
+
DSPy::Instrumentation.configure do |config|
|
389
|
+
config.enabled = true
|
341
390
|
end
|
342
|
-
# The answer and history will depend on the LLM's reasoning and live search results.
|
343
391
|
```
|
344
392
|
|
345
|
-
|
393
|
+
### Available Events
|
394
|
+
|
395
|
+
Subscribe to these events to monitor different aspects of your LLM operations:
|
396
|
+
|
397
|
+
| Event Name | Triggered When | Key Payload Fields |
|
398
|
+
|------------|----------------|-------------------|
|
399
|
+
| `dspy.lm.request` | LLM API request lifecycle | `gen_ai_system`, `model`, `provider`, `duration_ms`, `status` |
|
400
|
+
| `dspy.lm.tokens` | Token usage tracking | `tokens_input`, `tokens_output`, `tokens_total` |
|
401
|
+
| `dspy.predict` | Prediction operations | `signature_class`, `input_size`, `duration_ms`, `status` |
|
402
|
+
| `dspy.chain_of_thought` | CoT reasoning | `signature_class`, `model`, `duration_ms`, `status` |
|
403
|
+
| `dspy.react` | Agent operations | `max_iterations`, `tools_used`, `duration_ms`, `status` |
|
404
|
+
| `dspy.react.tool_call` | Tool execution | `tool_name`, `tool_input`, `tool_output`, `duration_ms` |
|
405
|
+
|
406
|
+
### Event Payloads
|
407
|
+
|
408
|
+
The instrumentation emits events with structured payloads you can process:
|
409
|
+
|
410
|
+
```ruby
|
411
|
+
# Example event payload for dspy.predict
|
412
|
+
{
|
413
|
+
signature_class: "QuestionAnswering",
|
414
|
+
model: "gpt-4o-mini",
|
415
|
+
provider: "openai",
|
416
|
+
input_size: 45,
|
417
|
+
duration_ms: 1234.56,
|
418
|
+
cpu_time_ms: 89.12,
|
419
|
+
status: "success",
|
420
|
+
timestamp: "2024-01-15T10:30:00Z"
|
421
|
+
}
|
422
|
+
|
423
|
+
# Example token usage payload
|
424
|
+
{
|
425
|
+
tokens_input: 150,
|
426
|
+
tokens_output: 45,
|
427
|
+
tokens_total: 195,
|
428
|
+
gen_ai_system: "openai",
|
429
|
+
signature_class: "QuestionAnswering"
|
430
|
+
}
|
431
|
+
```
|
432
|
+
|
433
|
+
Events are emitted via dry-monitor notifications, giving you flexibility to
|
434
|
+
process them however you need - logging, metrics, alerts, or custom monitoring.
|
346
435
|
|
347
|
-
###
|
348
|
-
- [x] Signatures and Predict module
|
349
|
-
- [x] RAG examples
|
350
|
-
- [x] Multi-Stage Pipelines
|
351
|
-
- [x] Validate inputs and outputs with JSON Schema
|
352
|
-
- [x] thread-safe global config
|
353
|
-
- [x] Convert responses from hashes to Dry Poros (currently tons of footguns with hashes :fire:)
|
354
|
-
- [ ] Cover unhappy paths: validation errors
|
355
|
-
- [x] Implement ReAct module for reasoning and acting
|
356
|
-
- [ ] Add OpenTelemetry instrumentation
|
357
|
-
- [ ] Improve logging
|
358
|
-
- [ ] Add streaming support (?)
|
359
|
-
- [x] Ensure thread safety
|
360
|
-
- [ ] Comprehensive initial documentation, LLM friendly.
|
436
|
+
### Token Tracking
|
361
437
|
|
362
|
-
|
438
|
+
Token usage is extracted from actual API responses (OpenAI and Anthropic only),
|
439
|
+
giving you precise cost tracking:
|
363
440
|
|
364
|
-
|
365
|
-
|
366
|
-
|
441
|
+
```ruby
|
442
|
+
# Token events include:
|
443
|
+
{
|
444
|
+
tokens_input: 150, # From API response
|
445
|
+
tokens_output: 45, # From API response
|
446
|
+
tokens_total: 195, # From API response
|
447
|
+
gen_ai_system: "openai",
|
448
|
+
gen_ai_request_model: "gpt-4o-mini"
|
449
|
+
}
|
450
|
+
```
|
367
451
|
|
368
|
-
###
|
452
|
+
### Configuration Options
|
369
453
|
|
370
|
-
|
371
|
-
|
372
|
-
|
373
|
-
|
454
|
+
```ruby
|
455
|
+
DSPy::Instrumentation.configure do |config|
|
456
|
+
config.enabled = true
|
457
|
+
config.log_to_stdout = false
|
458
|
+
config.log_file = 'log/dspy.log'
|
459
|
+
config.log_level = :info
|
460
|
+
|
461
|
+
# Custom payload enrichment
|
462
|
+
config.custom_options = lambda do |event|
|
463
|
+
{
|
464
|
+
timestamp: Time.current.iso8601,
|
465
|
+
hostname: Socket.gethostname,
|
466
|
+
request_id: Thread.current[:request_id]
|
467
|
+
}
|
468
|
+
end
|
469
|
+
end
|
470
|
+
```
|
471
|
+
|
472
|
+
### Integration with Monitoring Tools
|
473
|
+
|
474
|
+
Subscribe to events for custom processing:
|
374
475
|
|
375
|
-
|
476
|
+
```ruby
|
477
|
+
# Subscribe to all LM events
|
478
|
+
DSPy::Instrumentation.subscribe('dspy.lm.*') do |event|
|
479
|
+
puts "#{event.id}: #{event.payload[:duration_ms]}ms"
|
480
|
+
end
|
376
481
|
|
377
|
-
|
482
|
+
# Subscribe to specific events
|
483
|
+
DSPy::Instrumentation.subscribe('dspy.predict') do |event|
|
484
|
+
MyMetrics.histogram('dspy.predict.duration', event.payload[:duration_ms])
|
485
|
+
end
|
486
|
+
```
|
378
487
|
|
379
488
|
## License
|
380
489
|
|
381
|
-
|
490
|
+
This project is licensed under the MIT License.
|