dspy 0.26.0 → 0.26.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +0 -23
- data/lib/dspy/teleprompt/mipro_v2.rb +193 -222
- data/lib/dspy/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e243b7278275462baea2f493270166a1ae4b5419d4f072a769e4ba4b0f65e3e0
|
4
|
+
data.tar.gz: 13bcbcf4ee67c08f19ad619bc8118e39ca9a02045c5a18b3a664fb112724fa87
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 687385021bf9391b22ae51a3f7c05880bec9691347a4e8ecac9175b7e81190c9f63cb0670a94e7324a045d748346cc91b6f7e174808607eaf0d02b8a0a117992
|
7
|
+
data.tar.gz: 3212712d53aca34cbc475503396d4fbeb7b8c11632b673782e7cd2dc2e6fdb22a3ad67688d92b60bd72e845f7b598b446e94fd3f5c6efd5e87f212bb1be14b9e
|
data/README.md
CHANGED
@@ -157,29 +157,6 @@ Then run:
|
|
157
157
|
bundle install
|
158
158
|
```
|
159
159
|
|
160
|
-
#### System Dependencies for Ubuntu/Pop!_OS
|
161
|
-
|
162
|
-
If you need to compile the `numo-narray` dependency from source (used for numerical computing in Bayesian optimization), install these system packages:
|
163
|
-
|
164
|
-
```bash
|
165
|
-
# Update package list
|
166
|
-
sudo apt-get update
|
167
|
-
|
168
|
-
# Install Ruby development files (if not already installed)
|
169
|
-
sudo apt-get install ruby-full ruby-dev
|
170
|
-
|
171
|
-
# Install essential build tools
|
172
|
-
sudo apt-get install build-essential
|
173
|
-
|
174
|
-
# Install BLAS and LAPACK libraries (required for numo-narray)
|
175
|
-
sudo apt-get install libopenblas-dev liblapack-dev
|
176
|
-
|
177
|
-
# Install additional development libraries
|
178
|
-
sudo apt-get install libffi-dev libssl-dev
|
179
|
-
```
|
180
|
-
|
181
|
-
**Note**: The `numo-narray` gem typically compiles quickly (1-2 minutes). Pre-built binaries are available for most platforms, so compilation is only needed if a pre-built binary isn't available for your system.
|
182
|
-
|
183
160
|
## Recent Achievements
|
184
161
|
|
185
162
|
DSPy.rb has rapidly evolved from experimental to production-ready:
|
@@ -32,6 +32,7 @@ module DSPy
|
|
32
32
|
# instruction generation, and Bayesian optimization
|
33
33
|
class MIPROv2 < Teleprompter
|
34
34
|
extend T::Sig
|
35
|
+
include Dry::Configurable
|
35
36
|
|
36
37
|
# Auto-configuration modes for different optimization needs
|
37
38
|
module AutoMode
|
@@ -44,15 +45,17 @@ module DSPy
|
|
44
45
|
).returns(MIPROv2)
|
45
46
|
end
|
46
47
|
def self.light(metric: nil, **kwargs)
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
48
|
+
optimizer = MIPROv2.new(metric: metric, **kwargs)
|
49
|
+
optimizer.configure do |config|
|
50
|
+
config.num_trials = 6
|
51
|
+
config.num_instruction_candidates = 3
|
52
|
+
config.max_bootstrapped_examples = 2
|
53
|
+
config.max_labeled_examples = 8
|
54
|
+
config.bootstrap_sets = 3
|
55
|
+
config.optimization_strategy = :greedy
|
56
|
+
config.early_stopping_patience = 2
|
57
|
+
end
|
58
|
+
optimizer
|
56
59
|
end
|
57
60
|
|
58
61
|
sig do
|
@@ -62,15 +65,17 @@ module DSPy
|
|
62
65
|
).returns(MIPROv2)
|
63
66
|
end
|
64
67
|
def self.medium(metric: nil, **kwargs)
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
68
|
+
optimizer = MIPROv2.new(metric: metric, **kwargs)
|
69
|
+
optimizer.configure do |config|
|
70
|
+
config.num_trials = 12
|
71
|
+
config.num_instruction_candidates = 5
|
72
|
+
config.max_bootstrapped_examples = 4
|
73
|
+
config.max_labeled_examples = 16
|
74
|
+
config.bootstrap_sets = 5
|
75
|
+
config.optimization_strategy = :adaptive
|
76
|
+
config.early_stopping_patience = 3
|
77
|
+
end
|
78
|
+
optimizer
|
74
79
|
end
|
75
80
|
|
76
81
|
sig do
|
@@ -80,135 +85,102 @@ module DSPy
|
|
80
85
|
).returns(MIPROv2)
|
81
86
|
end
|
82
87
|
def self.heavy(metric: nil, **kwargs)
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
sig { returns(Float) }
|
115
|
-
attr_accessor :final_temperature
|
116
|
-
|
117
|
-
sig { returns(Integer) }
|
118
|
-
attr_accessor :early_stopping_patience
|
119
|
-
|
120
|
-
sig { returns(T::Boolean) }
|
121
|
-
attr_accessor :use_bayesian_optimization
|
122
|
-
|
123
|
-
sig { returns(T::Boolean) }
|
124
|
-
attr_accessor :track_diversity
|
125
|
-
|
126
|
-
sig { returns(DSPy::Propose::GroundedProposer::Config) }
|
127
|
-
attr_accessor :proposer_config
|
128
|
-
|
129
|
-
sig { void }
|
130
|
-
def initialize
|
131
|
-
super
|
132
|
-
@num_trials = 12
|
133
|
-
@num_instruction_candidates = 5
|
134
|
-
@bootstrap_sets = 5
|
135
|
-
@optimization_strategy = "adaptive" # greedy, adaptive, bayesian
|
136
|
-
@init_temperature = 1.0
|
137
|
-
@final_temperature = 0.1
|
138
|
-
@early_stopping_patience = 3
|
139
|
-
@use_bayesian_optimization = true
|
140
|
-
@track_diversity = true
|
141
|
-
@proposer_config = DSPy::Propose::GroundedProposer::Config.new
|
88
|
+
optimizer = MIPROv2.new(metric: metric, **kwargs)
|
89
|
+
optimizer.configure do |config|
|
90
|
+
config.num_trials = 18
|
91
|
+
config.num_instruction_candidates = 8
|
92
|
+
config.max_bootstrapped_examples = 6
|
93
|
+
config.max_labeled_examples = 24
|
94
|
+
config.bootstrap_sets = 8
|
95
|
+
config.optimization_strategy = :bayesian
|
96
|
+
config.early_stopping_patience = 5
|
97
|
+
end
|
98
|
+
optimizer
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
# Dry-configurable settings for MIPROv2
|
103
|
+
setting :num_trials, default: 12
|
104
|
+
setting :num_instruction_candidates, default: 5
|
105
|
+
setting :bootstrap_sets, default: 5
|
106
|
+
setting :max_bootstrapped_examples, default: 4
|
107
|
+
setting :max_labeled_examples, default: 16
|
108
|
+
setting :optimization_strategy, default: OptimizationStrategy::Adaptive, constructor: ->(value) {
|
109
|
+
# Coerce symbols to enum values
|
110
|
+
case value
|
111
|
+
when :greedy then OptimizationStrategy::Greedy
|
112
|
+
when :adaptive then OptimizationStrategy::Adaptive
|
113
|
+
when :bayesian then OptimizationStrategy::Bayesian
|
114
|
+
when OptimizationStrategy then value
|
115
|
+
when nil then OptimizationStrategy::Adaptive
|
116
|
+
else
|
117
|
+
raise ArgumentError, "Invalid optimization strategy: #{value}. Must be one of :greedy, :adaptive, :bayesian"
|
142
118
|
end
|
119
|
+
}
|
120
|
+
setting :init_temperature, default: 1.0
|
121
|
+
setting :final_temperature, default: 0.1
|
122
|
+
setting :early_stopping_patience, default: 3
|
123
|
+
setting :use_bayesian_optimization, default: true
|
124
|
+
setting :track_diversity, default: true
|
125
|
+
setting :max_errors, default: 3
|
126
|
+
setting :num_threads, default: 1
|
143
127
|
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
bootstrap_sets: @bootstrap_sets,
|
150
|
-
optimization_strategy: @optimization_strategy,
|
151
|
-
init_temperature: @init_temperature,
|
152
|
-
final_temperature: @final_temperature,
|
153
|
-
early_stopping_patience: @early_stopping_patience,
|
154
|
-
use_bayesian_optimization: @use_bayesian_optimization,
|
155
|
-
track_diversity: @track_diversity
|
156
|
-
})
|
128
|
+
# Class-level configuration method - sets defaults for new instances
|
129
|
+
def self.configure(&block)
|
130
|
+
if block_given?
|
131
|
+
# Store configuration in a class variable for new instances
|
132
|
+
@default_config_block = block
|
157
133
|
end
|
158
134
|
end
|
159
135
|
|
160
|
-
#
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
# Configuration settings
|
166
|
-
setting :instruction, default: ""
|
167
|
-
setting :few_shot_examples, default: []
|
168
|
-
setting :type, default: CandidateType::Baseline
|
169
|
-
setting :metadata, default: {}
|
136
|
+
# Get the default configuration block
|
137
|
+
def self.default_config_block
|
138
|
+
@default_config_block
|
139
|
+
end
|
170
140
|
|
171
|
-
sig { returns(String) }
|
172
|
-
def config_id
|
173
|
-
@config_id ||= generate_config_id
|
174
|
-
end
|
175
141
|
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
142
|
+
# Simple data structure for evaluated candidate configurations (immutable)
|
143
|
+
EvaluatedCandidate = Data.define(
|
144
|
+
:instruction,
|
145
|
+
:few_shot_examples,
|
146
|
+
:type,
|
147
|
+
:metadata,
|
148
|
+
:config_id
|
149
|
+
) do
|
150
|
+
extend T::Sig
|
151
|
+
|
152
|
+
# Generate a config ID based on content
|
153
|
+
sig { params(instruction: String, few_shot_examples: T::Array[T.untyped], type: CandidateType, metadata: T::Hash[Symbol, T.untyped]).returns(EvaluatedCandidate) }
|
154
|
+
def self.create(instruction:, few_shot_examples: [], type: CandidateType::Baseline, metadata: {})
|
155
|
+
content = "#{instruction}_#{few_shot_examples.size}_#{type.serialize}_#{metadata.hash}"
|
156
|
+
config_id = Digest::SHA256.hexdigest(content)[0, 12]
|
182
157
|
|
183
|
-
|
184
|
-
|
158
|
+
new(
|
159
|
+
instruction: instruction.freeze,
|
160
|
+
few_shot_examples: few_shot_examples.freeze,
|
161
|
+
type: type,
|
162
|
+
metadata: metadata.freeze,
|
163
|
+
config_id: config_id
|
164
|
+
)
|
185
165
|
end
|
186
166
|
|
187
167
|
sig { returns(T::Hash[Symbol, T.untyped]) }
|
188
168
|
def to_h
|
189
169
|
{
|
190
|
-
instruction:
|
191
|
-
few_shot_examples:
|
192
|
-
type:
|
193
|
-
metadata:
|
170
|
+
instruction: instruction,
|
171
|
+
few_shot_examples: few_shot_examples.size,
|
172
|
+
type: type.serialize,
|
173
|
+
metadata: metadata,
|
194
174
|
config_id: config_id
|
195
175
|
}
|
196
176
|
end
|
197
|
-
|
198
|
-
private
|
199
|
-
|
200
|
-
sig { returns(String) }
|
201
|
-
def generate_config_id
|
202
|
-
content = "#{config.instruction}_#{config.few_shot_examples.size}_#{config.type.serialize}_#{config.metadata.hash}"
|
203
|
-
Digest::SHA256.hexdigest(content)[0, 12]
|
204
|
-
end
|
205
177
|
end
|
206
178
|
|
207
179
|
# Result of MIPROv2 optimization
|
208
180
|
class MIPROv2Result < OptimizationResult
|
209
181
|
extend T::Sig
|
210
182
|
|
211
|
-
sig { returns(T::Array[
|
183
|
+
sig { returns(T::Array[EvaluatedCandidate]) }
|
212
184
|
attr_reader :evaluated_candidates
|
213
185
|
|
214
186
|
sig { returns(T::Hash[Symbol, T.untyped]) }
|
@@ -228,7 +200,7 @@ module DSPy
|
|
228
200
|
optimized_program: T.untyped,
|
229
201
|
scores: T::Hash[Symbol, T.untyped],
|
230
202
|
history: T::Hash[Symbol, T.untyped],
|
231
|
-
evaluated_candidates: T::Array[
|
203
|
+
evaluated_candidates: T::Array[EvaluatedCandidate],
|
232
204
|
optimization_trace: T::Hash[Symbol, T.untyped],
|
233
205
|
bootstrap_statistics: T::Hash[Symbol, T.untyped],
|
234
206
|
proposal_statistics: T::Hash[Symbol, T.untyped],
|
@@ -272,18 +244,25 @@ module DSPy
|
|
272
244
|
sig { returns(T.nilable(DSPy::Propose::GroundedProposer)) }
|
273
245
|
attr_reader :proposer
|
274
246
|
|
275
|
-
|
276
|
-
|
277
|
-
|
278
|
-
|
279
|
-
|
280
|
-
|
281
|
-
|
282
|
-
|
283
|
-
|
284
|
-
|
247
|
+
# Override dry-configurable's initialize to add our parameter validation
|
248
|
+
def initialize(metric: nil, **kwargs)
|
249
|
+
# Reject old config parameter pattern
|
250
|
+
if kwargs.key?(:config)
|
251
|
+
raise ArgumentError, "config parameter is no longer supported. Use .configure blocks instead."
|
252
|
+
end
|
253
|
+
|
254
|
+
# Let dry-configurable handle its initialization
|
255
|
+
super(**kwargs)
|
256
|
+
|
257
|
+
# Apply class-level configuration if it exists
|
258
|
+
if self.class.default_config_block
|
259
|
+
configure(&self.class.default_config_block)
|
260
|
+
end
|
285
261
|
|
286
|
-
@
|
262
|
+
@metric = metric
|
263
|
+
|
264
|
+
# Initialize proposer with a basic config for now (will be updated later)
|
265
|
+
@proposer = DSPy::Propose::GroundedProposer.new(config: DSPy::Propose::GroundedProposer::Config.new)
|
287
266
|
@optimization_trace = []
|
288
267
|
@evaluated_candidates = []
|
289
268
|
end
|
@@ -302,8 +281,8 @@ module DSPy
|
|
302
281
|
instrument_step('miprov2_compile', {
|
303
282
|
trainset_size: trainset.size,
|
304
283
|
valset_size: valset&.size || 0,
|
305
|
-
num_trials:
|
306
|
-
optimization_strategy:
|
284
|
+
num_trials: config.num_trials,
|
285
|
+
optimization_strategy: config.optimization_strategy,
|
307
286
|
mode: infer_auto_mode
|
308
287
|
}) do
|
309
288
|
# Convert examples to typed format
|
@@ -363,11 +342,11 @@ module DSPy
|
|
363
342
|
sig { params(program: T.untyped, trainset: T::Array[DSPy::Example]).returns(Utils::BootstrapResult) }
|
364
343
|
def phase_1_bootstrap(program, trainset)
|
365
344
|
bootstrap_config = Utils::BootstrapConfig.new
|
366
|
-
bootstrap_config.max_bootstrapped_examples =
|
367
|
-
bootstrap_config.max_labeled_examples =
|
368
|
-
bootstrap_config.num_candidate_sets =
|
369
|
-
bootstrap_config.max_errors =
|
370
|
-
bootstrap_config.num_threads =
|
345
|
+
bootstrap_config.max_bootstrapped_examples = config.max_bootstrapped_examples
|
346
|
+
bootstrap_config.max_labeled_examples = config.max_labeled_examples
|
347
|
+
bootstrap_config.num_candidate_sets = config.bootstrap_sets
|
348
|
+
bootstrap_config.max_errors = config.max_errors
|
349
|
+
bootstrap_config.num_threads = config.num_threads
|
371
350
|
|
372
351
|
Utils.create_n_fewshot_demo_sets(program, trainset, config: bootstrap_config, metric: @metric)
|
373
352
|
end
|
@@ -392,7 +371,7 @@ module DSPy
|
|
392
371
|
raise ArgumentError, "Cannot extract signature class from program" unless signature_class
|
393
372
|
|
394
373
|
# Configure proposer for this optimization run
|
395
|
-
@
|
374
|
+
@proposer.config.num_instruction_candidates = config.num_instruction_candidates
|
396
375
|
|
397
376
|
@proposer.propose_instructions(
|
398
377
|
signature_class,
|
@@ -425,7 +404,7 @@ module DSPy
|
|
425
404
|
best_program = nil
|
426
405
|
best_evaluation_result = nil
|
427
406
|
|
428
|
-
|
407
|
+
config.num_trials.times do |trial_idx|
|
429
408
|
trials_completed = trial_idx + 1
|
430
409
|
|
431
410
|
# Select next candidate based on optimization strategy
|
@@ -434,8 +413,8 @@ module DSPy
|
|
434
413
|
emit_event('trial_start', {
|
435
414
|
trial_number: trials_completed,
|
436
415
|
candidate_id: candidate.config_id,
|
437
|
-
instruction_preview: candidate.
|
438
|
-
num_few_shot: candidate.
|
416
|
+
instruction_preview: candidate.instruction[0, 50],
|
417
|
+
num_few_shot: candidate.few_shot_examples.size
|
439
418
|
})
|
440
419
|
|
441
420
|
begin
|
@@ -494,37 +473,40 @@ module DSPy
|
|
494
473
|
params(
|
495
474
|
proposal_result: DSPy::Propose::GroundedProposer::ProposalResult,
|
496
475
|
bootstrap_result: Utils::BootstrapResult
|
497
|
-
).returns(T::Array[
|
476
|
+
).returns(T::Array[EvaluatedCandidate])
|
498
477
|
end
|
499
478
|
def generate_candidate_configurations(proposal_result, bootstrap_result)
|
500
479
|
candidates = []
|
501
480
|
|
502
481
|
# Base configuration (no modifications)
|
503
|
-
candidates <<
|
504
|
-
|
505
|
-
|
506
|
-
|
507
|
-
|
508
|
-
|
482
|
+
candidates << EvaluatedCandidate.new(
|
483
|
+
instruction: "",
|
484
|
+
few_shot_examples: [],
|
485
|
+
type: CandidateType::Baseline,
|
486
|
+
metadata: {},
|
487
|
+
config_id: SecureRandom.hex(6)
|
488
|
+
)
|
509
489
|
|
510
490
|
# Instruction-only candidates
|
511
491
|
proposal_result.candidate_instructions.each_with_index do |instruction, idx|
|
512
|
-
candidates <<
|
513
|
-
|
514
|
-
|
515
|
-
|
516
|
-
|
517
|
-
|
492
|
+
candidates << EvaluatedCandidate.new(
|
493
|
+
instruction: instruction,
|
494
|
+
few_shot_examples: [],
|
495
|
+
type: CandidateType::InstructionOnly,
|
496
|
+
metadata: { proposal_rank: idx },
|
497
|
+
config_id: SecureRandom.hex(6)
|
498
|
+
)
|
518
499
|
end
|
519
500
|
|
520
501
|
# Few-shot only candidates
|
521
502
|
bootstrap_result.candidate_sets.each_with_index do |candidate_set, idx|
|
522
|
-
candidates <<
|
523
|
-
|
524
|
-
|
525
|
-
|
526
|
-
|
527
|
-
|
503
|
+
candidates << EvaluatedCandidate.new(
|
504
|
+
instruction: "",
|
505
|
+
few_shot_examples: candidate_set,
|
506
|
+
type: CandidateType::FewShotOnly,
|
507
|
+
metadata: { bootstrap_rank: idx },
|
508
|
+
config_id: SecureRandom.hex(6)
|
509
|
+
)
|
528
510
|
end
|
529
511
|
|
530
512
|
# Combined candidates (instruction + few-shot)
|
@@ -533,15 +515,16 @@ module DSPy
|
|
533
515
|
|
534
516
|
top_instructions.each_with_index do |instruction, i_idx|
|
535
517
|
top_bootstrap_sets.each_with_index do |candidate_set, b_idx|
|
536
|
-
candidates <<
|
537
|
-
|
538
|
-
|
539
|
-
|
540
|
-
|
518
|
+
candidates << EvaluatedCandidate.new(
|
519
|
+
instruction: instruction,
|
520
|
+
few_shot_examples: candidate_set,
|
521
|
+
type: CandidateType::Combined,
|
522
|
+
metadata: {
|
541
523
|
instruction_rank: i_idx,
|
542
524
|
bootstrap_rank: b_idx
|
543
|
-
}
|
544
|
-
|
525
|
+
},
|
526
|
+
config_id: SecureRandom.hex(6)
|
527
|
+
)
|
545
528
|
end
|
546
529
|
end
|
547
530
|
|
@@ -549,13 +532,13 @@ module DSPy
|
|
549
532
|
end
|
550
533
|
|
551
534
|
# Initialize optimization state for candidate selection
|
552
|
-
sig { params(candidates: T::Array[
|
535
|
+
sig { params(candidates: T::Array[EvaluatedCandidate]).returns(T::Hash[Symbol, T.untyped]) }
|
553
536
|
def initialize_optimization_state(candidates)
|
554
537
|
{
|
555
538
|
candidates: candidates,
|
556
539
|
scores: {},
|
557
540
|
exploration_counts: Hash.new(0),
|
558
|
-
temperature:
|
541
|
+
temperature: config.init_temperature,
|
559
542
|
best_score_history: [],
|
560
543
|
diversity_scores: {},
|
561
544
|
no_improvement_count: 0
|
@@ -565,18 +548,18 @@ module DSPy
|
|
565
548
|
# Select next candidate based on optimization strategy
|
566
549
|
sig do
|
567
550
|
params(
|
568
|
-
candidates: T::Array[
|
551
|
+
candidates: T::Array[EvaluatedCandidate],
|
569
552
|
state: T::Hash[Symbol, T.untyped],
|
570
553
|
trial_idx: Integer
|
571
|
-
).returns(
|
554
|
+
).returns(EvaluatedCandidate)
|
572
555
|
end
|
573
556
|
def select_next_candidate(candidates, state, trial_idx)
|
574
|
-
case
|
575
|
-
when
|
557
|
+
case config.optimization_strategy
|
558
|
+
when OptimizationStrategy::Greedy
|
576
559
|
select_candidate_greedy(candidates, state)
|
577
|
-
when
|
560
|
+
when OptimizationStrategy::Adaptive
|
578
561
|
select_candidate_adaptive(candidates, state, trial_idx)
|
579
|
-
when
|
562
|
+
when OptimizationStrategy::Bayesian
|
580
563
|
select_candidate_bayesian(candidates, state, trial_idx)
|
581
564
|
else
|
582
565
|
candidates.sample # Random fallback
|
@@ -584,7 +567,7 @@ module DSPy
|
|
584
567
|
end
|
585
568
|
|
586
569
|
# Greedy candidate selection (exploit best known configurations)
|
587
|
-
sig { params(candidates: T::Array[
|
570
|
+
sig { params(candidates: T::Array[EvaluatedCandidate], state: T::Hash[Symbol, T.untyped]).returns(EvaluatedCandidate) }
|
588
571
|
def select_candidate_greedy(candidates, state)
|
589
572
|
# Prioritize unexplored candidates, then highest scoring
|
590
573
|
unexplored = candidates.reject { |c| state[:scores].key?(c.config_id) }
|
@@ -598,15 +581,15 @@ module DSPy
|
|
598
581
|
# Adaptive candidate selection (balance exploration and exploitation)
|
599
582
|
sig do
|
600
583
|
params(
|
601
|
-
candidates: T::Array[
|
584
|
+
candidates: T::Array[EvaluatedCandidate],
|
602
585
|
state: T::Hash[Symbol, T.untyped],
|
603
586
|
trial_idx: Integer
|
604
|
-
).returns(
|
587
|
+
).returns(EvaluatedCandidate)
|
605
588
|
end
|
606
589
|
def select_candidate_adaptive(candidates, state, trial_idx)
|
607
590
|
# Update temperature based on progress
|
608
|
-
progress = trial_idx.to_f /
|
609
|
-
state[:temperature] =
|
591
|
+
progress = trial_idx.to_f / config.num_trials
|
592
|
+
state[:temperature] = config.init_temperature * (1 - progress) + config.final_temperature * progress
|
610
593
|
|
611
594
|
# Calculate selection scores combining exploitation and exploration
|
612
595
|
candidate_scores = candidates.map do |candidate|
|
@@ -639,10 +622,10 @@ module DSPy
|
|
639
622
|
# Bayesian candidate selection (use probabilistic model)
|
640
623
|
sig do
|
641
624
|
params(
|
642
|
-
candidates: T::Array[
|
625
|
+
candidates: T::Array[EvaluatedCandidate],
|
643
626
|
state: T::Hash[Symbol, T.untyped],
|
644
627
|
trial_idx: Integer
|
645
|
-
).returns(
|
628
|
+
).returns(EvaluatedCandidate)
|
646
629
|
end
|
647
630
|
def select_candidate_bayesian(candidates, state, trial_idx)
|
648
631
|
# Need at least 3 observations to fit GP, otherwise fall back to adaptive
|
@@ -686,21 +669,9 @@ module DSPy
|
|
686
669
|
|
687
670
|
private
|
688
671
|
|
689
|
-
# Helper method to create CandidateConfig with dry-configurable syntax
|
690
|
-
sig do
|
691
|
-
params(
|
692
|
-
block: T.proc.params(config: Dry::Configurable::Config).void
|
693
|
-
).returns(CandidateConfig)
|
694
|
-
end
|
695
|
-
def create_candidate_config(&block)
|
696
|
-
candidate = CandidateConfig.new
|
697
|
-
candidate.configure(&block)
|
698
|
-
candidate.finalize!
|
699
|
-
candidate
|
700
|
-
end
|
701
672
|
|
702
673
|
# Encode candidates as numerical features for Gaussian Process
|
703
|
-
sig { params(candidates: T::Array[
|
674
|
+
sig { params(candidates: T::Array[EvaluatedCandidate]).returns(T::Array[T::Array[Float]]) }
|
704
675
|
def encode_candidates_for_gp(candidates)
|
705
676
|
# Simple encoding: use hash of config as features
|
706
677
|
# In practice, this could be more sophisticated (e.g., instruction embeddings)
|
@@ -715,7 +686,7 @@ module DSPy
|
|
715
686
|
features << ((config_hash / 1_000_000) % 1000).to_f / 1000.0 # Feature 3: high bits
|
716
687
|
|
717
688
|
# Add instruction length if available
|
718
|
-
instruction = candidate.
|
689
|
+
instruction = candidate.instruction
|
719
690
|
if instruction && !instruction.empty?
|
720
691
|
features << [instruction.length.to_f / 100.0, 2.0].min # Instruction length, capped at 200 chars
|
721
692
|
else
|
@@ -730,7 +701,7 @@ module DSPy
|
|
730
701
|
sig do
|
731
702
|
params(
|
732
703
|
program: T.untyped,
|
733
|
-
candidate:
|
704
|
+
candidate: EvaluatedCandidate,
|
734
705
|
evaluation_set: T::Array[DSPy::Example]
|
735
706
|
).returns([Float, T.untyped, DSPy::Evaluate::BatchEvaluationResult])
|
736
707
|
end
|
@@ -748,18 +719,18 @@ module DSPy
|
|
748
719
|
end
|
749
720
|
|
750
721
|
# Apply candidate configuration to program
|
751
|
-
sig { params(program: T.untyped, candidate:
|
722
|
+
sig { params(program: T.untyped, candidate: EvaluatedCandidate).returns(T.untyped) }
|
752
723
|
def apply_candidate_configuration(program, candidate)
|
753
724
|
modified_program = program
|
754
725
|
|
755
726
|
# Apply instruction if provided
|
756
|
-
if !candidate.
|
757
|
-
modified_program = modified_program.with_instruction(candidate.
|
727
|
+
if !candidate.instruction.empty? && program.respond_to?(:with_instruction)
|
728
|
+
modified_program = modified_program.with_instruction(candidate.instruction)
|
758
729
|
end
|
759
730
|
|
760
731
|
# Apply few-shot examples if provided
|
761
|
-
if candidate.
|
762
|
-
few_shot_examples = candidate.
|
732
|
+
if candidate.few_shot_examples.any? && program.respond_to?(:with_examples)
|
733
|
+
few_shot_examples = candidate.few_shot_examples.map do |example|
|
763
734
|
DSPy::FewShotExample.new(
|
764
735
|
input: example.input_values,
|
765
736
|
output: example.expected_values,
|
@@ -776,7 +747,7 @@ module DSPy
|
|
776
747
|
sig do
|
777
748
|
params(
|
778
749
|
state: T::Hash[Symbol, T.untyped],
|
779
|
-
candidate:
|
750
|
+
candidate: EvaluatedCandidate,
|
780
751
|
score: Float
|
781
752
|
).void
|
782
753
|
end
|
@@ -786,7 +757,7 @@ module DSPy
|
|
786
757
|
state[:best_score_history] << score
|
787
758
|
|
788
759
|
# Track diversity if enabled
|
789
|
-
if
|
760
|
+
if config.track_diversity
|
790
761
|
state[:diversity_scores][candidate.config_id] = calculate_diversity_score(candidate)
|
791
762
|
end
|
792
763
|
|
@@ -802,18 +773,18 @@ module DSPy
|
|
802
773
|
sig { params(state: T::Hash[Symbol, T.untyped], trial_idx: Integer).returns(T::Boolean) }
|
803
774
|
def should_early_stop?(state, trial_idx)
|
804
775
|
# Don't stop too early
|
805
|
-
return false if trial_idx <
|
776
|
+
return false if trial_idx < config.early_stopping_patience
|
806
777
|
|
807
778
|
# Stop if no improvement for patience trials
|
808
|
-
state[:no_improvement_count] >=
|
779
|
+
state[:no_improvement_count] >= config.early_stopping_patience
|
809
780
|
end
|
810
781
|
|
811
782
|
# Calculate diversity score for candidate
|
812
|
-
sig { params(candidate:
|
783
|
+
sig { params(candidate: EvaluatedCandidate).returns(Float) }
|
813
784
|
def calculate_diversity_score(candidate)
|
814
785
|
# Simple diversity metric based on instruction length and few-shot count
|
815
|
-
instruction_diversity = candidate.
|
816
|
-
few_shot_diversity = candidate.
|
786
|
+
instruction_diversity = candidate.instruction.length / 200.0
|
787
|
+
few_shot_diversity = candidate.few_shot_examples.size / 10.0
|
817
788
|
|
818
789
|
[instruction_diversity + few_shot_diversity, 1.0].min
|
819
790
|
end
|
@@ -836,17 +807,17 @@ module DSPy
|
|
836
807
|
|
837
808
|
history = {
|
838
809
|
total_trials: optimization_result[:trials_completed],
|
839
|
-
optimization_strategy:
|
840
|
-
early_stopped: optimization_result[:trials_completed] <
|
810
|
+
optimization_strategy: config.optimization_strategy,
|
811
|
+
early_stopped: optimization_result[:trials_completed] < config.num_trials,
|
841
812
|
score_history: optimization_result[:optimization_state][:best_score_history]
|
842
813
|
}
|
843
814
|
|
844
815
|
metadata = {
|
845
816
|
optimizer: "MIPROv2",
|
846
817
|
auto_mode: infer_auto_mode,
|
847
|
-
best_instruction: best_candidate&.
|
848
|
-
best_few_shot_count: best_candidate&.
|
849
|
-
best_candidate_type: best_candidate&.
|
818
|
+
best_instruction: best_candidate&.instruction || "",
|
819
|
+
best_few_shot_count: best_candidate&.few_shot_examples&.size || 0,
|
820
|
+
best_candidate_type: best_candidate&.type&.serialize || "unknown",
|
850
821
|
optimization_timestamp: Time.now.iso8601
|
851
822
|
}
|
852
823
|
|
@@ -917,7 +888,7 @@ module DSPy
|
|
917
888
|
# Infer auto mode based on configuration
|
918
889
|
sig { returns(String) }
|
919
890
|
def infer_auto_mode
|
920
|
-
case
|
891
|
+
case config.num_trials
|
921
892
|
when 0..6 then "light"
|
922
893
|
when 7..12 then "medium"
|
923
894
|
else "heavy"
|
data/lib/dspy/version.rb
CHANGED
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: dspy
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.26.
|
4
|
+
version: 0.26.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Vicente Reig Rincón de Arellano
|
8
8
|
bindir: bin
|
9
9
|
cert_chain: []
|
10
|
-
date: 2025-09-
|
10
|
+
date: 2025-09-10 00:00:00.000000000 Z
|
11
11
|
dependencies:
|
12
12
|
- !ruby/object:Gem::Dependency
|
13
13
|
name: dry-configurable
|