dspy 0.6.3 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: ae1673ad31bc71e2573800124ebb74ea807cd94a1c14b9b58f3de94b7f62363a
4
- data.tar.gz: f52d96924b536e5941ff34b45f1080ddb08dc9243095df51f11352cb65979ba1
3
+ metadata.gz: cc32a4fe2ec14d442c2a0603b69992528ab81c67c4e61ceed6a6d337278d0220
4
+ data.tar.gz: f901d0336e0a4c912dfb37428bdd0e359b74de35b340eb769edcb5e811e257fe
5
5
  SHA512:
6
- metadata.gz: 98f1d7b285f9342b8dfc82cdc7606d2a4c8cbdb1df86713b7176a389893f7c652e7c5df473ddcf42335ab37a2cfffe35e9519310b45be396c9bfe8847c7fabc1
7
- data.tar.gz: cbee70fb3afc5b5b7dce3caf2efbca258d5cbd12680f950fe52be30bd84754369b620c21617315c04a62e2f982fe485b0c05f7ed30676007c19527715c3fc9c0
6
+ metadata.gz: db9477a7f7900559bb7c362104d9921d79c12b84f18ccfe0e51006adafd6e478624d20a930184e9e3dfacf9d1c88321ef54b6688f89ddf627fa0ecef3da12c53
7
+ data.tar.gz: 5ef621dfd8a8ef83f2bf0d3eabb32b07ae881636fe4c1826fddc3e544d3b5193448ff96bd28f970046d499dca69d23affa42596d41315e8c3245c4e7db5be0b3
data/README.md CHANGED
@@ -25,6 +25,9 @@ The result? LLM applications that actually scale and don't break when you sneeze
25
25
  - **Basic Optimization** - Simple prompt optimization techniques
26
26
 
27
27
  **Production Features:**
28
+ - **Reliable JSON Extraction** - Automatic strategy selection for OpenAI structured outputs, Anthropic patterns, and fallback modes
29
+ - **Smart Retry Logic** - Progressive fallback with exponential backoff for handling transient failures
30
+ - **Performance Caching** - Schema and capability caching for faster repeated operations
28
31
  - **File-based Storage** - Basic optimization result persistence
29
32
  - **Multi-Platform Observability** - OpenTelemetry, New Relic, and Langfuse integration
30
33
  - **Basic Instrumentation** - Event tracking and logging
@@ -78,7 +81,9 @@ end
78
81
 
79
82
  # Configure DSPy with your LLM
80
83
  DSPy.configure do |c|
81
- c.lm = DSPy::LM.new('openai/gpt-4o-mini', api_key: ENV['OPENAI_API_KEY'])
84
+ c.lm = DSPy::LM.new('openai/gpt-4o-mini',
85
+ api_key: ENV['OPENAI_API_KEY'],
86
+ structured_outputs: true) # Enable OpenAI's native JSON mode
82
87
  end
83
88
 
84
89
  # Create the predictor and run inference
@@ -14,9 +14,10 @@ module DSPy
14
14
 
15
15
  # Chat interface that all adapters must implement
16
16
  # @param messages [Array<Hash>] Array of message hashes with :role and :content
17
+ # @param signature [DSPy::Signature, nil] Optional signature for structured outputs
17
18
  # @param block [Proc] Optional streaming block
18
19
  # @return [DSPy::LM::Response] Normalized response
19
- def chat(messages:, &block)
20
+ def chat(messages:, signature: nil, &block)
20
21
  raise NotImplementedError, "Subclasses must implement #chat method"
21
22
  end
22
23
 
@@ -24,7 +25,12 @@ module DSPy
24
25
 
25
26
  def validate_configuration!
26
27
  raise ConfigurationError, "Model is required" if model.nil? || model.empty?
27
- raise ConfigurationError, "API key is required" if api_key.nil? || api_key.empty?
28
+ end
29
+
30
+ def validate_api_key!(api_key, provider)
31
+ if api_key.nil? || api_key.to_s.strip.empty?
32
+ raise MissingAPIKeyError.new(provider)
33
+ end
28
34
  end
29
35
 
30
36
  # Helper method to normalize message format
@@ -14,12 +14,17 @@ module DSPy
14
14
  # Creates an adapter instance based on model_id
15
15
  # @param model_id [String] Full model identifier (e.g., "openai/gpt-4")
16
16
  # @param api_key [String] API key for the provider
17
+ # @param options [Hash] Additional adapter-specific options
17
18
  # @return [DSPy::LM::Adapter] Appropriate adapter instance
18
- def create(model_id, api_key:)
19
+ def create(model_id, api_key:, **options)
19
20
  provider, model = parse_model_id(model_id)
20
21
  adapter_class = get_adapter_class(provider)
21
22
 
22
- adapter_class.new(model: model, api_key: api_key)
23
+ # Pass provider-specific options
24
+ adapter_options = { model: model, api_key: api_key }
25
+ adapter_options.merge!(options) if provider == 'openai' # Only OpenAI accepts structured_outputs for now
26
+
27
+ adapter_class.new(**adapter_options)
23
28
  end
24
29
 
25
30
  private
@@ -7,10 +7,11 @@ module DSPy
7
7
  class AnthropicAdapter < Adapter
8
8
  def initialize(model:, api_key:)
9
9
  super
10
+ validate_api_key!(api_key, 'anthropic')
10
11
  @client = Anthropic::Client.new(api_key: api_key)
11
12
  end
12
13
 
13
- def chat(messages:, &block)
14
+ def chat(messages:, signature: nil, **extra_params, &block)
14
15
  # Anthropic requires system message to be separate from messages
15
16
  system_message, user_messages = extract_system_message(normalize_messages(messages))
16
17
 
@@ -0,0 +1,269 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "sorbet-runtime"
4
+ require_relative "../../cache_manager"
5
+
6
+ module DSPy
7
+ class LM
8
+ module Adapters
9
+ module OpenAI
10
+ # Converts DSPy signatures to OpenAI structured output format
11
+ class SchemaConverter
12
+ extend T::Sig
13
+
14
+ # Models that support structured outputs as of July 2025
15
+ STRUCTURED_OUTPUT_MODELS = T.let([
16
+ "gpt-4o-mini",
17
+ "gpt-4o-2024-08-06",
18
+ "gpt-4o",
19
+ "gpt-4-turbo",
20
+ "gpt-4-turbo-2024-04-09"
21
+ ].freeze, T::Array[String])
22
+
23
+ sig { params(signature_class: T.class_of(DSPy::Signature), name: T.nilable(String), strict: T::Boolean).returns(T::Hash[Symbol, T.untyped]) }
24
+ def self.to_openai_format(signature_class, name: nil, strict: true)
25
+ # Build cache params from the method parameters
26
+ cache_params = { strict: strict }
27
+ cache_params[:name] = name if name
28
+
29
+ # Check cache first
30
+ cache_manager = DSPy::LM.cache_manager
31
+ cached_schema = cache_manager.get_schema(signature_class, "openai", cache_params)
32
+
33
+ if cached_schema
34
+ DSPy.logger.debug("Using cached schema for #{signature_class.name}")
35
+ return cached_schema
36
+ end
37
+
38
+ # Get the output JSON schema from the signature class
39
+ output_schema = signature_class.output_json_schema
40
+
41
+ # Build the complete schema
42
+ dspy_schema = {
43
+ "$schema": "http://json-schema.org/draft-06/schema#",
44
+ type: "object",
45
+ properties: output_schema[:properties] || {},
46
+ required: output_schema[:required] || []
47
+ }
48
+
49
+ # Generate a schema name if not provided
50
+ schema_name = name || generate_schema_name(signature_class)
51
+
52
+ # Remove the $schema field as OpenAI doesn't use it
53
+ openai_schema = dspy_schema.except(:$schema)
54
+
55
+ # Add additionalProperties: false for strict mode
56
+ if strict
57
+ openai_schema = add_additional_properties_recursively(openai_schema)
58
+ end
59
+
60
+ # Wrap in OpenAI's required format
61
+ result = {
62
+ type: "json_schema",
63
+ json_schema: {
64
+ name: schema_name,
65
+ strict: strict,
66
+ schema: openai_schema
67
+ }
68
+ }
69
+
70
+ # Cache the result with same params
71
+ cache_manager.cache_schema(signature_class, "openai", result, cache_params)
72
+
73
+ result
74
+ end
75
+
76
+ sig { params(model: String).returns(T::Boolean) }
77
+ def self.supports_structured_outputs?(model)
78
+ # Check cache first
79
+ cache_manager = DSPy::LM.cache_manager
80
+ cached_result = cache_manager.get_capability(model, "structured_outputs")
81
+
82
+ if !cached_result.nil?
83
+ DSPy.logger.debug("Using cached capability check for #{model}")
84
+ return cached_result
85
+ end
86
+
87
+ # Extract base model name without provider prefix
88
+ base_model = model.sub(/^openai\//, "")
89
+
90
+ # Check if it's a supported model or a newer version
91
+ result = STRUCTURED_OUTPUT_MODELS.any? { |supported| base_model.start_with?(supported) }
92
+
93
+ # Cache the result
94
+ cache_manager.cache_capability(model, "structured_outputs", result)
95
+
96
+ result
97
+ end
98
+
99
+ sig { params(schema: T::Hash[Symbol, T.untyped]).returns(T::Array[String]) }
100
+ def self.validate_compatibility(schema)
101
+ issues = []
102
+
103
+ # Check for deeply nested objects (OpenAI has depth limits)
104
+ depth = calculate_depth(schema)
105
+ if depth > 5
106
+ issues << "Schema depth (#{depth}) exceeds recommended limit of 5 levels"
107
+ end
108
+
109
+ # Check for unsupported JSON Schema features
110
+ if contains_pattern_properties?(schema)
111
+ issues << "Pattern properties are not supported in OpenAI structured outputs"
112
+ end
113
+
114
+ if contains_conditional_schemas?(schema)
115
+ issues << "Conditional schemas (if/then/else) are not supported"
116
+ end
117
+
118
+ issues
119
+ end
120
+
121
+ private
122
+
123
+ sig { params(schema: T::Hash[Symbol, T.untyped]).returns(T::Hash[Symbol, T.untyped]) }
124
+ def self.add_additional_properties_recursively(schema)
125
+ return schema unless schema.is_a?(Hash)
126
+
127
+ result = schema.dup
128
+
129
+ # Add additionalProperties: false if this is an object
130
+ if result[:type] == "object"
131
+ result[:additionalProperties] = false
132
+ end
133
+
134
+ # Process properties recursively
135
+ if result[:properties].is_a?(Hash)
136
+ result[:properties] = result[:properties].transform_values do |prop|
137
+ if prop.is_a?(Hash)
138
+ processed = add_additional_properties_recursively(prop)
139
+ # Special handling for arrays - ensure their items have additionalProperties if they're objects
140
+ if processed[:type] == "array" && processed[:items].is_a?(Hash)
141
+ processed[:items] = add_additional_properties_recursively(processed[:items])
142
+ end
143
+ processed
144
+ else
145
+ prop
146
+ end
147
+ end
148
+ end
149
+
150
+ # Process array items
151
+ if result[:items].is_a?(Hash)
152
+ processed_items = add_additional_properties_recursively(result[:items])
153
+ # OpenAI requires additionalProperties on all objects, even in array items
154
+ if processed_items.is_a?(Hash) && processed_items[:type] == "object" && !processed_items.key?(:additionalProperties)
155
+ processed_items[:additionalProperties] = false
156
+ end
157
+ result[:items] = processed_items
158
+ elsif result[:items].is_a?(Array)
159
+ # Handle tuple validation
160
+ result[:items] = result[:items].map do |item|
161
+ processed = item.is_a?(Hash) ? add_additional_properties_recursively(item) : item
162
+ if processed.is_a?(Hash) && processed[:type] == "object" && !processed.key?(:additionalProperties)
163
+ processed[:additionalProperties] = false
164
+ end
165
+ processed
166
+ end
167
+ end
168
+
169
+ # Process oneOf/anyOf/allOf
170
+ [:oneOf, :anyOf, :allOf].each do |key|
171
+ if result[key].is_a?(Array)
172
+ result[key] = result[key].map do |sub_schema|
173
+ sub_schema.is_a?(Hash) ? add_additional_properties_recursively(sub_schema) : sub_schema
174
+ end
175
+ end
176
+ end
177
+
178
+ result
179
+ end
180
+
181
+ sig { params(signature_class: T.class_of(DSPy::Signature)).returns(String) }
182
+ def self.generate_schema_name(signature_class)
183
+ # Use the signature class name
184
+ class_name = signature_class.name&.split("::")&.last
185
+ if class_name
186
+ class_name.gsub(/[^a-zA-Z0-9_]/, "_").downcase
187
+ else
188
+ # Fallback to a generic name
189
+ "dspy_output_#{Time.now.to_i}"
190
+ end
191
+ end
192
+
193
+ sig { params(schema: T::Hash[Symbol, T.untyped], current_depth: Integer).returns(Integer) }
194
+ def self.calculate_depth(schema, current_depth = 0)
195
+ return current_depth unless schema.is_a?(Hash)
196
+
197
+ max_depth = current_depth
198
+
199
+ # Check properties
200
+ if schema[:properties].is_a?(Hash)
201
+ schema[:properties].each_value do |prop|
202
+ if prop.is_a?(Hash)
203
+ prop_depth = calculate_depth(prop, current_depth + 1)
204
+ max_depth = [max_depth, prop_depth].max
205
+ end
206
+ end
207
+ end
208
+
209
+ # Check array items
210
+ if schema[:items].is_a?(Hash)
211
+ items_depth = calculate_depth(schema[:items], current_depth + 1)
212
+ max_depth = [max_depth, items_depth].max
213
+ end
214
+
215
+ # Check oneOf/anyOf/allOf
216
+ [:oneOf, :anyOf, :allOf].each do |key|
217
+ if schema[key].is_a?(Array)
218
+ schema[key].each do |sub_schema|
219
+ if sub_schema.is_a?(Hash)
220
+ sub_depth = calculate_depth(sub_schema, current_depth + 1)
221
+ max_depth = [max_depth, sub_depth].max
222
+ end
223
+ end
224
+ end
225
+ end
226
+
227
+ max_depth
228
+ end
229
+
230
+ sig { params(schema: T::Hash[Symbol, T.untyped]).returns(T::Boolean) }
231
+ def self.contains_pattern_properties?(schema)
232
+ return true if schema[:patternProperties]
233
+
234
+ # Recursively check nested schemas
235
+ [:properties, :items, :oneOf, :anyOf, :allOf].each do |key|
236
+ value = schema[key]
237
+ case value
238
+ when Hash
239
+ return true if contains_pattern_properties?(value)
240
+ when Array
241
+ return true if value.any? { |v| v.is_a?(Hash) && contains_pattern_properties?(v) }
242
+ end
243
+ end
244
+
245
+ false
246
+ end
247
+
248
+ sig { params(schema: T::Hash[Symbol, T.untyped]).returns(T::Boolean) }
249
+ def self.contains_conditional_schemas?(schema)
250
+ return true if schema[:if] || schema[:then] || schema[:else]
251
+
252
+ # Recursively check nested schemas
253
+ [:properties, :items, :oneOf, :anyOf, :allOf].each do |key|
254
+ value = schema[key]
255
+ case value
256
+ when Hash
257
+ return true if contains_conditional_schemas?(value)
258
+ when Array
259
+ return true if value.any? { |v| v.is_a?(Hash) && contains_conditional_schemas?(v) }
260
+ end
261
+ end
262
+
263
+ false
264
+ end
265
+ end
266
+ end
267
+ end
268
+ end
269
+ end
@@ -1,22 +1,34 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require 'openai'
4
+ require_relative 'openai/schema_converter'
4
5
 
5
6
  module DSPy
6
7
  class LM
7
8
  class OpenAIAdapter < Adapter
8
- def initialize(model:, api_key:)
9
- super
9
+ def initialize(model:, api_key:, structured_outputs: false)
10
+ super(model: model, api_key: api_key)
11
+ validate_api_key!(api_key, 'openai')
10
12
  @client = OpenAI::Client.new(api_key: api_key)
13
+ @structured_outputs_enabled = structured_outputs
11
14
  end
12
15
 
13
- def chat(messages:, &block)
16
+ def chat(messages:, signature: nil, response_format: nil, &block)
14
17
  request_params = {
15
18
  model: model,
16
19
  messages: normalize_messages(messages),
17
20
  temperature: 0.0 # DSPy default for deterministic responses
18
21
  }
19
22
 
23
+ # Add response format if provided by strategy
24
+ if response_format
25
+ request_params[:response_format] = response_format
26
+ elsif @structured_outputs_enabled && signature && supports_structured_outputs?
27
+ # Legacy behavior for backward compatibility
28
+ response_format = DSPy::LM::Adapters::OpenAI::SchemaConverter.to_openai_format(signature)
29
+ request_params[:response_format] = response_format
30
+ end
31
+
20
32
  # Add streaming if block provided
21
33
  if block_given?
22
34
  request_params[:stream] = proc do |chunk, _bytesize|
@@ -31,9 +43,15 @@ module DSPy
31
43
  raise AdapterError, "OpenAI API error: #{response.error}"
32
44
  end
33
45
 
34
- content = response.choices.first.message.content
46
+ message = response.choices.first.message
47
+ content = message.content
35
48
  usage = response.usage
36
49
 
50
+ # Handle structured output refusals
51
+ if message.respond_to?(:refusal) && message.refusal
52
+ raise AdapterError, "OpenAI refused to generate output: #{message.refusal}"
53
+ end
54
+
37
55
  Response.new(
38
56
  content: content,
39
57
  usage: usage.respond_to?(:to_h) ? usage.to_h : usage,
@@ -41,13 +59,20 @@ module DSPy
41
59
  provider: 'openai',
42
60
  model: model,
43
61
  response_id: response.id,
44
- created: response.created
62
+ created: response.created,
63
+ structured_output: @structured_outputs_enabled && signature && supports_structured_outputs?
45
64
  }
46
65
  )
47
66
  rescue => e
48
67
  raise AdapterError, "OpenAI adapter error: #{e.message}"
49
68
  end
50
69
  end
70
+
71
+ private
72
+
73
+ def supports_structured_outputs?
74
+ DSPy::LM::Adapters::OpenAI::SchemaConverter.supports_structured_outputs?(model)
75
+ end
51
76
  end
52
77
  end
53
78
  end
@@ -0,0 +1,151 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "sorbet-runtime"
4
+
5
+ module DSPy
6
+ class LM
7
+ # Manages caching for schemas and capability detection
8
+ class CacheManager
9
+ extend T::Sig
10
+
11
+ # Cache entry with TTL
12
+ class CacheEntry < T::Struct
13
+ extend T::Sig
14
+
15
+ const :value, T.untyped
16
+ const :expires_at, Time
17
+
18
+ sig { returns(T::Boolean) }
19
+ def expired?
20
+ Time.now > expires_at
21
+ end
22
+ end
23
+
24
+ DEFAULT_TTL = 3600 # 1 hour
25
+
26
+ sig { void }
27
+ def initialize
28
+ @schema_cache = {}
29
+ @capability_cache = {}
30
+ @mutex = Mutex.new
31
+ end
32
+
33
+ # Cache a schema for a signature class
34
+ sig { params(signature_class: T.class_of(DSPy::Signature), provider: String, schema: T.untyped, cache_params: T::Hash[Symbol, T.untyped]).void }
35
+ def cache_schema(signature_class, provider, schema, cache_params = {})
36
+ key = schema_key(signature_class, provider, cache_params)
37
+
38
+ @mutex.synchronize do
39
+ @schema_cache[key] = CacheEntry.new(
40
+ value: schema,
41
+ expires_at: Time.now + DEFAULT_TTL
42
+ )
43
+ end
44
+
45
+ DSPy.logger.debug("Cached schema for #{signature_class.name} (#{provider})")
46
+ end
47
+
48
+ # Get cached schema if available
49
+ sig { params(signature_class: T.class_of(DSPy::Signature), provider: String, cache_params: T::Hash[Symbol, T.untyped]).returns(T.nilable(T.untyped)) }
50
+ def get_schema(signature_class, provider, cache_params = {})
51
+ key = schema_key(signature_class, provider, cache_params)
52
+
53
+ @mutex.synchronize do
54
+ entry = @schema_cache[key]
55
+
56
+ if entry.nil?
57
+ nil
58
+ elsif entry.expired?
59
+ @schema_cache.delete(key)
60
+ nil
61
+ else
62
+ entry.value
63
+ end
64
+ end
65
+ end
66
+
67
+ # Cache capability detection result
68
+ sig { params(model: String, capability: String, result: T::Boolean).void }
69
+ def cache_capability(model, capability, result)
70
+ key = capability_key(model, capability)
71
+
72
+ @mutex.synchronize do
73
+ @capability_cache[key] = CacheEntry.new(
74
+ value: result,
75
+ expires_at: Time.now + DEFAULT_TTL * 24 # Capabilities change less frequently
76
+ )
77
+ end
78
+
79
+ DSPy.logger.debug("Cached capability #{capability} for #{model}: #{result}")
80
+ end
81
+
82
+ # Get cached capability if available
83
+ sig { params(model: String, capability: String).returns(T.nilable(T::Boolean)) }
84
+ def get_capability(model, capability)
85
+ key = capability_key(model, capability)
86
+
87
+ @mutex.synchronize do
88
+ entry = @capability_cache[key]
89
+
90
+ if entry.nil?
91
+ nil
92
+ elsif entry.expired?
93
+ @capability_cache.delete(key)
94
+ nil
95
+ else
96
+ entry.value
97
+ end
98
+ end
99
+ end
100
+
101
+ # Clear all caches
102
+ sig { void }
103
+ def clear!
104
+ @mutex.synchronize do
105
+ @schema_cache.clear
106
+ @capability_cache.clear
107
+ end
108
+
109
+ DSPy.logger.debug("Cleared all caches")
110
+ end
111
+
112
+ # Get cache statistics
113
+ sig { returns(T::Hash[Symbol, Integer]) }
114
+ def stats
115
+ @mutex.synchronize do
116
+ {
117
+ schema_entries: @schema_cache.size,
118
+ capability_entries: @capability_cache.size,
119
+ total_entries: @schema_cache.size + @capability_cache.size
120
+ }
121
+ end
122
+ end
123
+
124
+ private
125
+
126
+ sig { params(signature_class: T.class_of(DSPy::Signature), provider: String, cache_params: T::Hash[Symbol, T.untyped]).returns(String) }
127
+ def schema_key(signature_class, provider, cache_params = {})
128
+ params_str = cache_params.sort.map { |k, v| "#{k}:#{v}" }.join(":")
129
+ base_key = "schema:#{provider}:#{signature_class.name}"
130
+ params_str.empty? ? base_key : "#{base_key}:#{params_str}"
131
+ end
132
+
133
+ sig { params(model: String, capability: String).returns(String) }
134
+ def capability_key(model, capability)
135
+ "capability:#{model}:#{capability}"
136
+ end
137
+ end
138
+
139
+ # Global cache instance
140
+ @cache_manager = T.let(nil, T.nilable(CacheManager))
141
+
142
+ class << self
143
+ extend T::Sig
144
+
145
+ sig { returns(CacheManager) }
146
+ def cache_manager
147
+ @cache_manager ||= CacheManager.new
148
+ end
149
+ end
150
+ end
151
+ end
@@ -6,5 +6,18 @@ module DSPy
6
6
  class AdapterError < Error; end
7
7
  class UnsupportedProviderError < Error; end
8
8
  class ConfigurationError < Error; end
9
+
10
+ # Raised when API key is missing or invalid
11
+ class MissingAPIKeyError < Error
12
+ def initialize(provider)
13
+ env_var = case provider
14
+ when 'openai' then 'OPENAI_API_KEY'
15
+ when 'anthropic' then 'ANTHROPIC_API_KEY'
16
+ else "#{provider.upcase}_API_KEY"
17
+ end
18
+
19
+ super("API key is required but was not provided. Set it via the api_key parameter or #{env_var} environment variable.")
20
+ end
21
+ end
9
22
  end
10
23
  end