dspy 0.34.0 → 0.34.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3bf98e1e8f5f939799d7e14717d8859b10830144a9c23f1d4818e6fa021fb46a
-  data.tar.gz: 154e27f97ed2c3ae5b8a04f2d3941a93e79a0e88bae8459fd73a85a9d03ed186
+  metadata.gz: 100e82f4aeff8020a845aa80a63ad86278ead32f34b7846b0624db99dc060325
+  data.tar.gz: 43ed18798e67e829e2decdd8ef0519751d6f4fc2cf52571850f1acfd671f2780
 SHA512:
-  metadata.gz: 298305a05b5a38806d67989c01ed2a476f291d13ec6c8a228c8d465fa925d56b178b6c0865d4e9282b4e6ab29aa9655e9b7fc64a228478524d2ce94d3015758f
-  data.tar.gz: 207ff7188ff0bcb16bfd6b45893672085363683464889a4d3e3ba666d2989f2daa0260660da00f7e29ba0df360ca85a430536f38fae319a2e2106405e99e0af9
+  metadata.gz: 3081d9fada92dcf1f7f5b003212fd6d5b94787b6982c6828bd95704e84f197be3017d08eff81c4c1271e4a1d442e6a235973f4c2ab69c47493e025b2d155b1ca
+  data.tar.gz: 32454139be26918608d4c63f58a706d762caa457ca3df9addb238eb9c9bfd5e97f749054923573838b6e983db129732de290387076423e39293bcf02e2747bc4

data/lib/dspy/ext/struct_descriptions.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+require 'sorbet-runtime'
+
+module DSPy
+  module Ext
+    # Extends T::Struct to support field descriptions via the :description kwarg.
+    #
+    # This module is prepended to T::Struct to intercept const/prop definitions
+    # and capture descriptions before they reach Sorbet (which doesn't support them).
+    #
+    # @example
+    #   class ASTNode < T::Struct
+    #     const :node_type, String, description: 'The type of AST node'
+    #     const :text, String, default: "", description: 'Text content of the node'
+    #     const :children, T::Array[ASTNode], default: []
+    #   end
+    #
+    #   ASTNode.field_descriptions[:node_type]  # => "The type of AST node"
+    #   ASTNode.field_descriptions[:text]       # => "Text content of the node"
+    #   ASTNode.field_descriptions[:children]   # => nil (no description)
+    #
+    module StructDescriptions
+      def self.prepended(base)
+        base.singleton_class.prepend(ClassMethods)
+      end
+
+      module ClassMethods
+        # Returns a hash of field names to their descriptions.
+        # Only fields with explicit :description kwargs are included.
+        #
+        # @return [Hash{Symbol => String}]
+        def field_descriptions
+          @field_descriptions ||= {}
+        end
+
+        # Intercepts const definitions to capture :description before Sorbet sees it.
+        def const(name, type, **kwargs)
+          if kwargs.key?(:description)
+            field_descriptions[name] = kwargs.delete(:description)
+          end
+          super(name, type, **kwargs)
+        end
+
+        # Intercepts prop definitions to capture :description before Sorbet sees it.
+        def prop(name, type, **kwargs)
+          if kwargs.key?(:description)
+            field_descriptions[name] = kwargs.delete(:description)
+          end
+          super(name, type, **kwargs)
+        end
+      end
+    end
+  end
+end
+
+# Apply the extension to T::Struct globally
+T::Struct.prepend(DSPy::Ext::StructDescriptions)

data/lib/dspy/schema/sorbet_json_schema.rb

@@ -12,10 +12,33 @@ module DSPy
       extend T::Sig
       extend T::Helpers
 
+      # Result type that includes both schema and any accumulated definitions
+      class SchemaResult < T::Struct
+        const :schema, T::Hash[Symbol, T.untyped]
+        const :definitions, T::Hash[String, T::Hash[Symbol, T.untyped]], default: {}
+      end
+
+      # Convert a Sorbet type to JSON Schema format with definitions tracking
+      # Returns a SchemaResult with the schema and any $defs needed
+      sig { params(type: T.untyped, visited: T.nilable(T::Set[T.untyped]), definitions: T.nilable(T::Hash[String, T::Hash[Symbol, T.untyped]])).returns(SchemaResult) }
+      def self.type_to_json_schema_with_defs(type, visited = nil, definitions = nil)
+        visited ||= Set.new
+        definitions ||= {}
+        schema = type_to_json_schema_internal(type, visited, definitions)
+        SchemaResult.new(schema: schema, definitions: definitions)
+      end
+
       # Convert a Sorbet type to JSON Schema format
+      # For backward compatibility, this method returns just the schema hash
       sig { params(type: T.untyped, visited: T.nilable(T::Set[T.untyped])).returns(T::Hash[Symbol, T.untyped]) }
       def self.type_to_json_schema(type, visited = nil)
         visited ||= Set.new
+        type_to_json_schema_internal(type, visited, {})
+      end
+
+      # Internal implementation that tracks definitions
+      sig { params(type: T.untyped, visited: T::Set[T.untyped], definitions: T::Hash[String, T::Hash[Symbol, T.untyped]]).returns(T::Hash[Symbol, T.untyped]) }
+      def self.type_to_json_schema_internal(type, visited, definitions)
         
         # Handle T::Boolean type alias first
         if type == T::Boolean
@@ -24,7 +47,7 @@ module DSPy
 
         # Handle type aliases by resolving to their underlying type
         if type.is_a?(T::Private::Types::TypeAlias)
-          return self.type_to_json_schema(type.aliased_type, visited)
+          return type_to_json_schema_internal(type.aliased_type, visited, definitions)
         end
 
         # Handle raw class types first
@@ -54,12 +77,13 @@ module DSPy
             # Check for recursion
             if visited.include?(type)
               # Return a reference to avoid infinite recursion
+              # Use #/$defs/ format for OpenAI/Gemini compatibility
+              simple_name = type.name.split('::').last
               {
-                "$ref" => "#/definitions/#{type.name.split('::').last}",
-                description: "Recursive reference to #{type.name}"
+                "$ref" => "#/$defs/#{simple_name}"
               }
             else
-              self.generate_struct_schema(type, visited)
+              generate_struct_schema_internal(type, visited, definitions)
             end
           else
             { type: "string" }  # Default fallback
@@ -93,12 +117,13 @@ module DSPy
             elsif type.raw_type < T::Struct
               # Handle custom T::Struct classes
               if visited.include?(type.raw_type)
+                # Use #/$defs/ format for OpenAI/Gemini compatibility
+                simple_name = type.raw_type.name.split('::').last
                 {
-                  "$ref" => "#/definitions/#{type.raw_type.name.split('::').last}",
-                  description: "Recursive reference to #{type.raw_type.name}"
+                  "$ref" => "#/$defs/#{simple_name}"
                 }
               else
-                generate_struct_schema(type.raw_type, visited)
+                generate_struct_schema_internal(type.raw_type, visited, definitions)
               end
             else
               { type: "string" }  # Default fallback
@@ -108,13 +133,13 @@ module DSPy
           # Handle arrays properly with nested item type
           {
             type: "array",
-            items: self.type_to_json_schema(type.type, visited)
+            items: type_to_json_schema_internal(type.type, visited, definitions)
           }
         elsif type.is_a?(T::Types::TypedHash)
           # Handle hashes as objects with additionalProperties
           # TypedHash has keys and values methods to access its key and value types
           # Note: propertyNames is NOT supported by OpenAI structured outputs, so we omit it
-          value_schema = self.type_to_json_schema(type.values, visited)
+          value_schema = type_to_json_schema_internal(type.values, visited, definitions)
           key_type_desc = type.keys.respond_to?(:raw_type) ? type.keys.raw_type.to_s : "string"
           value_type_desc = value_schema[:description] || value_schema[:type].to_s
 
@@ -129,9 +154,9 @@ module DSPy
           # Handle fixed hashes (from type aliases like { "key" => Type })
           properties = {}
           required = []
-          
+
           type.types.each do |key, value_type|
-            properties[key] = self.type_to_json_schema(value_type, visited)
+            properties[key] = type_to_json_schema_internal(value_type, visited, definitions)
             required << key
           end
           
@@ -155,9 +180,9 @@ module DSPy
               !(t.respond_to?(:raw_type) && t.raw_type == NilClass) &&
               !(t.respond_to?(:name) && t.name == "NilClass")
             end
-            
+
             if non_nil_type
-              base_schema = self.type_to_json_schema(non_nil_type, visited)
+              base_schema = type_to_json_schema_internal(non_nil_type, visited, definitions)
               if base_schema[:type].is_a?(String)
                 # Convert single type to array with null
                 { type: [base_schema[:type], "null"] }.merge(base_schema.except(:type))
@@ -173,13 +198,13 @@ module DSPy
             # Generate oneOf schema for all types
             if type.respond_to?(:types) && type.types.length > 1
               {
-                oneOf: type.types.map { |t| self.type_to_json_schema(t, visited) },
+                oneOf: type.types.map { |t| type_to_json_schema_internal(t, visited, definitions) },
                 description: "Union of multiple types"
               }
             else
               # Single type or fallback
               first_type = type.respond_to?(:types) ? type.types.first : type
-              self.type_to_json_schema(first_type, visited)
+              type_to_json_schema_internal(first_type, visited, definitions)
             end
           end
         elsif type.is_a?(T::Types::Union)
@@ -200,7 +225,7 @@ module DSPy
           
           if non_nil_types.size == 1 && is_nilable
             # This is T.nilable(SomeType) - generate proper schema with null allowed
-            base_schema = self.type_to_json_schema(non_nil_types.first, visited)
+            base_schema = type_to_json_schema_internal(non_nil_types.first, visited, definitions)
             if base_schema[:type].is_a?(String)
               # Convert single type to array with null
               { type: [base_schema[:type], "null"] }.merge(base_schema.except(:type))
@@ -210,11 +235,11 @@ module DSPy
             end
           elsif non_nil_types.size == 1
             # Non-nilable single type union (shouldn't happen in practice)
-            self.type_to_json_schema(non_nil_types.first, visited)
+            type_to_json_schema_internal(non_nil_types.first, visited, definitions)
           elsif non_nil_types.size > 1
             # Handle complex unions with oneOf for better JSON schema compliance
             base_schema = {
-              oneOf: non_nil_types.map { |t| self.type_to_json_schema(t, visited) },
+              oneOf: non_nil_types.map { |t| type_to_json_schema_internal(t, visited, definitions) },
               description: "Union of multiple types"
             }
             if is_nilable
@@ -237,12 +262,31 @@ module DSPy
       end
 
       # Generate JSON schema for custom T::Struct classes
+      # For backward compatibility, this returns just the schema hash
       sig { params(struct_class: T.class_of(T::Struct), visited: T.nilable(T::Set[T.untyped])).returns(T::Hash[Symbol, T.untyped]) }
       def self.generate_struct_schema(struct_class, visited = nil)
         visited ||= Set.new
-        
+        generate_struct_schema_internal(struct_class, visited, {})
+      end
+
+      # Generate JSON schema with $defs tracking
+      # Returns a SchemaResult with schema and accumulated definitions
+      sig { params(struct_class: T.class_of(T::Struct), visited: T.nilable(T::Set[T.untyped]), definitions: T.nilable(T::Hash[String, T::Hash[Symbol, T.untyped]])).returns(SchemaResult) }
+      def self.generate_struct_schema_with_defs(struct_class, visited = nil, definitions = nil)
+        visited ||= Set.new
+        definitions ||= {}
+        schema = generate_struct_schema_internal(struct_class, visited, definitions)
+        SchemaResult.new(schema: schema, definitions: definitions)
+      end
+
+      # Internal implementation that tracks definitions for $defs
+      sig { params(struct_class: T.class_of(T::Struct), visited: T::Set[T.untyped], definitions: T::Hash[String, T::Hash[Symbol, T.untyped]]).returns(T::Hash[Symbol, T.untyped]) }
+      def self.generate_struct_schema_internal(struct_class, visited, definitions)
         return { type: "string", description: "Struct (schema introspection not available)" } unless struct_class.respond_to?(:props)
 
+        struct_name = struct_class.name || "Struct#{format('%x', struct_class.object_id)}"
+        simple_name = struct_name.split('::').last || struct_name
+
         # Add this struct to visited set to detect recursion
         visited.add(struct_class)
 
@@ -255,9 +299,6 @@ module DSPy
                                        "DSPy uses _type for automatic type detection in union types."
         end
 
-        struct_name = struct_class.name || "Struct#{format('%x', struct_class.object_id)}"
-        simple_name = struct_name.split('::').last || struct_name
-
         # Add automatic _type field for type detection
         properties[:_type] = {
           type: "string",
@@ -265,10 +306,20 @@ module DSPy
         }
         required << "_type"
 
+        # Get field descriptions if the struct supports them (via DSPy::Ext::StructDescriptions)
+        field_descs = struct_class.respond_to?(:field_descriptions) ? struct_class.field_descriptions : {}
+
         struct_class.props.each do |prop_name, prop_info|
           prop_type = prop_info[:type_object] || prop_info[:type]
-          properties[prop_name] = self.type_to_json_schema(prop_type, visited)
-          
+          prop_schema = type_to_json_schema_internal(prop_type, visited, definitions)
+
+          # Add field description if available
+          if field_descs[prop_name]
+            prop_schema[:description] = field_descs[prop_name]
+          end
+
+          properties[prop_name] = prop_schema
+
           # A field is required if it's not fully optional
           # fully_optional is true for nilable prop fields
           # immutable const fields are required unless nilable
@@ -280,12 +331,18 @@ module DSPy
         # Remove this struct from visited set after processing
         visited.delete(struct_class)
 
-        {
+        schema = {
           type: "object",
           properties: properties,
           required: required,
           description: "#{struct_name} struct"
         }
+
+        # Add this struct's schema to definitions for $defs
+        # This allows recursive references to be resolved
+        definitions[simple_name] = schema
+
+        schema
       end
 
       private

data/lib/dspy/signature.rb

@@ -174,6 +174,35 @@ module DSPy
         }
       end
 
+      # Returns output JSON schema with accumulated $defs for recursive types
+      # This is needed for providers like OpenAI and Gemini that require $defs at the root
+      sig { returns(DSPy::TypeSystem::SorbetJsonSchema::SchemaResult) }
+      def output_json_schema_with_defs
+        properties = {}
+        required = []
+        all_definitions = {}
+
+        @output_field_descriptors&.each do |name, descriptor|
+          result = DSPy::TypeSystem::SorbetJsonSchema.type_to_json_schema_with_defs(descriptor.type, nil, all_definitions)
+          schema = result.schema
+          schema[:description] = descriptor.description if descriptor.description
+          properties[name] = schema
+          required << name.to_s unless descriptor.has_default
+        end
+
+        final_schema = {
+          "$schema": "http://json-schema.org/draft-06/schema#",
+          type: "object",
+          properties: properties,
+          required: required
+        }
+
+        DSPy::TypeSystem::SorbetJsonSchema::SchemaResult.new(
+          schema: final_schema,
+          definitions: all_definitions
+        )
+      end
+
       sig { returns(T.nilable(T.class_of(T::Struct))) }
       def output_schema
         @output_struct_class

data/lib/dspy/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module DSPy
-  VERSION = "0.34.0"
+  VERSION = "0.34.1"
 end

data/lib/dspy.rb

@@ -5,6 +5,9 @@ require 'dry-configurable'
 require 'dry/logger'
 require 'securerandom'
 
+# Extensions to core classes (must be loaded early)
+require_relative 'dspy/ext/struct_descriptions'
+
 require_relative 'dspy/version'
 require_relative 'dspy/errors'
 require_relative 'dspy/type_serializer'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: dspy
 version: !ruby/object:Gem::Version
-  version: 0.34.0
+  version: 0.34.1
 platform: ruby
 authors:
 - Vicente Reig Rincón de Arellano
@@ -174,6 +174,7 @@ files:
 - lib/dspy/events/subscribers.rb
 - lib/dspy/events/types.rb
 - lib/dspy/example.rb
+- lib/dspy/ext/struct_descriptions.rb
 - lib/dspy/few_shot_example.rb
 - lib/dspy/field.rb
 - lib/dspy/image.rb