dspy 0.24.1 → 0.25.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '0946116ac08ee09e62d204db418f2d45f62eb4d4b1eff8306de1780a8cdfba8f'
-  data.tar.gz: a294c49d86d084940738ebb39da6c7d3b8fef15064ff2c2cb15c07a90acdec8f
+  metadata.gz: d45d1ee46193e0bad5339bb7a05732797fa4c2dcc7fafcbaef0bee1aa7ecd591
+  data.tar.gz: 5ca7952a3119f1177ff034c55819e266a443d6fa48c975af15195f937560c10e
 SHA512:
-  metadata.gz: 91141a65593592604c301f3c51f48fac2c29b654c871c3155a055202d25efdcf7511ff4fda0372021f686b864f1e078e89fed27d245300f20966de62e3e295c5
-  data.tar.gz: e4885370fde056ff4dda5464901f0f3e460bbf0cf2985e079088adf67d091718fe72e8ee0efc34999b021bb3e238649a782f12c3fec1162a8b6b13576bd25dc0
+  metadata.gz: 0e29265f2bf13028265591281536baa229cab475dabaf0318045d40d4568eb4329b23b07c6f8b70c83588fb53e5059cb854ab8e68b5716e78f5368adad7a02bb
+  data.tar.gz: 714a2864f33cf2f22e27e9962368c277f8f684b22e751fb25fc03c14db3e42c06e95ec035fe99c7fba5295f98440895831ab3db6c45fd8859a450f2a4fbe6b7d

data/README.md

@@ -74,6 +74,7 @@ puts result.confidence   # => 0.85
 - **Typed Examples** - Type-safe training data with automatic validation
 - **Evaluation Framework** - Advanced metrics beyond simple accuracy with error-resilient pipelines
 - **MIPROv2 Optimization** - Automatic prompt optimization with storage and persistence
+- **GEPA Optimization** - Genetic-Pareto optimization for multi-objective prompt improvement
 
 **Production Features:**
 - **Reliable JSON Extraction** - Native OpenAI structured outputs, Anthropic extraction patterns, and automatic strategy selection with fallback
@@ -128,6 +129,7 @@ For LLMs and AI assistants working with DSPy.rb:
 - **[Evaluation Framework](docs/src/optimization/evaluation.md)** - Advanced metrics beyond simple accuracy
 - **[Prompt Optimization](docs/src/optimization/prompt-optimization.md)** - Manipulate prompts as objects
 - **[MIPROv2 Optimizer](docs/src/optimization/miprov2.md)** - Automatic optimization algorithms
+- **[GEPA Optimizer](docs/src/optimization/gepa.md)** - Genetic-Pareto optimization for multi-objective prompt optimization
 
 ### Production Features
 - **[Storage System](docs/src/production/storage.md)** - Persistence and optimization result storage
@@ -191,7 +193,7 @@ DSPy.rb has rapidly evolved from experimental to production-ready:
 - ✅ **Optimization Framework** - MIPROv2 algorithm with storage & persistence
 
 ### Recent Advances  
-- ✅ **Enhanced Langfuse Integration (v0.24.1)** - Comprehensive OpenTelemetry span reporting with proper input/output, hierarchical nesting, accurate timing, and observation types
+- ✅ **Enhanced Langfuse Integration (v0.25.0)** - Comprehensive OpenTelemetry span reporting with proper input/output, hierarchical nesting, accurate timing, and observation types
 - ✅ **Comprehensive Multimodal Framework** - Complete image analysis with `DSPy::Image`, type-safe bounding boxes, vision model integration
 - ✅ **Advanced Type System** - `T::Enum` integration, union types for agentic workflows, complex type coercion
 - ✅ **Production-Ready Evaluation** - Multi-factor metrics beyond accuracy, error-resilient evaluation pipelines

data/lib/dspy/chain_of_thought.rb

@@ -3,6 +3,7 @@
 
 require 'sorbet-runtime'
 require_relative 'predict'
+require_relative 'utils/serialization'
 require_relative 'signature'
 require_relative 'mixins/struct_builder'
 
@@ -87,10 +88,11 @@ module DSPy
     def forward_untyped(**input_values)
       # Wrap in chain-specific span tracking (overrides parent's span attributes)
       DSPy::Context.with_span(
-        operation: "ChainOfThought.forward",
-        'langfuse.observation.type' => 'chain',
+        operation: "#{self.class.name}.forward",
+        'langfuse.observation.type' => 'span',  # Use 'span' for proper timing
         'langfuse.observation.input' => input_values.to_json,
         'dspy.module' => 'ChainOfThought',
+        'dspy.module_type' => 'chain_of_thought',  # Semantic identifier
         'dspy.signature' => @original_signature.name
       ) do |span|
         # Call parent prediction logic (which will create its own nested span)
@@ -106,7 +108,7 @@ module DSPy
             prediction_result.respond_to?(:to_h) ? prediction_result.to_h : prediction_result.to_s
           end
           
-          span.set_attribute('langfuse.observation.output', output_with_reasoning.to_json)
+          span.set_attribute('langfuse.observation.output', DSPy::Utils::Serialization.to_json(output_with_reasoning))
           
           # Add reasoning metrics
           if prediction_result.respond_to?(:reasoning) && prediction_result.reasoning

data/lib/dspy/context.rb

@@ -6,10 +6,29 @@ module DSPy
   class Context
     class << self
       def current
-        Thread.current[:dspy_context] ||= {
-          trace_id: SecureRandom.uuid,
-          span_stack: []
-        }
+        # Check if we're in an async context (fiber created by async gem)
+        if in_async_context?
+          # Use Fiber storage for async contexts to enable inheritance
+          # Inherit from Thread.current if Fiber storage is not set
+          Fiber[:dspy_context] ||= Thread.current[:dspy_context] || {
+            trace_id: SecureRandom.uuid,
+            span_stack: []
+          }
+          
+          # Return Fiber storage in async contexts
+          Fiber[:dspy_context]
+        else
+          # Use Thread.current for regular synchronous contexts
+          Thread.current[:dspy_context] ||= {
+            trace_id: SecureRandom.uuid,
+            span_stack: []
+          }
+          
+          # Also sync to Fiber storage so async contexts can inherit it
+          Fiber[:dspy_context] = Thread.current[:dspy_context]
+          
+          Thread.current[:dspy_context]
+        end
       end
       
       def with_span(operation:, **attributes)
@@ -27,7 +46,7 @@ module DSPy
         }
         
         # Log span start with proper hierarchy (internal logging only)
-        DSPy.log('span.start', **span_attributes)
+        DSPy.log('span.start', **span_attributes) if DSPy::Observability.enabled?
         
         # Push to stack for child spans tracking
         current[:span_stack].push(span_id)
@@ -43,12 +62,25 @@ module DSPy
               span_attributes['langfuse.trace.name'] = operation
             end
             
+            # Record start time for explicit duration tracking
+            otel_start_time = Time.now
+            
             DSPy::Observability.tracer.in_span(
               operation,
               attributes: span_attributes,
               kind: :internal
             ) do |span|
-              yield(span)
+              result = yield(span)
+              
+              # Add explicit timing information to help Langfuse
+              if span
+                duration_ms = ((Time.now - otel_start_time) * 1000).round(3)
+                span.set_attribute('duration.ms', duration_ms)
+                span.set_attribute('langfuse.observation.startTime', otel_start_time.iso8601(3))
+                span.set_attribute('langfuse.observation.endTime', Time.now.iso8601(3))
+              end
+              
+              result
             end
           else
             yield(nil)
@@ -58,17 +90,29 @@ module DSPy
           current[:span_stack].pop
           
           # Log span end with duration (internal logging only)
-          duration_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time) * 1000).round(2)
-          DSPy.log('span.end',
-            trace_id: current[:trace_id],
-            span_id: span_id,
-            duration_ms: duration_ms
-          )
+          if DSPy::Observability.enabled?
+            duration_ms = ((Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time) * 1000).round(2)
+            DSPy.log('span.end',
+              trace_id: current[:trace_id],
+              span_id: span_id,
+              duration_ms: duration_ms
+            )
+          end
         end
       end
       
       def clear!
         Thread.current[:dspy_context] = nil
+        Fiber[:dspy_context] = nil
+      end
+      
+      private
+      
+      # Check if we're running in an async context
+      def in_async_context?
+        defined?(Async::Task) && Async::Task.current?
+      rescue
+        false
       end
     end
   end

data/lib/dspy/lm/adapters/openai/schema_converter.rb

@@ -38,12 +38,12 @@ module DSPy
             # Get the output JSON schema from the signature class
             output_schema = signature_class.output_json_schema
             
-            # Build the complete schema
+            # Build the complete schema with OpenAI-specific modifications
             dspy_schema = {
               "$schema": "http://json-schema.org/draft-06/schema#",
               type: "object",
               properties: output_schema[:properties] || {},
-              required: output_schema[:required] || []
+              required: openai_required_fields(signature_class, output_schema)
             }
 
             # Generate a schema name if not provided
@@ -52,9 +52,10 @@ module DSPy
             # Remove the $schema field as OpenAI doesn't use it
             openai_schema = dspy_schema.except(:$schema)
 
-            # Add additionalProperties: false for strict mode
+            # Add additionalProperties: false for strict mode and fix nested struct schemas
             if strict
               openai_schema = add_additional_properties_recursively(openai_schema)
+              openai_schema = fix_nested_struct_required_fields(openai_schema)
             end
 
             # Wrap in OpenAI's required format
@@ -120,6 +121,65 @@ module DSPy
 
           private
 
+          # OpenAI structured outputs requires ALL properties to be in the required array
+          # For T.nilable fields without defaults, we warn the user and mark as required
+          sig { params(signature_class: T.class_of(DSPy::Signature), output_schema: T::Hash[Symbol, T.untyped]).returns(T::Array[String]) }
+          def self.openai_required_fields(signature_class, output_schema)
+            all_properties = output_schema[:properties]&.keys || []
+            original_required = output_schema[:required] || []
+            
+            # For OpenAI structured outputs, we need ALL properties to be required
+            # but warn about T.nilable fields without defaults
+            field_descriptors = signature_class.instance_variable_get(:@output_field_descriptors) || {}
+            
+            all_properties.each do |property_name|
+              descriptor = field_descriptors[property_name.to_sym]
+              
+              # If field is not originally required and doesn't have a default
+              if !original_required.include?(property_name.to_s) && descriptor && !descriptor.has_default
+                DSPy.logger.warn(
+                  "OpenAI structured outputs: T.nilable field '#{property_name}' without default will be marked as required. " \
+                  "Consider adding a default value or using a different provider for optional fields."
+                )
+              end
+            end
+            
+            # Return all properties as required (OpenAI requirement)
+            all_properties.map(&:to_s)
+          end
+
+          # Fix nested struct schemas to include all properties in required array (OpenAI requirement)
+          sig { params(schema: T::Hash[Symbol, T.untyped]).returns(T::Hash[Symbol, T.untyped]) }
+          def self.fix_nested_struct_required_fields(schema)
+            return schema unless schema.is_a?(Hash)
+            
+            result = schema.dup
+            
+            # If this is an object with properties, make all properties required
+            if result[:type] == "object" && result[:properties].is_a?(Hash)
+              all_property_names = result[:properties].keys.map(&:to_s)
+              result[:required] = all_property_names unless result[:required] == all_property_names
+            end
+            
+            # Process nested objects recursively
+            if result[:properties].is_a?(Hash)
+              result[:properties] = result[:properties].transform_values do |prop|
+                if prop.is_a?(Hash)
+                  processed = fix_nested_struct_required_fields(prop)
+                  # Handle arrays with object items
+                  if processed[:type] == "array" && processed[:items].is_a?(Hash)
+                    processed[:items] = fix_nested_struct_required_fields(processed[:items])
+                  end
+                  processed
+                else
+                  prop
+                end
+              end
+            end
+            
+            result
+          end
+
           sig { params(schema: T::Hash[Symbol, T.untyped]).returns(T::Hash[Symbol, T.untyped]) }
           def self.add_additional_properties_recursively(schema)
             return schema unless schema.is_a?(Hash)

data/lib/dspy/lm/retry_handler.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "sorbet-runtime"
+require "async"
 
 module DSPy
   class LM
@@ -28,6 +29,11 @@ module DSPy
           .returns(T.type_parameter(:T))
       end
       def with_retry(initial_strategy, &block)
+        # Skip retries entirely if disabled
+        unless DSPy.config.structured_outputs.retry_enabled
+          return yield(initial_strategy)
+        end
+        
         strategies = build_fallback_chain(initial_strategy)
         last_error = nil
 
@@ -62,7 +68,7 @@ module DSPy
                 "Retrying #{strategy.name} after error (attempt #{retry_count}/#{max_retries_for_strategy(strategy)}): #{e.message}"
               )
               
-              sleep(backoff_time) if backoff_time > 0
+              Async::Task.current.sleep(backoff_time) if backoff_time > 0
               retry
             else
               DSPy.logger.info("Max retries reached for #{strategy.name}, trying next strategy")
@@ -107,8 +113,6 @@ module DSPy
       # Calculate exponential backoff with jitter
       sig { params(attempt: Integer).returns(Float) }
       def calculate_backoff(attempt)
-        return 0.0 if DSPy.config.test_mode # No sleep in tests
-        
         base_delay = BACKOFF_BASE * (2 ** (attempt - 1))
         jitter = rand * 0.1 * base_delay
         

data/lib/dspy/lm.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'sorbet-runtime'
+require 'async'
 
 # Load adapter infrastructure
 require_relative 'lm/errors'
@@ -41,20 +42,22 @@ module DSPy
     end
 
     def chat(inference_module, input_values, &block)
-      signature_class = inference_module.signature_class
-      
-      # Build messages from inference module
-      messages = build_messages(inference_module, input_values)
-      
-      # Execute with instrumentation
-      response = instrument_lm_request(messages, signature_class.name) do
-        chat_with_strategy(messages, signature_class, &block)
+      Sync do
+        signature_class = inference_module.signature_class
+        
+        # Build messages from inference module
+        messages = build_messages(inference_module, input_values)
+        
+        # Execute with instrumentation
+        response = instrument_lm_request(messages, signature_class.name) do
+          chat_with_strategy(messages, signature_class, &block)
+        end
+        
+        # Parse response (no longer needs separate instrumentation)
+        parsed_result = parse_response(response, input_values, signature_class)
+        
+        parsed_result
       end
-      
-      # Parse response (no longer needs separate instrumentation)
-      parsed_result = parse_response(response, input_values, signature_class)
-      
-      parsed_result
     end
 
     def raw_chat(messages = nil, &block)

data/lib/dspy/observability/async_span_processor.rb

@@ -0,0 +1,274 @@
+# frozen_string_literal: true
+
+require 'async'
+require 'async/queue'
+require 'async/barrier'
+require 'opentelemetry/sdk'
+require 'opentelemetry/sdk/trace/export'
+
+module DSPy
+  class Observability
+    # AsyncSpanProcessor provides truly non-blocking span export using Async gem.
+    # Spans are queued and exported using async tasks with fiber-based concurrency.
+    # Implements the same interface as OpenTelemetry::SDK::Trace::Export::BatchSpanProcessor
+    class AsyncSpanProcessor
+      # Default configuration values
+      DEFAULT_QUEUE_SIZE = 1000
+      DEFAULT_EXPORT_INTERVAL = 1.0  # seconds
+      DEFAULT_EXPORT_BATCH_SIZE = 100
+      DEFAULT_SHUTDOWN_TIMEOUT = 10.0  # seconds
+      DEFAULT_MAX_RETRIES = 3
+
+      def initialize(
+        exporter,
+        queue_size: DEFAULT_QUEUE_SIZE,
+        export_interval: DEFAULT_EXPORT_INTERVAL,
+        export_batch_size: DEFAULT_EXPORT_BATCH_SIZE,
+        shutdown_timeout: DEFAULT_SHUTDOWN_TIMEOUT,
+        max_retries: DEFAULT_MAX_RETRIES
+      )
+        @exporter = exporter
+        @queue_size = queue_size
+        @export_interval = export_interval
+        @export_batch_size = export_batch_size
+        @shutdown_timeout = shutdown_timeout
+        @max_retries = max_retries
+
+        # Use thread-safe queue for cross-fiber communication
+        @queue = Thread::Queue.new
+        @barrier = Async::Barrier.new
+        @shutdown_requested = false
+        @export_task = nil
+
+        start_export_task
+      end
+
+      def on_start(span, parent_context)
+        # Non-blocking - no operation needed on span start
+      end
+
+      def on_finish(span)
+        # Only process sampled spans to match BatchSpanProcessor behavior
+        return unless span.context.trace_flags.sampled?
+
+        # Non-blocking enqueue with overflow protection
+        # Note: on_finish is only called for already ended spans
+        begin
+          # Check queue size (non-blocking)
+          if @queue.size >= @queue_size
+            # Drop oldest span
+            begin
+              dropped_span = @queue.pop(true) # non-blocking pop
+              DSPy.log('observability.span_dropped',
+                       reason: 'queue_full',
+                       queue_size: @queue_size)
+            rescue ThreadError
+              # Queue was empty, continue
+            end
+          end
+
+          @queue.push(span)
+          
+          # Log span queuing activity
+          DSPy.log('observability.span_queued', queue_size: @queue.size)
+
+          # Trigger immediate export if batch size reached
+          trigger_export_if_batch_full
+        rescue => e
+          DSPy.log('observability.enqueue_error', error: e.message)
+        end
+      end
+
+      def shutdown(timeout: nil)
+        timeout ||= @shutdown_timeout
+        @shutdown_requested = true
+
+        begin
+          # Export any remaining spans
+          export_remaining_spans
+
+          # Shutdown exporter
+          @exporter.shutdown(timeout: timeout)
+
+          OpenTelemetry::SDK::Trace::Export::SUCCESS
+        rescue => e
+          DSPy.log('observability.shutdown_error', error: e.message, class: e.class.name)
+          OpenTelemetry::SDK::Trace::Export::FAILURE
+        end
+      end
+
+      def force_flush(timeout: nil)
+        return OpenTelemetry::SDK::Trace::Export::SUCCESS if @queue.empty?
+
+        export_remaining_spans
+      end
+
+      private
+
+      def start_export_task
+        return if @export_interval <= 0 # Disable timer for testing
+
+        # Start timer-based export task in background
+        Thread.new do
+          loop do
+            break if @shutdown_requested
+
+            sleep(@export_interval)
+
+            # Export queued spans in sync block
+            unless @queue.empty?
+              Sync do
+                export_queued_spans
+              end
+            end
+          end
+        rescue => e
+          DSPy.log('observability.export_task_error', error: e.message, class: e.class.name)
+        end
+      end
+
+      def trigger_export_if_batch_full
+        return if @queue.size < @export_batch_size
+
+        # Trigger immediate export in background
+        Thread.new do
+          Sync do
+            export_queued_spans
+          end
+        rescue => e
+          DSPy.log('observability.batch_export_error', error: e.message)
+        end
+      end
+
+      def export_remaining_spans
+        spans = []
+
+        # Drain entire queue
+        until @queue.empty?
+          begin
+            spans << @queue.pop(true) # non-blocking pop
+          rescue ThreadError
+            break
+          end
+        end
+
+        return OpenTelemetry::SDK::Trace::Export::SUCCESS if spans.empty?
+
+        export_spans_with_retry(spans)
+      end
+
+      def export_queued_spans
+        spans = []
+
+        # Collect up to batch size
+        @export_batch_size.times do
+          begin
+            spans << @queue.pop(true) # non-blocking pop
+          rescue ThreadError
+            break
+          end
+        end
+
+        return if spans.empty?
+
+        # Export using async I/O
+        Sync do
+          export_spans_with_retry_async(spans)
+        end
+      end
+
+      def export_spans_with_retry(spans)
+        retries = 0
+
+        # Convert spans to SpanData objects (required by OTLP exporter)
+        span_data_batch = spans.map(&:to_span_data)
+        
+        # Log export attempt
+        DSPy.log('observability.export_attempt',
+                 spans_count: span_data_batch.size,
+                 batch_size: span_data_batch.size)
+
+        loop do
+          result = @exporter.export(span_data_batch, timeout: @shutdown_timeout)
+
+          case result
+          when OpenTelemetry::SDK::Trace::Export::SUCCESS
+            DSPy.log('observability.export_success',
+                     spans_count: span_data_batch.size,
+                     export_result: 'SUCCESS')
+            return result
+          when OpenTelemetry::SDK::Trace::Export::FAILURE
+            retries += 1
+            if retries <= @max_retries
+              backoff_seconds = 0.1 * (2 ** retries)
+              DSPy.log('observability.export_retry',
+                       attempt: retries,
+                       spans_count: span_data_batch.size,
+                       backoff_seconds: backoff_seconds)
+              # Exponential backoff
+              sleep(backoff_seconds)
+              next
+            else
+              DSPy.log('observability.export_failed',
+                       spans_count: span_data_batch.size,
+                       retries: retries)
+              return result
+            end
+          else
+            return result
+          end
+        end
+      rescue => e
+        DSPy.log('observability.export_error', error: e.message, class: e.class.name)
+        OpenTelemetry::SDK::Trace::Export::FAILURE
+      end
+
+      def export_spans_with_retry_async(spans)
+        retries = 0
+
+        # Convert spans to SpanData objects (required by OTLP exporter)
+        span_data_batch = spans.map(&:to_span_data)
+        
+        # Log export attempt
+        DSPy.log('observability.export_attempt',
+                 spans_count: span_data_batch.size,
+                 batch_size: span_data_batch.size)
+
+        loop do
+          # Use current async task for potentially non-blocking export
+          result = @exporter.export(span_data_batch, timeout: @shutdown_timeout)
+
+          case result
+          when OpenTelemetry::SDK::Trace::Export::SUCCESS
+            DSPy.log('observability.export_success',
+                     spans_count: span_data_batch.size,
+                     export_result: 'SUCCESS')
+            return result
+          when OpenTelemetry::SDK::Trace::Export::FAILURE
+            retries += 1
+            if retries <= @max_retries
+              backoff_seconds = 0.1 * (2 ** retries)
+              DSPy.log('observability.export_retry',
+                       attempt: retries,
+                       spans_count: span_data_batch.size,
+                       backoff_seconds: backoff_seconds)
+              # Async sleep for exponential backoff
+              Async::Task.current.sleep(backoff_seconds)
+              next
+            else
+              DSPy.log('observability.export_failed',
+                       spans_count: span_data_batch.size,
+                       retries: retries)
+              return result
+            end
+          else
+            return result
+          end
+        end
+      rescue => e
+        DSPy.log('observability.export_error', error: e.message, class: e.class.name)
+        OpenTelemetry::SDK::Trace::Export::FAILURE
+      end
+    end
+  end
+end

data/lib/dspy/observability.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'base64'
+require_relative 'observability/async_span_processor'
 
 module DSPy
   class Observability
@@ -35,18 +36,26 @@ module DSPy
             config.service_name = 'dspy-ruby'
             config.service_version = DSPy::VERSION
             
-            # Add OTLP exporter for Langfuse
+            # Add OTLP exporter for Langfuse using AsyncSpanProcessor
+            exporter = OpenTelemetry::Exporter::OTLP::Exporter.new(
+              endpoint: @endpoint,
+              headers: {
+                'Authorization' => "Basic #{auth_string}",
+                'Content-Type' => 'application/x-protobuf'
+              },
+              compression: 'gzip'
+            )
+            
+            # Configure AsyncSpanProcessor with environment variables
+            async_config = {
+              queue_size: (ENV['DSPY_TELEMETRY_QUEUE_SIZE'] || AsyncSpanProcessor::DEFAULT_QUEUE_SIZE).to_i,
+              export_interval: (ENV['DSPY_TELEMETRY_EXPORT_INTERVAL'] || AsyncSpanProcessor::DEFAULT_EXPORT_INTERVAL).to_f,
+              export_batch_size: (ENV['DSPY_TELEMETRY_BATCH_SIZE'] || AsyncSpanProcessor::DEFAULT_EXPORT_BATCH_SIZE).to_i,
+              shutdown_timeout: (ENV['DSPY_TELEMETRY_SHUTDOWN_TIMEOUT'] || AsyncSpanProcessor::DEFAULT_SHUTDOWN_TIMEOUT).to_f
+            }
+            
             config.add_span_processor(
-              OpenTelemetry::SDK::Trace::Export::BatchSpanProcessor.new(
-                OpenTelemetry::Exporter::OTLP::Exporter.new(
-                  endpoint: @endpoint,
-                  headers: {
-                    'Authorization' => "Basic #{auth_string}",
-                    'Content-Type' => 'application/x-protobuf'
-                  },
-                  compression: 'gzip'
-                )
-              )
+              AsyncSpanProcessor.new(exporter, **async_config)
             )
             
             # Add resource attributes
@@ -103,6 +112,15 @@ module DSPy
         DSPy.log('observability.span_finish_error', error: e.message)
       end
 
+      def flush!
+        return unless enabled?
+        
+        # Force flush any pending spans
+        OpenTelemetry.tracer_provider.force_flush
+      rescue StandardError => e
+        DSPy.log('observability.flush_error', error: e.message)
+      end
+
       def reset!
         @enabled = false
         @tracer = nil