dspy 0.28.2 → 0.29.0

data/lib/gepa/proposer/reflective_mutation/reflective_mutation.rb

@@ -0,0 +1,188 @@
+# frozen_string_literal: true
+
+require 'sorbet-runtime'
+
+require_relative '../base'
+require_relative 'base'
+
+module GEPA
+  module Proposer
+    class ReflectiveMutationProposer
+      extend T::Sig
+      include ProposeNewCandidate
+
+      sig do
+        params(
+          logger: T.untyped,
+          trainset: T::Array[T.untyped],
+          adapter: T.untyped,
+          candidate_selector: T.untyped,
+          module_selector: T.untyped,
+          batch_sampler: T.untyped,
+          perfect_score: Float,
+          skip_perfect_score: T::Boolean,
+          experiment_tracker: T.untyped,
+          reflection_lm: T.nilable(T.proc.params(prompt: String).returns(String)),
+          telemetry: T.nilable(T.untyped)
+        ).void
+      end
+      def initialize(
+        logger:,
+        trainset:,
+        adapter:,
+        candidate_selector:,
+        module_selector:,
+        batch_sampler:,
+        perfect_score:,
+        skip_perfect_score:,
+        experiment_tracker:,
+        reflection_lm: nil,
+        telemetry: nil
+      )
+        @logger = logger
+        @trainset = trainset
+        @adapter = adapter
+        @candidate_selector = candidate_selector
+        @module_selector = module_selector
+        @batch_sampler = batch_sampler
+        @perfect_score = perfect_score
+        @skip_perfect_score = skip_perfect_score
+        @experiment_tracker = experiment_tracker
+        @reflection_lm = reflection_lm
+        @telemetry = telemetry || GEPA::Telemetry
+      end
+
+      sig { override.params(state: GEPA::Core::State).returns(T.nilable(CandidateProposal)) }
+      def propose(state)
+        iteration = state.i + 1
+
+        with_span('gepa.proposer.reflective_mutation.propose', iteration: iteration) do
+          proposal_for_iteration(state, iteration)
+        end
+      end
+
+      private
+
+      def proposal_for_iteration(state, iteration)
+        curr_prog_id = @candidate_selector.select_candidate_idx(state)
+        curr_prog = state.program_candidates[curr_prog_id]
+        ensure_trace_slot(state)
+        state.full_program_trace.last[:selected_program_candidate] = curr_prog_id
+
+        @logger.log("Iteration #{iteration}: Selected program #{curr_prog_id} score: #{state.per_program_tracked_scores[curr_prog_id]}")
+        @experiment_tracker.log_metrics({ iteration: iteration, selected_program_candidate: curr_prog_id }, step: iteration)
+
+        subsample_ids = @batch_sampler.next_minibatch_indices(@trainset.length, iteration - 1)
+        state.full_program_trace.last[:subsample_ids] = subsample_ids
+        minibatch = subsample_ids.map { |idx| @trainset[idx] }
+
+        eval_curr = with_span('gepa.proposer.evaluate_current', iteration: iteration) do
+          @adapter.evaluate(minibatch, curr_prog, capture_traces: true)
+        end
+
+        unless eval_curr.trajectories && !eval_curr.trajectories.empty?
+          @logger.log("Iteration #{iteration}: No trajectories captured. Skipping.")
+          return nil
+        end
+
+        state.total_num_evals += subsample_ids.length
+        state.full_program_trace.last[:subsample_scores] = eval_curr.scores
+
+        if @skip_perfect_score && eval_curr.scores.all? { |score| score >= @perfect_score }
+          @logger.log("Iteration #{iteration}: All subsample scores perfect. Skipping.")
+          return nil
+        end
+
+        @experiment_tracker.log_metrics({ subsample_score: eval_curr.scores.sum }, step: iteration)
+
+        predictor_names = @module_selector.select_modules(
+          state,
+          eval_curr.trajectories,
+          eval_curr.scores,
+          curr_prog_id,
+          curr_prog
+        )
+
+        reflective_dataset = nil
+        new_texts = nil
+
+        with_span('gepa.proposer.build_reflective_dataset', iteration: iteration) do
+          reflective_dataset = @adapter.make_reflective_dataset(curr_prog, eval_curr, predictor_names)
+        end
+
+        begin
+          new_texts = with_span('gepa.proposer.propose_texts', iteration: iteration) do
+            propose_new_texts(curr_prog, reflective_dataset, predictor_names)
+          end
+
+          new_texts.each do |name, text|
+            @logger.log("Iteration #{iteration}: Proposed new text for #{name}: #{text}")
+          end
+          @experiment_tracker.log_metrics(new_texts.transform_keys { |name| "new_instruction_#{name}" }, step: iteration)
+        rescue StandardError => e
+          @logger.log("Iteration #{iteration}: Exception during reflection/proposal: #{e}")
+          @logger.log(e.backtrace&.join("\n"))
+          return nil
+        end
+
+        new_candidate = curr_prog.dup
+        new_texts.each do |name, text|
+          raise ArgumentError, "Missing component #{name}" unless new_candidate.key?(name)
+          new_candidate[name] = text
+        end
+
+        eval_new = with_span('gepa.proposer.evaluate_new_candidate', iteration: iteration) do
+          @adapter.evaluate(minibatch, new_candidate, capture_traces: false)
+        end
+
+        state.total_num_evals += subsample_ids.length
+        state.full_program_trace.last[:new_subsample_scores] = eval_new.scores
+        @experiment_tracker.log_metrics({ new_subsample_score: eval_new.scores.sum }, step: iteration)
+
+        CandidateProposal.new(
+          candidate: new_candidate,
+          parent_program_ids: [curr_prog_id],
+          subsample_indices: subsample_ids,
+          subsample_scores_before: eval_curr.scores,
+          subsample_scores_after: eval_new.scores,
+          metadata: { iteration: iteration }
+        )
+      end
+
+      sig do
+        params(
+          candidate: T::Hash[String, String],
+          reflective_dataset: T::Hash[String, T::Array[T::Hash[String, T.untyped]]],
+          components_to_update: T::Array[String]
+        ).returns(T::Hash[String, String])
+      end
+      def propose_new_texts(candidate, reflective_dataset, components_to_update)
+        if @adapter.respond_to?(:propose_new_texts)
+          return @adapter.propose_new_texts(candidate, reflective_dataset, components_to_update)
+        end
+
+        raise ArgumentError, 'reflection_lm is required when adapter lacks propose_new_texts' unless @reflection_lm
+
+        components_to_update.each_with_object({}) do |name, acc|
+          signature_input = {
+            'current_instruction_doc' => candidate[name],
+            'dataset_with_feedback' => reflective_dataset.fetch(name)
+          }
+          acc[name] = GEPA::Strategies::InstructionProposalSignature.run(@reflection_lm, signature_input)['new_instruction']
+        end
+      end
+
+      sig { params(state: GEPA::Core::State).void }
+      def ensure_trace_slot(state)
+        state.full_program_trace << {} if state.full_program_trace.empty? || state.full_program_trace.last.nil?
+      end
+
+      sig do
+        params(operation: String, attrs: T::Hash[Symbol, T.untyped], block: T.proc.returns(T.untyped)).returns(T.untyped)
+      end
+      def with_span(operation, attrs = {}, &block)
+        @telemetry.with_span(operation, attrs, &block)
+      end
+    end
+  end
+end

data/lib/gepa/strategies/batch_sampler.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+require 'sorbet-runtime'
+
+module GEPA
+  module Strategies
+    class EpochShuffledBatchSampler
+      extend T::Sig
+
+      sig { params(minibatch_size: Integer, rng: T.nilable(Random), telemetry: T.nilable(T.untyped)).void }
+      def initialize(minibatch_size, rng: nil, telemetry: nil)
+        @minibatch_size = minibatch_size
+        @rng = rng || Random.new(0)
+        @telemetry = telemetry
+        @shuffled_ids = []
+        @epoch = -1
+        @id_freqs = Hash.new(0)
+      end
+
+      sig { params(trainset_size: Integer, iteration: Integer).returns(T::Array[Integer]) }
+      def next_minibatch_indices(trainset_size, iteration)
+        with_span(
+          'gepa.strategies.batch_sampler',
+          minibatch_size: @minibatch_size,
+          trainset_size: trainset_size,
+          iteration: iteration
+        ) do
+          ensure_epoch(trainset_size, iteration)
+          base_idx = (iteration * @minibatch_size) % @shuffled_ids.length
+          end_idx = base_idx + @minibatch_size
+          @shuffled_ids[base_idx...end_idx]
+        end
+      end
+
+      private
+
+      sig { returns(T.untyped) }
+      def telemetry
+        @telemetry || GEPA::Telemetry
+      end
+
+      sig { params(trainset_size: Integer, iteration: Integer).void }
+      def ensure_epoch(trainset_size, iteration)
+        update_shuffled(trainset_size) if @shuffled_ids.empty?
+
+        curr_epoch = if @epoch == -1
+          0
+        else
+          (iteration * @minibatch_size) / [@shuffled_ids.length, 1].max
+        end
+
+        return unless curr_epoch > @epoch
+
+        @epoch = curr_epoch
+        update_shuffled(trainset_size)
+      end
+
+      sig { params(trainset_size: Integer).void }
+      def update_shuffled(trainset_size)
+        @shuffled_ids = Array.new(trainset_size) { |idx| idx }
+        @shuffled_ids = @shuffled_ids.shuffle(random: @rng)
+
+        @shuffled_ids.each { |idx| @id_freqs[idx] += 1 }
+
+        remainder = trainset_size % @minibatch_size
+        num_to_pad = remainder.zero? ? 0 : (@minibatch_size - remainder)
+
+        num_to_pad.times do
+          least_used = @id_freqs.min_by { |_idx, count| count }&.first || 0
+          @shuffled_ids << least_used
+          @id_freqs[least_used] += 1
+        end
+
+        raise ArgumentError, 'minibatch size must be positive' if @minibatch_size <= 0
+        raise 'shuffled ids shorter than minibatch size' if @shuffled_ids.length < @minibatch_size
+        raise 'shuffled ids not aligned to minibatch size' unless (@shuffled_ids.length % @minibatch_size).zero?
+      end
+
+      sig do
+        params(
+          operation: String,
+          attrs: T::Hash[Symbol, T.untyped],
+          block: T.proc.returns(T.untyped)
+        ).returns(T.untyped)
+      end
+      def with_span(operation, attrs = {}, &block)
+        telemetry.with_span(operation, attrs, &block)
+      end
+    end
+  end
+end

data/lib/gepa/strategies/candidate_selector.rb

@@ -0,0 +1,97 @@
+# frozen_string_literal: true
+
+require 'sorbet-runtime'
+
+module GEPA
+  module Strategies
+    class ParetoCandidateSelector
+      extend T::Sig
+
+      sig { params(rng: T.nilable(Random), telemetry: T.nilable(T.untyped)).void }
+      def initialize(rng: nil, telemetry: nil)
+        @rng = rng || Random.new(0)
+        @telemetry = telemetry
+      end
+
+      sig { params(state: GEPA::Core::State).returns(Integer) }
+      def select_candidate_idx(state)
+        ensure_lengths!(state)
+        with_span('gepa.strategies.candidate_selector', strategy: 'pareto') do
+          scores = state.per_program_tracked_scores.each_with_index.to_h { |score, idx| [idx, score] }
+          GEPA::Utils::Pareto.select_program_candidate_from_pareto_front(
+            state.program_at_pareto_front_valset,
+            scores,
+            @rng
+          )
+        end
+      end
+
+      private
+
+      sig { params(state: GEPA::Core::State).void }
+      def ensure_lengths!(state)
+        return if state.per_program_tracked_scores.length == state.program_candidates.length
+
+        raise ArgumentError, 'per_program_tracked_scores and program_candidates length mismatch'
+      end
+
+      sig { returns(T.untyped) }
+      def telemetry
+        @telemetry || GEPA::Telemetry
+      end
+
+      sig do
+        params(
+          operation: String,
+          attrs: T::Hash[Symbol, T.untyped],
+          block: T.proc.returns(T.untyped)
+        ).returns(T.untyped)
+      end
+      def with_span(operation, attrs = {}, &block)
+        telemetry.with_span(operation, attrs, &block)
+      end
+    end
+
+    class CurrentBestCandidateSelector
+      extend T::Sig
+
+      sig { params(telemetry: T.nilable(T.untyped)).void }
+      def initialize(telemetry: nil)
+        @telemetry = telemetry
+      end
+
+      sig { params(state: GEPA::Core::State).returns(Integer) }
+      def select_candidate_idx(state)
+        ensure_lengths!(state)
+        with_span('gepa.strategies.candidate_selector', strategy: 'current_best') do
+          GEPA::Utils::Pareto.idxmax(state.per_program_tracked_scores)
+        end
+      end
+
+      private
+
+      sig { params(state: GEPA::Core::State).void }
+      def ensure_lengths!(state)
+        return if state.per_program_tracked_scores.length == state.program_candidates.length
+
+        raise ArgumentError, 'per_program_tracked_scores and program_candidates length mismatch'
+      end
+
+      sig { returns(T.untyped) }
+      def telemetry
+        @telemetry || GEPA::Telemetry
+      end
+
+      sig do
+        params(
+          operation: String,
+          attrs: T::Hash[Symbol, T.untyped],
+          block: T.proc.returns(T.untyped)
+        ).returns(T.untyped)
+      end
+      def with_span(operation, attrs = {}, &block)
+        telemetry.with_span(operation, attrs, &block)
+      end
+    end
+  end
+end

data/lib/gepa/strategies/component_selector.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require 'sorbet-runtime'
+
+module GEPA
+  module Strategies
+    class RoundRobinReflectionComponentSelector
+      extend T::Sig
+
+      sig { params(telemetry: T.nilable(T.untyped)).void }
+      def initialize(telemetry: nil)
+        @telemetry = telemetry
+      end
+
+      sig do
+        params(
+          state: GEPA::Core::State,
+          trajectories: T::Array[T.untyped],
+          subsample_scores: T::Array[Float],
+          candidate_idx: Integer,
+          candidate: T::Hash[String, String]
+        ).returns(T::Array[String])
+      end
+      def select_modules(state, trajectories, subsample_scores, candidate_idx, candidate)
+        with_span(
+          'gepa.strategies.component_selector',
+          strategy: 'round_robin',
+          candidate_idx: candidate_idx
+        ) do
+          predictor_id = state.named_predictor_id_to_update_next_for_program_candidate[candidate_idx]
+          state.named_predictor_id_to_update_next_for_program_candidate[candidate_idx] =
+            (predictor_id + 1) % state.list_of_named_predictors.length
+
+          [state.list_of_named_predictors[predictor_id]]
+        end
+      end
+
+      private
+
+      sig { returns(T.untyped) }
+      def telemetry
+        @telemetry || GEPA::Telemetry
+      end
+
+      sig do
+        params(
+          operation: String,
+          attrs: T::Hash[Symbol, T.untyped],
+          block: T.proc.returns(T.untyped)
+        ).returns(T.untyped)
+      end
+      def with_span(operation, attrs, &block)
+        telemetry.with_span(operation, attrs, &block)
+      end
+    end
+  end
+end

data/lib/gepa/strategies/instruction_proposal.rb

@@ -0,0 +1,120 @@
+# frozen_string_literal: true
+
+require 'sorbet-runtime'
+
+module GEPA
+  module Strategies
+    class InstructionProposalSignature
+      extend T::Sig
+
+      PROMPT_TEMPLATE = <<~PROMPT
+        I provided an assistant with the following instructions to perform a task for me:
+        ```
+        <curr_instructions>
+        ```
+
+        The following are examples of different task inputs provided to the assistant along with the assistant's response for each of them, and some feedback on how the assistant's response could be better:
+        ```
+        <inputs_outputs_feedback>
+        ```
+
+        Your task is to write a new instruction for the assistant.
+
+        Read the inputs carefully and identify the input format and infer detailed task description about the task I wish to solve with the assistant.
+
+        Read all the assistant responses and the corresponding feedback. Identify all niche and domain specific factual information about the task and include it in the instruction, as a lot of it may not be available to the assistant in the future. The assistant may have utilized a generalizable strategy to solve the task, if so, include that in the instruction as well.
+
+        Provide the new instructions within ``` blocks.
+      PROMPT
+
+      sig { returns(T::Array[String]) }
+      def self.input_keys
+        %w[current_instruction_doc dataset_with_feedback]
+      end
+
+      sig { returns(T::Array[String]) }
+      def self.output_keys
+        %w[new_instruction]
+      end
+
+      sig { params(input: T::Hash[String, T.untyped]).returns(String) }
+      def self.prompt_renderer(input)
+        prompt = PROMPT_TEMPLATE.dup
+        prompt = prompt.sub('<curr_instructions>', input.fetch('current_instruction_doc', ''))
+        prompt.sub('<inputs_outputs_feedback>', render_samples(input.fetch('dataset_with_feedback', [])))
+      end
+
+      sig { params(output: String).returns(T::Hash[String, String]) }
+      def self.output_extractor(output)
+        stripped = output.to_s.strip
+        return { 'new_instruction' => stripped } if stripped.count('```') < 2
+
+        first = stripped.index('```')
+        last = stripped.rindex('```')
+        if first.nil? || last.nil? || first == last
+          { 'new_instruction' => stripped.delete_prefix('```').delete_suffix('```').strip }
+        else
+          inner = stripped[(first + 3)...last].strip
+          { 'new_instruction' => inner.empty? ? stripped : inner }
+        end
+      end
+
+      sig do
+        params(
+          lm: T.untyped,
+          input_dict: T::Hash[String, T.untyped]
+        ).returns(T::Hash[String, String])
+      end
+      def self.run(lm, input_dict)
+        prompt = prompt_renderer(input_dict)
+        raw_output = if lm.respond_to?(:call)
+          lm.call(prompt)
+        else
+          response = lm.raw_chat([{ role: 'user', content: prompt }])
+          response.respond_to?(:content) ? response.content : response
+        end
+
+        output_extractor(raw_output.to_s)
+      end
+
+      class << self
+        extend T::Sig
+        private
+
+        sig { params(samples: T::Array[T.untyped]).returns(String) }
+        def render_samples(samples)
+          samples.each_with_index.map do |sample, index|
+            convert_sample_to_markdown(sample, index + 1)
+          end.join("\n\n")
+        end
+
+        sig { params(sample: T.untyped, index: Integer).returns(String) }
+        def convert_sample_to_markdown(sample, index)
+          return '' unless sample.is_a?(Hash)
+
+          sample.map do |key, value|
+            "## Example #{index}\n### #{key}\n#{render_value(value, 4)}"
+          end.join
+        end
+
+        sig { params(value: T.untyped, level: Integer).returns(String) }
+        def render_value(value, level)
+          case value
+          when Hash
+            value.map do |key, val|
+              heading = '#' * [level, 6].min
+              "#{heading} #{key}\n#{render_value(val, level + 1)}"
+            end.join
+          when Array
+            value.each_with_index.map do |item, idx|
+              heading = '#' * [level, 6].min
+              "#{heading} Item #{idx + 1}\n#{render_value(item, level + 1)}"
+            end.join
+          else
+            "#{value}\n\n"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/gepa/telemetry.rb

@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+require 'securerandom'
+require 'sorbet-runtime'
+require 'dspy'
+
+module GEPA
+  # Telemetry helpers for the GEPA optimizer.
+  #
+  # The helpers wrap DSPy context spans and structured logs so that the GEPA
+  # port can attach observability data consistently across the optimization
+  # lifecycle. They mirror the phases from the Python sequence diagrams:
+  #
+  # - `gepa.optimize` (API entry)
+  # - `gepa.state.initialize`
+  # - `gepa.engine.run` / `gepa.engine.iteration`
+  # - `gepa.proposer.*` (selection, evaluation, reflection, acceptance)
+  #
+  # Later phases of the port can depend on these helpers without reimplementing
+  # span naming or default attributes.
+  module Telemetry
+    extend T::Sig
+
+    DEFAULT_ATTRIBUTES = T.let({
+      optimizer: 'GEPA',
+      'gepa.instrumentation_version': 'phase0',
+      'langfuse.observation.type': 'span'
+    }.freeze, T::Hash[Symbol, T.untyped])
+
+    class Context < T::Struct
+      extend T::Sig
+
+      const :run_id, String
+      const :attributes, T::Hash[Symbol, T.untyped]
+
+      sig do
+        params(
+          operation: String,
+          metadata: T::Hash[T.any(String, Symbol), T.untyped],
+          block: T.proc.returns(T.untyped)
+        ).returns(T.untyped)
+      end
+      def with_span(operation, metadata = {}, &block)
+        Telemetry.with_span(operation, base_attributes.merge(Telemetry.send(:symbolize, metadata)), &block)
+      end
+
+      sig do
+        params(
+          event_name: String,
+          metadata: T::Hash[T.any(String, Symbol), T.untyped]
+        ).void
+      end
+      def emit(event_name, metadata = {})
+        Telemetry.emit(event_name, base_attributes.merge(Telemetry.send(:symbolize, metadata)))
+      end
+
+      private
+
+      sig { returns(T::Hash[Symbol, T.untyped]) }
+      def base_attributes
+        attributes.merge(run_id: run_id)
+      end
+    end
+
+    sig do
+      params(
+        additional_attributes: T::Hash[T.any(String, Symbol), T.untyped]
+      ).returns(Context)
+    end
+    def self.build_context(additional_attributes = {})
+      attributes = DEFAULT_ATTRIBUTES.merge(symbolize(additional_attributes.dup))
+      run_id = attributes.delete(:run_id) || SecureRandom.uuid
+
+      Context.new(run_id: run_id, attributes: attributes)
+    end
+
+    sig do
+      params(
+        operation: String,
+        attributes: T::Hash[T.any(String, Symbol), T.untyped],
+        block: T.proc.returns(T.untyped)
+      ).returns(T.untyped)
+    end
+    def self.with_span(operation, attributes = {}, &block)
+      operation_name = normalize_operation(operation)
+      span_attributes = DEFAULT_ATTRIBUTES.merge(symbolize(attributes))
+
+      DSPy::Context.with_span(operation: operation_name, **span_attributes, &block)
+    end
+
+    sig do
+      params(
+        event_name: String,
+        attributes: T::Hash[T.any(String, Symbol), T.untyped]
+      ).void
+    end
+    def self.emit(event_name, attributes = {})
+      payload = DEFAULT_ATTRIBUTES.merge(symbolize(attributes))
+      DSPy.log("gepa.#{event_name}", **payload)
+    end
+
+    sig { params(operation: String).returns(String) }
+    def self.normalize_operation(operation)
+      return operation if operation.start_with?('gepa.')
+
+      "gepa.#{operation}"
+    end
+    private_class_method :normalize_operation
+
+    sig do
+      params(
+        attributes: T::Hash[T.any(String, Symbol), T.untyped]
+      ).returns(T::Hash[Symbol, T.untyped])
+    end
+    def self.symbolize(attributes)
+      attributes.each_with_object({}) do |(key, value), acc|
+        acc[key.to_sym] = value
+      end
+    end
+    private_class_method :symbolize
+  end
+end