desiru 0.1.0 → 0.2.0

data/lib/desiru/models/open_router.rb

@@ -0,0 +1,161 @@
+# frozen_string_literal: true
+
+require 'open_router'
+
+module Desiru
+  module Models
+    # OpenRouter model adapter - provides access to multiple models through a single API
+    class OpenRouter < Base
+      DEFAULT_MODEL = 'anthropic/claude-3-haiku'
+
+      def initialize(config = {})
+        super
+        @api_key = config[:api_key] || ENV.fetch('OPENROUTER_API_KEY', nil)
+        raise ArgumentError, 'OpenRouter API key is required' unless @api_key
+
+        # Configure OpenRouter client
+        ::OpenRouter.configure do |c|
+          c.access_token = @api_key
+          c.site_name = config[:site_name] || 'Desiru'
+          c.site_url = config[:site_url] || 'https://github.com/obie/desiru'
+        end
+
+        @client = ::OpenRouter::Client.new
+        @models_cache = nil
+        @models_fetched_at = nil
+      end
+
+      def models
+        # Cache models for 1 hour
+        fetch_models if @models_cache.nil? || @models_fetched_at.nil? || (Time.now - @models_fetched_at) > 3600
+        @models_cache
+      end
+
+      protected
+
+      def perform_completion(messages, options)
+        model = options[:model] || @config[:model] || DEFAULT_MODEL
+        temperature = options[:temperature] || @config[:temperature] || 0.7
+        max_tokens = options[:max_tokens] || @config[:max_tokens] || 4096
+
+        # Prepare request parameters
+        params = {
+          model: model,
+          messages: messages,
+          temperature: temperature,
+          max_tokens: max_tokens
+        }
+
+        # Add provider-specific options if needed
+        params[:provider] = options[:provider] if options[:provider]
+
+        # Add response format if specified
+        params[:response_format] = options[:response_format] if options[:response_format]
+
+        # Add tools if provided (for models that support function calling)
+        if options[:tools]
+          params[:tools] = options[:tools]
+          params[:tool_choice] = options[:tool_choice] if options[:tool_choice]
+        end
+
+        # Make API call
+        response = @client.complete(params)
+
+        # Format response
+        format_response(response, model)
+      rescue StandardError => e
+        handle_api_error(e)
+      end
+
+      def stream_complete(prompt, **options, &block)
+        messages = prepare_messages(prompt, options[:messages])
+        model = options[:model] || @config[:model] || DEFAULT_MODEL
+        temperature = options[:temperature] || @config[:temperature] || 0.7
+        max_tokens = options[:max_tokens] || @config[:max_tokens] || 4096
+
+        # Prepare streaming request
+        params = {
+          model: model,
+          messages: messages,
+          temperature: temperature,
+          max_tokens: max_tokens,
+          stream: true
+        }
+
+        # Stream response
+        @client.complete(params) do |chunk|
+          if chunk.dig('choices', 0, 'delta', 'content')
+            content = chunk.dig('choices', 0, 'delta', 'content')
+            block.call(content) if block_given?
+          end
+        end
+      rescue StandardError => e
+        handle_api_error(e)
+      end
+
+      private
+
+      def fetch_models
+        # OpenRouter provides models at https://openrouter.ai/api/v1/models
+        response = @client.models
+
+        @models_cache = {}
+        response['data'].each do |model|
+          @models_cache[model['id']] = {
+            name: model['name'] || model['id'],
+            context_length: model['context_length'],
+            pricing: model['pricing'],
+            top_provider: model['top_provider']
+          }
+        end
+
+        @models_fetched_at = Time.now
+        @models_cache
+      rescue StandardError => e
+        Desiru.logger.warn("Failed to fetch OpenRouter models: #{e.message}")
+        # Fallback to commonly used models
+        @models_cache = {
+          'anthropic/claude-3-haiku' => { name: 'Claude 3 Haiku' },
+          'anthropic/claude-3-sonnet' => { name: 'Claude 3 Sonnet' },
+          'openai/gpt-4o-mini' => { name: 'GPT-4o Mini' },
+          'openai/gpt-4o' => { name: 'GPT-4o' },
+          'google/gemini-pro' => { name: 'Gemini Pro' }
+        }
+        @models_fetched_at = Time.now
+        @models_cache
+      end
+
+      def format_response(response, model)
+        # OpenRouter uses OpenAI-compatible response format
+        content = response.dig('choices', 0, 'message', 'content') || ''
+        usage = response['usage'] || {}
+
+        {
+          content: content,
+          raw: response,
+          model: model,
+          usage: {
+            prompt_tokens: usage['prompt_tokens'] || 0,
+            completion_tokens: usage['completion_tokens'] || 0,
+            total_tokens: usage['total_tokens'] || 0
+          }
+        }
+      end
+
+      def handle_api_error(error)
+        case error
+        when ::Faraday::UnauthorizedError
+          raise AuthenticationError, 'Invalid OpenRouter API key'
+        when ::Faraday::BadRequestError
+          raise InvalidRequestError, "Invalid request: #{error.message}"
+        when ::Faraday::TooManyRequestsError
+          raise RateLimitError, 'OpenRouter API rate limit exceeded'
+        when ::Faraday::PaymentRequiredError
+          raise APIError, 'OpenRouter payment required - check your account balance'
+        else
+          raise APIError, "OpenRouter API error: #{error.message}"
+        end
+      end
+    end
+  end
+end

data/lib/desiru/module.rb

@@ -1,13 +1,16 @@
 # frozen_string_literal: true
 
-# require_relative 'async_capable'
+require_relative 'async_capable'
+require_relative 'assertions'
+require_relative 'core/traceable'
 
 module Desiru
   # Base class for all Desiru modules
   # Implements the core module pattern with service-oriented design
   class Module
     extend Forwardable
-    # include AsyncCapable
+    include AsyncCapable
+    prepend Core::Traceable
 
     attr_reader :signature, :model, :config, :demos, :metadata
 
@@ -40,23 +43,23 @@ module Desiru
 
       begin
         # Validate inputs first, then coerce
-        signature.validate_inputs(inputs)
+        signature.valid_inputs?(inputs)
         coerced_inputs = signature.coerce_inputs(inputs)
 
         # Execute the module logic
         result = forward(**coerced_inputs)
 
         # Validate outputs first, then coerce
-        signature.validate_outputs(result)
+        signature.valid_outputs?(result)
         coerced_outputs = signature.coerce_outputs(result)
 
         # Return result object
         ModuleResult.new(coerced_outputs, metadata: execution_metadata)
       rescue StandardError => e
-        if config[:retry_on_failure] && @retry_count < Desiru.configuration.max_retries
+        if should_retry?(e)
           @retry_count += 1
-          Desiru.configuration.logger&.warn("Retrying module execution (attempt #{@retry_count}/#{Desiru.configuration.max_retries})")
-          sleep(Desiru.configuration.retry_delay)
+          log_retry(e)
+          sleep(retry_delay_for(e))
           retry
         else
           handle_error(e)
@@ -110,6 +113,44 @@ module Desiru
 
     private
 
+    def should_retry?(error)
+      return false unless config[:retry_on_failure]
+
+      # Handle assertion errors specifically
+      return error.retriable? && @retry_count < max_retries_for(error) if error.is_a?(Assertions::AssertionError)
+
+      # Default retry logic for other errors
+      @retry_count < Desiru.configuration.max_retries
+    end
+
+    def max_retries_for(error)
+      if error.is_a?(Assertions::AssertionError)
+        Assertions.configuration.max_assertion_retries
+      else
+        Desiru.configuration.max_retries
+      end
+    end
+
+    def retry_delay_for(error)
+      if error.is_a?(Assertions::AssertionError)
+        Assertions.configuration.assertion_retry_delay
+      else
+        Desiru.configuration.retry_delay
+      end
+    end
+
+    def log_retry(error)
+      if error.is_a?(Assertions::AssertionError)
+        Desiru.configuration.logger&.warn(
+          "[ASSERTION RETRY] #{error.message} (attempt #{@retry_count}/#{max_retries_for(error)})"
+        )
+      else
+        Desiru.configuration.logger&.warn(
+          "Retrying module execution (attempt #{@retry_count}/#{Desiru.configuration.max_retries})"
+        )
+      end
+    end
+
     def validate_model!
       return if model.nil? # Will use default
 
@@ -133,8 +174,19 @@ module Desiru
     end
 
     def handle_error(error)
-      Desiru.configuration.logger&.error("Module execution failed: #{error.message}")
-      raise ModuleError, "Module execution failed: #{error.message}"
+      if error.is_a?(Assertions::AssertionError)
+        # Update the assertion error with module context
+        error.instance_variable_set(:@module_name, self.class.name)
+        error.instance_variable_set(:@retry_count, @retry_count)
+
+        Desiru.configuration.logger&.error(
+          "[ASSERTION FAILED] #{error.message} in #{self.class.name} after #{@retry_count} retries"
+        )
+        raise error
+      else
+        Desiru.configuration.logger&.error("Module execution failed: #{error.message}")
+        raise ModuleError, "Module execution failed: #{error.message}"
+      end
     end
   end
 
@@ -166,6 +218,12 @@ module Desiru
       end
     end
 
+    def key?(key)
+      @data.key?(key.to_sym) || @data.key?(key.to_s)
+    end
+
+    alias has_key? key?
+
     def method_missing(method_name, *args, &)
       method_str = method_name.to_s
       if method_str.end_with?('?')

data/lib/desiru/modules/best_of_n.rb

@@ -0,0 +1,306 @@
+# frozen_string_literal: true
+
+module Desiru
+  module Modules
+    # BestOfN module that samples N outputs from a predictor and selects the best one
+    # based on configurable criteria (confidence, consistency, or external validation)
+    class BestOfN < Desiru::Module
+      SELECTION_CRITERIA = %i[confidence consistency llm_judge custom].freeze
+
+      DEFAULT_SIGNATURE = 'question: string -> answer: string'
+
+      def initialize(signature = nil, model: nil, **kwargs)
+        # Extract our specific options before passing to parent
+        @n_samples = kwargs.delete(:n_samples) || 5
+        @selection_criterion = validate_criterion(kwargs.delete(:selection_criterion) || :consistency)
+        @temperature = kwargs.delete(:temperature) || 0.7
+        @custom_selector = kwargs.delete(:custom_selector) # Proc that takes array of results
+        @base_module = kwargs.delete(:base_module) || Modules::Predict
+        @include_metadata = kwargs.delete(:include_metadata) || false
+
+        # Use default signature if none provided
+        signature ||= DEFAULT_SIGNATURE
+
+        # Pass remaining kwargs to parent (config, demos, metadata)
+        super
+      end
+
+      def forward(**inputs)
+        # Generate N samples
+        samples = generate_samples(inputs)
+
+        # Select the best sample based on criterion
+        best_sample = select_best(samples, inputs)
+
+        # Include metadata if requested
+        if @include_metadata || signature.output_fields.key?(:selection_metadata)
+          best_sample[:selection_metadata] = build_metadata(samples, best_sample)
+        end
+
+        # Clean up internal fields
+        best_sample.delete(:_confidence_score)
+
+        best_sample
+      rescue ArgumentError => e
+        # Re-raise ArgumentError for missing custom selector
+        raise e
+      rescue StandardError => e
+        Desiru.logger.error("BestOfN error: #{e.message}")
+        # Fallback to single sample
+        fallback_sample(inputs)
+      end
+
+      private
+
+      def validate_criterion(criterion)
+        unless SELECTION_CRITERIA.include?(criterion)
+          raise ArgumentError, "Invalid selection criterion: #{criterion}. " \
+                               "Must be one of: #{SELECTION_CRITERIA.join(', ')}"
+        end
+        criterion
+      end
+
+      def generate_samples(inputs)
+        samples = []
+
+        # Create module instance for generation
+        generator = if @base_module.is_a?(Class)
+                      @base_module.new(signature, model: model)
+                    else
+                      @base_module
+                    end
+
+        @n_samples.times do |i|
+          # Add variation seed to inputs for diversity
+          sample_inputs = inputs.merge(_sample_index: i)
+
+          # Use higher temperature for diversity
+          original_temp = model.instance_variable_get(:@temperature) if model.respond_to?(:instance_variable_get)
+
+          begin
+            # Temporarily set temperature if possible
+            model.temperature = @temperature if model.respond_to?(:temperature=)
+
+            # Generate sample
+            sample = if generator.respond_to?(:forward)
+                       generator.forward(**sample_inputs)
+                     else
+                       generator.call(**sample_inputs)
+                     end
+
+            # Remove the sample index from results
+            sample.delete(:_sample_index)
+            samples << sample
+          ensure
+            # Restore original temperature
+            model.temperature = original_temp if model.respond_to?(:temperature=) && original_temp
+          end
+        end
+
+        samples
+      end
+
+      def select_best(samples, inputs)
+        case @selection_criterion
+        when :confidence
+          select_by_confidence(samples)
+        when :consistency
+          select_by_consistency(samples)
+        when :llm_judge
+          select_by_llm_judge(samples, inputs)
+        when :custom
+          select_by_custom(samples)
+        else
+          samples.first # Fallback
+        end
+      end
+
+      def select_by_confidence(samples)
+        # Ask model to rate confidence for each sample
+        samples_with_scores = samples.map do |sample|
+          confidence = calculate_confidence(sample)
+          sample.merge(_confidence_score: confidence)
+        end
+
+        # Return sample with highest confidence (keep score for metadata)
+        samples_with_scores.max_by { |s| s[:_confidence_score] }
+      end
+
+      def calculate_confidence(sample)
+        # Build confidence prompt
+        prompt = "Rate the confidence (0-100) for this response:\n\n"
+
+        sample.each do |key, value|
+          next if key.to_s.start_with?('_')
+
+          prompt += "#{key}: #{value}\n"
+        end
+
+        prompt += "\nProvide only a number between 0 and 100:"
+
+        response = model.complete(
+          messages: [{ role: 'user', content: prompt }],
+          temperature: 0.1
+        )
+
+        # Extract confidence score
+        score = response[:content].scan(/\d+/).first&.to_i || 50
+        score.clamp(0, 100)
+      end
+
+      def select_by_consistency(samples)
+        # Group samples by their main output values
+        output_groups = Hash.new { |h, k| h[k] = [] }
+
+        # Find the main output field (first non-metadata field)
+        main_field = signature.output_fields.keys.find do |k|
+          !k.to_s.start_with?('_') && k.to_s != 'selection_metadata'
+        end
+
+        return samples.first unless main_field
+
+        # Convert to symbol to match sample keys
+        field_sym = main_field.to_sym
+
+        # Group samples by their main output
+        samples.each do |sample|
+          if sample[field_sym]
+            key = normalize_output(sample[field_sym])
+            output_groups[key] << sample
+          end
+        end
+
+        # Select the most consistent group
+        largest_group = output_groups.values.max_by(&:length)
+
+        # From the largest group, select the "centroid" - the one most similar to others
+        select_centroid(largest_group)
+      end
+
+      def normalize_output(value)
+        case value
+        when String
+          value.downcase.strip.gsub(/[[:punct:]]/, '')
+        when Numeric
+          value.round(2)
+        when Array
+          value.map { |v| normalize_output(v) }.sort
+        when Hash
+          value.transform_values { |v| normalize_output(v) }
+        else
+          value.to_s
+        end
+      end
+
+      def select_centroid(group)
+        return group.first if group.length == 1
+
+        # For now, return the middle element (could be improved with similarity metrics)
+        group[group.length / 2]
+      end
+
+      def select_by_llm_judge(samples, inputs)
+        # Build judge prompt
+        judge_prompt = "Given the following input and multiple response options, " \
+                       "select the best response:\n\n"
+
+        # Add original inputs
+        judge_prompt += "Input:\n"
+        inputs.each do |key, value|
+          judge_prompt += "  #{key}: #{value}\n"
+        end
+
+        # Add all samples
+        judge_prompt += "\nResponse Options:\n"
+        samples.each_with_index do |sample, i|
+          judge_prompt += "\n--- Option #{i + 1} ---\n"
+          sample.each do |key, value|
+            next if key.to_s.start_with?('_')
+
+            judge_prompt += "#{key}: #{value}\n"
+          end
+        end
+
+        judge_prompt += "\nSelect the best option (1-#{samples.length}) and briefly explain why:"
+
+        response = model.complete(
+          messages: [{ role: 'user', content: judge_prompt }],
+          temperature: 0.1
+        )
+
+        # Extract selected index
+        selection_match = response[:content].match(/option\s*#?(\d+)/i)
+        selected_index = if selection_match
+                           selection_match[1].to_i - 1
+                         else
+                           0
+                         end
+
+        selected_index = selected_index.clamp(0, samples.length - 1)
+        samples[selected_index]
+      end
+
+      def select_by_custom(samples)
+        unless @custom_selector.respond_to?(:call)
+          raise ArgumentError, "Custom selector must be provided when using :custom criterion"
+        end
+
+        @custom_selector.call(samples) || samples.first
+      end
+
+      def build_metadata(samples, selected)
+        metadata = {
+          total_samples: samples.length,
+          selection_criterion: @selection_criterion,
+          temperature: @temperature
+        }
+
+        # Add criterion-specific metadata
+        case @selection_criterion
+        when :consistency
+          # Count how many samples agree with the selected one
+          main_field = signature.output_fields.keys.find do |k|
+            !k.to_s.start_with?('_') && k.to_s != 'selection_metadata'
+          end
+
+          if main_field
+            # Convert to symbol to match sample keys
+            field_sym = main_field.to_sym
+            if selected[field_sym]
+              selected_value = normalize_output(selected[field_sym])
+              agreement_count = samples.count do |s|
+                normalize_output(s[field_sym]) == selected_value
+              end
+              metadata[:agreement_rate] = agreement_count.to_f / samples.length
+            end
+          end
+        when :confidence
+          # Include confidence scores if available
+          metadata[:selected_confidence] = selected[:_confidence_score] if selected[:_confidence_score]
+        end
+
+        metadata
+      end
+
+      def fallback_sample(inputs)
+        # Generate a single sample as fallback
+        generator = if @base_module.is_a?(Class)
+                      @base_module.new(signature, model: model)
+                    else
+                      @base_module
+                    end
+
+        if generator.respond_to?(:forward)
+          generator.forward(**inputs)
+        else
+          generator.call(**inputs)
+        end
+      end
+    end
+  end
+end
+
+# Register in the main module namespace for convenience
+module Desiru
+  BestOfN = Modules::BestOfN
+end

data/lib/desiru/modules/chain_of_thought.rb

@@ -21,9 +21,9 @@ module Desiru
 
           Before providing the final answer, you must show your reasoning process. Think through the problem step by step.
 
-          Format your response as:
-          reasoning: [Your step-by-step thought process]
-          [output fields]: [Your final answers]
+          Always format your response with each field on its own line like this:
+          reasoning: Your step-by-step thought process here
+          #{@original_signature.output_fields.keys.map { |field| "#{field}: Your #{field} here" }.join("\n")}
 
           #{format_descriptions}
         PROMPT

data/lib/desiru/modules/majority.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+module Desiru
+  module Modules
+    # Function-style module for majority voting
+    # Returns the most common response from multiple completions
+    def self.majority(module_instance, **inputs)
+      raise ArgumentError, "First argument must be a Desiru module instance" unless module_instance.respond_to?(:call)
+
+      # Number of completions to generate
+      num_completions = inputs.delete(:num_completions) || 5
+
+      # Generate multiple completions
+      results = []
+      num_completions.times do
+        result = module_instance.call(**inputs)
+        results << result
+      end
+
+      # Find the majority answer
+      # For simplicity, we'll compare the first output field
+      output_fields = module_instance.signature.output_fields.keys
+      main_field = output_fields.first
+
+      # Count occurrences of each answer
+      answer_counts = Hash.new(0)
+      answer_to_result = {}
+
+      results.each do |result|
+        answer = result[main_field]
+        answer_counts[answer] += 1
+        answer_to_result[answer] ||= result
+      end
+
+      # Return the result with the most common answer
+      majority_answer = answer_counts.max_by { |_, count| count }&.first
+      winning_result = answer_to_result[majority_answer] || results.first
+
+      # Add voting metadata if requested
+      if output_fields.include?(:voting_data)
+        winning_result[:voting_data] = {
+          votes: answer_counts,
+          num_completions: num_completions,
+          consensus_rate: answer_counts[majority_answer].to_f / num_completions
+        }
+      end
+
+      winning_result
+    end
+  end
+end