desiru 0.1.0

data/lib/desiru/graphql/schema_generator.rb

@@ -0,0 +1,301 @@
+# frozen_string_literal: true
+
+require 'graphql'
+require_relative 'data_loader'
+
+module Desiru
+  module GraphQL
+    # Generates GraphQL schemas from Desiru signatures
+    class SchemaGenerator
+      attr_reader :signatures, :modules, :data_loader
+
+      def initialize
+        @signatures = {}
+        @modules = {}
+        @type_cache = {}
+        @schema_class = nil
+        @data_loader = DataLoader.new
+      end
+
+      # Register a signature with a name for GraphQL query/mutation
+      def register_signature(name, signature)
+        @signatures[name] = signature
+      end
+
+      # Register a module instance to handle a specific operation
+      def register_module(name, module_instance)
+        @modules[name] = module_instance
+        # Auto-register signature if module has one
+        @signatures[name] ||= module_instance.signature if module_instance.respond_to?(:signature)
+      end
+
+      # Register multiple modules at once
+      def register_modules(modules_hash)
+        modules_hash.each { |name, mod| register_module(name, mod) }
+      end
+
+      # Generate a GraphQL schema from registered signatures
+      def generate_schema
+        return @schema_class if @schema_class && @signatures.empty?
+
+        query_class = build_query_type
+
+        @schema_class = Class.new(::GraphQL::Schema) do
+          query(query_class) if query_class
+        end
+
+        @schema_class
+      end
+
+      private
+
+      def build_query_type
+        return nil if @signatures.empty?
+
+        query_fields = build_query_fields
+
+        Class.new(::GraphQL::Schema::Object) do
+          graphql_name 'Query'
+          description 'Desiru query operations'
+
+          query_fields.each do |field_name, field_def|
+            # Create a resolver class for each field
+            resolver_class = Class.new(::GraphQL::Schema::Resolver) do
+              # Set the return type
+              type field_def[:type], null: false
+
+              # Add arguments
+              field_def[:arguments].each do |arg_name, arg_def|
+                argument arg_name, arg_def[:type], required: arg_def[:required]
+              end
+
+              # Define resolve method
+              define_method :resolve do |**args|
+                field_def[:resolver].call(args)
+              end
+            end
+
+            # Add field with resolver
+            field field_name, resolver: resolver_class, description: field_def[:description]
+          end
+        end
+      end
+
+      def build_query_fields
+        fields = {}
+
+        @signatures.each do |operation_name, signature|
+          output_type = build_output_type(signature)
+
+          arguments = {}
+          signature.input_fields.each do |field_name, field|
+            arguments[camelcase_field_name(field_name)] = {
+              type: graphql_type_for_field(field),
+              required: !field.optional
+            }
+          end
+
+          fields[operation_name.to_sym] = {
+            type: output_type,
+            description: "Generated from signature: #{signature.raw_signature}",
+            arguments: arguments,
+            resolver: ->(args) { execute_signature(operation_name, signature, args) }
+          }
+        end
+
+        fields
+      end
+
+      def build_mutation_type
+        # Mutations could be added for signatures that modify state
+        nil
+      end
+
+      def build_output_type(signature)
+        type_name = "Output#{signature.object_id}"
+        return @type_cache[type_name] if @type_cache[type_name]
+
+        output_field_defs = {}
+        signature.output_fields.each do |field_name, field|
+          output_field_defs[camelcase_field_name(field_name)] = {
+            type: graphql_type_for_field(field),
+            null: field.optional,
+            description: field.description
+          }
+        end
+
+        output_type = Class.new(::GraphQL::Schema::Object) do
+          graphql_name type_name
+          description 'Generated output type'
+
+          output_field_defs.each do |field_name, field_def|
+            field field_name, field_def[:type],
+                  null: field_def[:null],
+                  description: field_def[:description]
+          end
+        end
+
+        @type_cache[type_name] = output_type
+      end
+
+      def graphql_type_for_field(field)
+        base_type = case field.type
+                    when :string
+                      ::GraphQL::Types::String
+                    when :int, :integer
+                      ::GraphQL::Types::Int
+                    when :float
+                      ::GraphQL::Types::Float
+                    when :bool, :boolean
+                      ::GraphQL::Types::Boolean
+                    when :list
+                      # Handle list types
+                      element_type = graphql_type_for_element(field.element_type)
+                      [element_type]
+                    when :literal
+                      # Create enum type for literal values
+                      create_enum_type(field)
+                    else
+                      ::GraphQL::Types::String
+                    end
+
+        if field.optional
+          base_type
+        else
+          # Arrays are already wrapped, so handle them differently
+          base_type.is_a?(Array) ? [base_type.first, { null: false }] : base_type.to_non_null_type
+        end
+      end
+
+      def graphql_type_for_element(element_type)
+        case element_type
+        when Hash
+          # Handle typed arrays like List[Literal['yes', 'no']]
+          if element_type[:type] == :literal
+            create_enum_type_from_values(element_type[:values])
+          else
+            ::GraphQL::Types::String
+          end
+        else
+          # Simple types
+          case element_type
+          when :string then ::GraphQL::Types::String
+          when :int, :integer then ::GraphQL::Types::Int
+          when :float then ::GraphQL::Types::Float
+          when :bool, :boolean then ::GraphQL::Types::Boolean
+          else ::GraphQL::Types::String
+          end
+        end
+      end
+
+      def create_enum_type(field)
+        enum_name = "#{field.name.to_s.capitalize}Enum"
+        return @type_cache[enum_name] if @type_cache[enum_name]
+
+        # Extract literal values from the field's validator
+        values = extract_literal_values(field)
+
+        enum_type = Class.new(::GraphQL::Schema::Enum) do
+          graphql_name enum_name
+          description "Enum for #{field.name}"
+
+          values.each do |val|
+            value val.upcase.gsub(/[^A-Z0-9_]/, '_'), value: val
+          end
+        end
+
+        @type_cache[enum_name] = enum_type
+      end
+
+      def create_enum_type_from_values(values)
+        enum_name = "Literal#{values.map(&:capitalize).join}Enum"
+        return @type_cache[enum_name] if @type_cache[enum_name]
+
+        enum_type = Class.new(::GraphQL::Schema::Enum) do
+          graphql_name enum_name
+
+          values.each do |val|
+            value val.upcase.gsub(/[^A-Z0-9_]/, '_'), value: val
+          end
+        end
+
+        @type_cache[enum_name] = enum_type
+      end
+
+      def extract_literal_values(field)
+        # Try to extract values from the field's validator
+        if field.respond_to?(:validator) && field.validator.respond_to?(:instance_variable_get)
+          field.validator.instance_variable_get(:@values) || []
+        elsif field.respond_to?(:element_type) && field.element_type.is_a?(Hash)
+          field.element_type[:values] || []
+        else
+          []
+        end
+      end
+
+      def execute_signature(operation_name, signature, args)
+        # Convert GraphQL arguments from camelCase to snake_case
+        inputs = transform_graphql_args(args)
+
+        # Check if we have a registered module for this operation
+        if @modules[operation_name]
+          # Use DataLoader for batch optimization
+          loader = @data_loader.for(@modules[operation_name].class)
+          promise = loader.load(inputs)
+
+          # In a real GraphQL implementation, this would be handled by the executor
+          # For now, we'll resolve immediately
+          result = promise.value
+
+          # Transform module result to GraphQL response format
+          transform_module_result(result)
+        else
+          # Fallback: create a module instance on the fly
+          module_class = infer_module_class(signature)
+          module_instance = module_class.new(signature)
+          result = module_instance.call(inputs)
+          transform_module_result(result)
+        end
+      end
+
+      def transform_graphql_args(args)
+        # Convert camelCase keys to snake_case, but handle single-word keys correctly
+        args.transform_keys do |key|
+          key_str = key.to_s
+          # Only convert if there's actually a capital letter after the first character
+          if key_str =~ /[a-z][A-Z]/
+            key_str.gsub(/([A-Z])/, '_\1').downcase.to_sym
+          else
+            key_str.downcase.to_sym
+          end
+        end
+      end
+
+      def transform_module_result(result)
+        # Convert ModuleResult to hash with camelCase keys
+        if result.respond_to?(:to_h)
+          result.to_h.transform_keys { |key| camelcase_field_name(key) }
+        else
+          result
+        end
+      end
+
+      def infer_module_class(signature)
+        # Infer the appropriate module class based on signature characteristics
+        if signature.raw_signature.include?('reasoning')
+          Desiru::Modules::ChainOfThought
+        else
+          Desiru::Modules::Predict
+        end
+      end
+
+      def camelcase_field_name(field_name)
+        # Convert snake_case to camelCase for GraphQL conventions
+        # Remove trailing '?' for optional fields
+        clean_name = field_name.to_s.gsub('?', '')
+        parts = clean_name.split('_')
+        parts[0] + parts[1..-1].map(&:capitalize).join
+      end
+    end
+  end
+end

data/lib/desiru/jobs/async_predict.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require_relative 'base'
+
+module Desiru
+  module Jobs
+    class AsyncPredict < Base
+      sidekiq_options queue: 'critical'
+
+      def perform(job_id, module_class_name, signature_str, inputs, options = {})
+        update_status(job_id, 'running', message: 'Initializing module')
+        
+        module_class = Object.const_get(module_class_name)
+
+        # Extract module initialization parameters
+        model_class = options.delete('model_class')
+        model_config = options.delete('model_config') || {}
+        config = options.delete('config') || {}
+        demos = options.delete('demos') || []
+
+        # Initialize model if provided
+        model = (Object.const_get(model_class).new(**model_config) if model_class && model_config)
+
+        module_instance = module_class.new(
+          signature_str,
+          model: model,
+          config: config,
+          demos: demos
+        )
+
+        update_status(job_id, 'running', progress: 50, message: 'Processing request')
+        result = module_instance.call(**inputs)
+
+        update_status(job_id, 'completed', progress: 100, message: 'Request completed successfully')
+        store_result(job_id, {
+                       success: true,
+                       result: result.to_h,
+                       completed_at: Time.now.iso8601
+                     })
+      rescue StandardError => e
+        update_status(job_id, 'failed', message: "Error: #{e.message}")
+        store_result(job_id, {
+                       success: false,
+                       error: e.message,
+                       error_class: e.class.name,
+                       completed_at: Time.now.iso8601
+                     })
+        raise
+      end
+    end
+  end
+end

data/lib/desiru/jobs/base.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require 'sidekiq'
+require 'redis'
+require 'json'
+
+module Desiru
+  module Jobs
+    class Base
+      include Sidekiq::Job
+
+      sidekiq_options retry: 3, dead: true
+
+      def perform(*)
+        raise NotImplementedError, "#{self.class} must implement #perform"
+      end
+
+      protected
+
+      def store_result(job_id, result, ttl: 3600)
+        redis.setex(result_key(job_id), ttl, result.to_json)
+      end
+
+      def fetch_result(job_id)
+        result = redis.get(result_key(job_id))
+        result ? JSON.parse(result, symbolize_names: true) : nil
+      end
+
+      def result_key(job_id)
+        "desiru:results:#{job_id}"
+      end
+
+      def redis
+        @redis ||= Redis.new(url: Desiru.configuration.redis_url || ENV.fetch('REDIS_URL', nil))
+      end
+
+      def update_status(job_id, status, progress: nil, message: nil)
+        status_data = {
+          status: status,
+          updated_at: Time.now.iso8601
+        }
+        status_data[:progress] = progress if progress
+        status_data[:message] = message if message
+
+        redis.setex(status_key(job_id), 86_400, status_data.to_json)
+      end
+
+      def status_key(job_id)
+        "desiru:status:#{job_id}"
+      end
+    end
+  end
+end

data/lib/desiru/jobs/batch_processor.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require_relative 'base'
+
+module Desiru
+  module Jobs
+    class BatchProcessor < Base
+      sidekiq_options queue: 'default'
+
+      def perform(batch_id, module_class_name, signature_str, inputs_array, options = {})
+        total_items = inputs_array.size
+        update_status(batch_id, 'running', progress: 0, message: "Processing #{total_items} items")
+        
+        module_class = Object.const_get(module_class_name)
+
+        # Extract module initialization parameters
+        model_class = options.delete('model_class')
+        model_config = options.delete('model_config') || {}
+        config = options.delete('config') || {}
+        demos = options.delete('demos') || []
+
+        # Initialize model if provided
+        model = (Object.const_get(model_class).new(**model_config) if model_class && model_config)
+
+        module_instance = module_class.new(
+          signature_str,
+          model: model,
+          config: config,
+          demos: demos
+        )
+
+        results = []
+        errors = []
+
+        inputs_array.each_with_index do |inputs, index|
+          progress = ((index + 1).to_f / total_items * 100).round
+          update_status(batch_id, 'running', progress: progress, 
+                       message: "Processing item #{index + 1} of #{total_items}")
+          
+          result = module_instance.call(**inputs)
+          results << {
+            index: index,
+            success: true,
+            result: result.to_h
+          }
+        rescue StandardError => e
+          errors << {
+            index: index,
+            success: false,
+            error: e.message,
+            error_class: e.class.name
+          }
+        end
+
+        final_status = errors.empty? ? 'completed' : 'completed_with_errors'
+        update_status(batch_id, final_status, progress: 100, 
+                     message: "Processed #{results.size} successfully, #{errors.size} failed")
+
+        store_result(batch_id, {
+                       success: errors.empty?,
+                       total: inputs_array.size,
+                       successful: results.size,
+                       failed: errors.size,
+                       results: results,
+                       errors: errors,
+                       completed_at: Time.now.iso8601
+                     }, ttl: 7200) # 2 hours TTL for batch results
+      end
+    end
+  end
+end

data/lib/desiru/jobs/optimizer_job.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require_relative 'base'
+
+module Desiru
+  module Jobs
+    class OptimizerJob < Base
+      sidekiq_options queue: 'low', retry: 1
+
+      def perform(job_id, optimizer_class_name, program_class_name, trainset, optimizer_options = {})
+        optimizer_class = Object.const_get(optimizer_class_name)
+        program_class = Object.const_get(program_class_name)
+        optimizer = optimizer_class.new(**optimizer_options)
+        program = program_class.new
+
+        # Store initial status
+        update_status(job_id, 'running', progress: 0, message: 'Starting optimization')
+
+        # Compile the program with progress tracking
+        optimized_program = optimizer.compile(program, trainset: trainset) do |progress|
+          update_status(job_id, 'running', progress: progress, message: "Optimizing... #{progress}% complete")
+        end
+
+        # Store the optimized program configuration
+        store_result(job_id, {
+                       success: true,
+                       optimized_config: optimized_program.to_config,
+                       metrics: optimizer.final_metrics,
+                       completed_at: Time.now.iso8601
+                     }, ttl: 86_400) # 24 hours TTL
+
+        update_status(job_id, 'completed', progress: 100, message: 'Optimization completed successfully')
+      rescue StandardError => e
+        store_result(job_id, {
+                       success: false,
+                       error: e.message,
+                       error_class: e.class.name,
+                       completed_at: Time.now.iso8601
+                     })
+        update_status(job_id, 'failed', message: "Optimization failed: #{e.message}")
+        raise
+      end
+    end
+  end
+end

data/lib/desiru/models/base.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+module Desiru
+  module Models
+    # Base adapter class for language model integrations
+    # Defines the interface all model adapters must implement
+    class Base
+      attr_reader :config, :client
+
+      def initialize(config = {})
+        @config = default_config.merge(config)
+        @client = build_client
+        @request_count = 0
+        @token_count = 0
+
+        validate_config!
+      end
+
+      # Main interface method - must be implemented by subclasses
+      def complete(prompt, **options)
+        raise NotImplementedError, 'Subclasses must implement #complete'
+      end
+
+      # Stream completion - optional implementation
+      def stream_complete(prompt, **options, &)
+        raise NotImplementedError, "Streaming not supported by #{self.class.name}"
+      end
+
+      # Get available models
+      def models
+        raise NotImplementedError, 'Subclasses must implement #models'
+      end
+
+      # Health check
+      def healthy?
+        models
+        true
+      rescue StandardError
+        false
+      end
+
+      # Usage statistics
+      def stats
+        {
+          request_count: @request_count,
+          token_count: @token_count,
+          model: config[:model]
+        }
+      end
+
+      def reset_stats
+        @request_count = 0
+        @token_count = 0
+      end
+
+      protected
+
+      def default_config
+        {
+          model: nil,
+          temperature: 0.7,
+          max_tokens: 1000,
+          timeout: 30,
+          retry_on_failure: true,
+          max_retries: 3
+        }
+      end
+
+      def build_client
+        # Override in subclasses to build the actual client
+        nil
+      end
+
+      def validate_config!
+        # Override in subclasses for specific validation
+      end
+
+      def increment_stats(tokens_used = 0)
+        @request_count += 1
+        @token_count += tokens_used
+      end
+
+      # Common error handling
+      def with_retry(max_attempts = nil)
+        max_attempts ||= config[:max_retries]
+        attempts = 0
+
+        begin
+          attempts += 1
+          yield
+        rescue StandardError => e
+          raise unless attempts < max_attempts && retryable_error?(e)
+
+          sleep(retry_delay(attempts))
+          retry
+        end
+      end
+
+      def retryable_error?(error)
+        # Override in subclasses for specific error types
+        error.message.include?('timeout') || error.message.include?('rate limit')
+      end
+
+      def retry_delay(attempt)
+        # Exponential backoff with jitter
+        base_delay = 2**attempt
+        jitter = rand(0..1.0)
+        base_delay + jitter
+      end
+    end
+  end
+end