desiru 0.1.0 → 0.1.1

data/lib/desiru/persistence/setup.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require 'sequel'
+
+module Desiru
+  module Persistence
+    # Setup and initialization for persistence layer
+    module Setup
+      @initialized = false
+
+      def self.initialized?
+        @initialized
+      end
+
+      def self.initialize!(db_connection)
+        # Always reinitialize if the database connection has changed
+        return if @initialized && Sequel::Model.db == db_connection
+
+        # Ensure we have a valid connection
+        raise 'Database connection required' unless db_connection
+
+        # Create the base model with a specific database
+        Sequel::Model.db = db_connection
+
+        # Define the base model class
+        base_class = Class.new(Sequel::Model) do
+          # This is an abstract model - no table
+          def self.inherited(subclass)
+            super
+            subclass.plugin :timestamps, update_on_create: true
+            subclass.plugin :json_serializer
+            subclass.plugin :validation_helpers
+          end
+
+          # Helper to create JSON columns that work across databases
+          def self.json_column(name)
+            case db.adapter_scheme
+            when :postgres
+              # PostgreSQL has native JSON support
+              nil # No special handling needed
+            when :sqlite, :mysql, :mysql2
+              # SQLite and MySQL store JSON as text
+              plugin :serialization
+              serialize_attributes :json, name
+            end
+          end
+        end
+
+        # Make it unrestricted so it doesn't look for a table
+        base_class.unrestrict_primary_key
+
+        # Set the Base constant
+        Models.send(:remove_const, :Base) if Models.const_defined?(:Base)
+        Models.const_set(:Base, base_class)
+
+        # Remove existing model constants to force reload
+        %i[ModuleExecution ApiRequest OptimizationResult TrainingExample JobResult].each do |model|
+          Models.send(:remove_const, model) if Models.const_defined?(model)
+        end
+
+        # Now load all model classes - use load instead of require to force re-execution
+        load File.expand_path('models/module_execution.rb', __dir__)
+        load File.expand_path('models/api_request.rb', __dir__)
+        load File.expand_path('models/optimization_result.rb', __dir__)
+        load File.expand_path('models/training_example.rb', __dir__)
+        load File.expand_path('models/job_result.rb', __dir__)
+
+        # Setup repositories
+        require_relative 'repository'
+        Repository.setup!
+
+        @initialized = true
+        true
+      end
+    end
+  end
+end

data/lib/desiru/persistence.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require_relative 'persistence/database'
+require_relative 'persistence/models'
+require_relative 'persistence/repository'
+
+module Desiru
+  # Database persistence layer for Desiru
+  module Persistence
+    class << self
+      attr_writer :database_url
+
+      def database_url
+        @database_url ||= ENV['DESIRU_DATABASE_URL'] || 'sqlite://desiru.db'
+      end
+
+      def connect!
+        Database.connect(database_url)
+      end
+
+      def disconnect!
+        Database.disconnect
+      end
+
+      def migrate!
+        Database.migrate!
+        Repository.setup!
+      end
+
+      def repositories
+        @repositories ||= {}
+      end
+
+      def register_repository(name, repository)
+        repositories[name] = repository
+      end
+
+      def [](name)
+        repositories[name] || raise("Repository #{name} not found")
+      end
+
+      def enabled?
+        !Database.connection.nil?
+      rescue StandardError
+        false
+      end
+    end
+  end
+end

data/lib/desiru/registry.rb

@@ -32,11 +32,9 @@ module Desiru
     def get(name, version: nil)
       name = name.to_sym
 
-      if version
-        @module_versions[name][version] || raise(ModuleError, "Module #{name} v#{version} not found")
-      else
-        @modules[name] || raise(ModuleError, "Module #{name} not found")
-      end
+      return @module_versions[name][version] || raise(ModuleError, "Module #{name} v#{version} not found") if version
+
+      @modules[name] || raise(ModuleError, "Module #{name} not found")
     end
 
     def list

data/lib/desiru/signature.rb

@@ -4,10 +4,46 @@ module Desiru
   # Represents input/output specifications for modules
   # Supports DSL-style signature strings like "question -> answer"
   class Signature
+    # FieldWrapper provides a test-compatible interface for Field objects
+    class FieldWrapper
+      def initialize(field)
+        @field = field
+      end
+
+      def type
+        @field.type
+      end
+
+      def description
+        @field.description
+      end
+
+      def optional
+        @field.optional
+      end
+
+      def optional?
+        @field.optional?
+      end
+
+      def name
+        @field.name
+      end
+
+      def method_missing(method, *, &)
+        @field.send(method, *, &)
+      end
+
+      def respond_to_missing?(method, include_private = false)
+        @field.respond_to?(method, include_private)
+      end
+    end
+
     # FieldHash allows access by both string and symbol keys
     class FieldHash < Hash
       def [](key)
-        super(key.to_sym)
+        field = super(key.to_sym)
+        field ? FieldWrapper.new(field) : nil
       end
 
       def []=(key, value)
@@ -18,12 +54,12 @@ module Desiru
         super.map(&:to_s)
       end
 
-      def has_key?(key)
+      def key?(key)
         super(key.to_sym)
       end
 
-      alias include? has_key?
-      alias key? has_key?
+      alias include? key?
+      alias has_key? key?
     end
 
     attr_reader :raw_signature
@@ -51,7 +87,7 @@ module Desiru
       parse_signature!
     end
 
-    def validate_inputs(inputs)
+    def valid_inputs?(inputs)
       missing = required_input_fields - inputs.keys.map(&:to_sym)
       raise SignatureError, "Missing required inputs: #{missing.join(', ')}" if missing.any?
 
@@ -60,13 +96,13 @@ module Desiru
         next unless field
 
         # Field.validate will raise ValidationError if validation fails
-        field.validate(value)
+        field.valid?(value)
       end
 
       true
     end
 
-    def validate_outputs(outputs)
+    def valid_outputs?(outputs)
       missing = required_output_fields - outputs.keys.map(&:to_sym)
       raise ValidationError, "Missing required outputs: #{missing.join(', ')}" if missing.any?
 
@@ -75,7 +111,7 @@ module Desiru
         next unless field
 
         # Field.validate will raise ValidationError if validation fails
-        field.validate(value)
+        field.valid?(value)
       end
 
       true
@@ -138,7 +174,7 @@ module Desiru
           bracket_count += 1
         elsif char == ']'
           bracket_count -= 1
-        elsif char == ',' && bracket_count == 0
+        elsif char == ',' && bracket_count.zero?
           fields << current_field.strip
           current_field = String.new
           next
@@ -155,23 +191,37 @@ module Desiru
           optional = name.end_with?('?') || type_info.include?('?')
           # Clean field name by removing trailing ?
           name = name.gsub(/\?$/, '')
+
+          # Extract description if present (in quotes at the end)
+          description = nil
+          # Look for description only after the type definition, not within brackets
+          if type_info =~ /^([^"]+?)\s+"([^"]+)"$/
+            type_info = ::Regexp.last_match(1).strip
+            description = ::Regexp.last_match(2)
+          end
+
           # Remove optional marker before parsing type
           clean_type_info = type_info.gsub('?', '').strip
           type_data = parse_type(clean_type_info)
+          # Store the original type string for tests
+          type_data[:original_type] = clean_type_info
         else
           name = field_str
           # Check if field name ends with ?
           optional = name.end_with?('?')
           name = name.gsub(/\?$/, '')
-          type_data = { type: :string }
+          type_data = { type: :string, original_type: 'string' }
+          description = nil
         end
 
-        description = @descriptions[name.to_sym] || @descriptions[name.to_s]
+        # Use extracted description or fallback to descriptions hash
+        description ||= @descriptions[name.to_sym] || @descriptions[name.to_s]
 
         # Create field with parsed type data
         field_args = {
           description: description,
-          optional: optional
+          optional: optional,
+          original_type: type_data[:original_type]
         }
 
         # Add literal values if present
@@ -209,17 +259,17 @@ module Desiru
             bracket_count += 1
           elsif char == ']'
             bracket_count -= 1
-            if bracket_count == 0
+            if bracket_count.zero?
               end_index = index
               break
             end
           end
         end
 
-        if end_index > 0
+        if end_index.positive?
           literal_content = type_string[8...end_index] # Extract content between 'Literal[' and ']'
           values = parse_literal_values(literal_content)
-          return { type: :literal, literal_values: values }
+          return { type: :literal, literal_values: values, original_type: type_string }
         end
       end
 
@@ -237,33 +287,36 @@ module Desiru
             bracket_count += 1
           elsif char == ']'
             bracket_count -= 1
-            if bracket_count == 0
+            if bracket_count.zero?
               end_index = index
               break
             end
           end
         end
 
-        if end_index > 0
+        if end_index.positive?
           element_type_str = type_string[(start_index + 1)...end_index]
           element_type_data = parse_type(element_type_str) # Recursive for nested types
-          return { type: :list, element_type: element_type_data }
+          return { type: :list, element_type: element_type_data, original_type: type_string }
         end
       end
 
       # Handle Union types (for future implementation)
       if type_string.start_with?('Union[')
         # Placeholder for union type parsing
-        return { type: :union, union_types: [] } # To be implemented
+        return { type: :union, union_types: [], original_type: type_string } # To be implemented
       end
 
       # Handle basic types
       # First check if it's a list/array with simple element type (e.g., list[str])
       if type_string.downcase.start_with?('list[', 'array[')
         # Even if we couldn't parse the brackets properly above, it's still a list
-        return { type: :list }
+        return { type: :list, original_type: type_string }
       end
 
+      # Handle dict/dictionary types with special parsing
+      return { type: :hash, original_type: type_string } if type_string.downcase.start_with?('dict[', 'dictionary[')
+
       clean_type = type_string.gsub(/[?\[\]]/, '').downcase
       type_sym = case clean_type
                  when 'str', 'string' then :string
@@ -275,7 +328,7 @@ module Desiru
                  else clean_type.to_sym
                  end
 
-      { type: type_sym }
+      { type: type_sym, original_type: type_string }
     end
 
     def parse_literal_values(literal_content)
@@ -289,16 +342,23 @@ module Desiru
         if !in_quotes && ['"', "'"].include?(char)
           in_quotes = true
           quote_char = char
+          current_value << char # Include the quote in the value
         elsif in_quotes && char == quote_char
           # Check if it's escaped
-          if index > 0 && literal_content[index - 1] != '\\'
+          if index.positive? && literal_content[index - 1] != '\\'
             in_quotes = false
+            current_value << char # Include the closing quote
             quote_char = nil
           else
             current_value << char
           end
         elsif !in_quotes && char == ','
-          values << current_value.strip.gsub(/^['"]|['"]$/, '')
+          val = current_value.strip
+          # Remove outer quotes if present
+          if (val.start_with?('"') && val.end_with?('"')) || (val.start_with?("'") && val.end_with?("'"))
+            val = val[1...-1]
+          end
+          values << val
           current_value = String.new
         else
           current_value << char
@@ -306,7 +366,14 @@ module Desiru
       end
 
       # Add the last value
-      values << current_value.strip.gsub(/^['"]|['"]$/, '') unless current_value.empty?
+      unless current_value.empty?
+        val = current_value.strip
+        # Remove outer quotes if present
+        if (val.start_with?('"') && val.end_with?('"')) || (val.start_with?("'") && val.end_with?("'"))
+          val = val[1...-1]
+        end
+        values << val
+      end
 
       values
     end

data/lib/desiru/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Desiru
-  VERSION = '0.1.0'
+  VERSION = '0.1.1'
 end

data/lib/desiru.rb

@@ -5,13 +5,6 @@ require 'singleton'
 
 # Main namespace for Desiru - Declarative Self-Improving Ruby
 module Desiru
-  class Error < StandardError; end
-  class ConfigurationError < Error; end
-  class SignatureError < Error; end
-  class ModuleError < Error; end
-  class ValidationError < Error; end
-  class TimeoutError < Error; end
-
   class << self
     attr_writer :configuration
 
@@ -26,14 +19,20 @@ module Desiru
     def reset_configuration!
       @configuration = Configuration.new
     end
+
+    def logger
+      configuration.logger
+    end
   end
 end
 
 # Core components
 require_relative 'desiru/version'
+require_relative 'desiru/errors'
 require_relative 'desiru/configuration'
 require_relative 'desiru/field'
 require_relative 'desiru/signature'
+require_relative 'desiru/assertions'
 require_relative 'desiru/module'
 require_relative 'desiru/program'
 require_relative 'desiru/registry'
@@ -41,23 +40,39 @@ require_relative 'desiru/cache'
 
 # Model adapters
 require_relative 'desiru/models/base'
-require_relative 'desiru/models/raix_adapter'
+require_relative 'desiru/models/anthropic'
+require_relative 'desiru/models/open_ai'
+require_relative 'desiru/models/open_router'
 
 # Built-in modules
 require_relative 'desiru/modules/predict'
 require_relative 'desiru/modules/chain_of_thought'
 require_relative 'desiru/modules/retrieve'
+require_relative 'desiru/modules/react'
+require_relative 'desiru/modules/program_of_thought'
+require_relative 'desiru/modules/multi_chain_comparison'
+require_relative 'desiru/modules/majority'
 
 # Optimizers
 require_relative 'desiru/optimizers/base'
 require_relative 'desiru/optimizers/bootstrap_few_shot'
+require_relative 'desiru/optimizers/knn_few_shot'
+require_relative 'desiru/optimizers/copro'
 
 # Background jobs
+require_relative 'desiru/async_capable'
+require_relative 'desiru/async_status'
 require_relative 'desiru/jobs/base'
 require_relative 'desiru/jobs/async_predict'
 require_relative 'desiru/jobs/batch_processor'
+
+# API integrations
+require_relative 'desiru/api'
 require_relative 'desiru/jobs/optimizer_job'
 
+# Persistence layer
+require_relative 'desiru/persistence'
+
 # GraphQL integration (optional, requires 'graphql' gem)
 begin
   require 'graphql'

data/missing-features-analysis.md

@@ -0,0 +1,192 @@
+# Missing DSPy Features in Desiru
+
+Based on analysis of Python DSPy vs current Desiru implementation.
+
+## Missing Modules
+
+### 1. **ProgramOfThought**
+- Generates executable code instead of natural language
+- Critical for math/logic problems requiring computation
+- Uses code execution environment
+
+### 2. **MultiChainComparison**
+- Runs multiple ChainOfThought instances
+- Compares and selects best reasoning path
+- Useful for complex reasoning tasks
+
+### 3. **BestOfN**
+- Samples N outputs from any module
+- Selects best based on metric/scoring
+- Simple but effective ensemble technique
+
+### 4. **Refine**
+- Iterative refinement of outputs
+- Takes initial output and improves it
+- Works with constraints and feedback
+
+### 5. **ChainOfThoughtWithHint**
+- ChainOfThought variant with guided hints
+- Provides additional context for reasoning
+- Better control over reasoning direction
+
+## Missing Optimizers
+
+### 1. **MIPROv2**
+- Most advanced DSPy optimizer
+- Uses Bayesian optimization
+- Optimizes both instructions and demonstrations
+- Significantly better than BootstrapFewShot
+
+### 2. **COPRO (Collaborative Prompt Optimization)**
+- Coordinates multiple optimization strategies
+- Collaborative approach to prompt engineering
+- Handles complex multi-module programs
+
+### 3. **BootstrapFewShotWithRandomSearch**
+- Enhanced version of BootstrapFewShot
+- Adds hyperparameter random search
+- Better exploration of optimization space
+
+### 4. **LabeledFewShot**
+- Simple optimizer using provided examples
+- No bootstrapping, just uses given labels
+- Good baseline optimizer
+
+### 5. **KNNFewShot**
+- K-nearest neighbor example selection
+- Dynamic example selection based on input
+- Better than static few-shot examples
+
+### 6. **BootstrapFinetune**
+- Generates training data for model finetuning
+- Alternative to prompt optimization
+- For when you can modify the model
+
+### 7. **Ensemble**
+- Combines multiple optimized programs
+- Voting or weighted combination
+- Improved robustness
+
+### 8. **SignatureOptimizer**
+- Optimizes signature descriptions themselves
+- Rewrites field descriptions for clarity
+- Meta-optimization approach
+
+### 9. **BayesianSignatureOptimizer**
+- Bayesian approach to signature optimization
+- More sophisticated than SignatureOptimizer
+- Better exploration of description space
+
+## Missing Core Features
+
+### 1. **Example and Prediction Classes**
+- Special data containers with utilities
+- Flexible field access (dot notation)
+- Completion tracking for Predictions
+- Integration with trace system
+
+### 2. **Typed Predictors**
+- Type-safe field handling
+- Pydantic integration in Python
+- Automatic validation and parsing
+- Better IDE support
+
+### 3. **Suggestions (Soft Constraints)**
+- Unlike Assertions (hard constraints)
+- Guide optimization without failing
+- Used during compilation phase
+
+### 4. **Trace Collection System**
+- Detailed execution tracking
+- Records all LLM calls and transformations
+- Critical for optimization
+- Enables debugging and analysis
+
+### 5. **Compilation Infrastructure**
+- Full compilation pipeline
+- Trace filtering and selection
+- Demonstration ranking
+- Parameter update mechanism
+
+### 6. **Instruction Generation**
+- Some optimizers generate custom instructions
+- Not just examples but rewritten prompts
+- Adaptive to task requirements
+
+## Missing Utilities & Capabilities
+
+### 1. **Data Loaders**
+- HuggingFace dataset integration
+- CSV/JSON loaders with DSPy formatting
+- Train/dev/test split utilities
+- Batch processing support
+
+### 2. **LLM Provider Abstractions**
+- Unified interface for multiple providers
+- Beyond just OpenAI (Anthropic, Cohere, etc.)
+- Local model support (Ollama, etc.)
+- Token counting and cost tracking
+
+### 3. **Advanced Metrics**
+- F1, BLEU, ROUGE scores
+- LLM-as-Judge implementations
+- Composite metric builders
+- Batch evaluation utilities
+
+### 4. **Streaming Support**
+- Token-by-token streaming
+- Progressive output display
+- Useful for long generations
+
+### 5. **Serialization**
+- Save/load compiled programs
+- Export optimized parameters
+- Model versioning support
+
+### 6. **Settings Management**
+- Global configuration system
+- Provider-specific settings
+- Experiment tracking
+
+### 7. **Advanced Caching**
+- Request deduplication
+- Semantic caching options
+- Cache invalidation strategies
+
+### 8. **Parallel/Async Execution**
+- Batch processing optimizations
+- Concurrent module execution
+- Async compilation runs
+
+### 9. **ColBERTv2 Integration**
+- Advanced retrieval model
+- Better than basic vector search
+- Optimized for retrieval tasks
+
+### 10. **Logging and Debugging**
+- Detailed trace visualization
+- Cost tracking and reporting
+- Performance profiling
+
+## Priority Recommendations
+
+### High Priority (Core Functionality)
+1. Example/Prediction classes
+2. Trace collection system
+3. MIPROv2 optimizer
+4. ProgramOfThought module
+5. Compilation infrastructure
+
+### Medium Priority (Enhanced Capabilities)
+1. MultiChainComparison
+2. BestOfN module
+3. Typed predictors
+4. Additional optimizers (COPRO, KNNFewShot)
+5. Data loaders
+
+### Low Priority (Nice to Have)
+1. Advanced metrics
+2. Streaming support
+3. ColBERTv2 integration
+4. Ensemble optimizer
+5. Signature optimizers