desiru 0.1.0 → 0.1.1

data/db/migrations/001_create_initial_tables.rb

@@ -0,0 +1,96 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  up do
+    # API Requests table
+    create_table(:api_requests) do
+      primary_key :id
+      String :method, null: false
+      String :path, null: false
+      String :remote_ip
+      Integer :status_code, null: false
+      Float :response_time
+      String :headers, text: true # JSON
+      String :params, text: true # JSON
+      String :response_body, text: true # JSON
+      String :error_message
+      DateTime :created_at, null: false
+      DateTime :updated_at, null: false
+
+      index :path
+      index :status_code
+      index :created_at
+    end
+
+    # Module Executions table
+    create_table(:module_executions) do
+      primary_key :id
+      foreign_key :api_request_id, :api_requests, on_delete: :cascade
+      String :module_name, null: false
+      String :module_type
+      String :status, null: false, default: 'pending'
+      String :inputs, text: true # JSON
+      String :outputs, text: true # JSON
+      String :metadata, text: true # JSON
+      String :error_message
+      String :error_backtrace, text: true
+      DateTime :started_at, null: false
+      DateTime :finished_at
+      DateTime :created_at, null: false
+      DateTime :updated_at, null: false
+
+      index :module_name
+      index :status
+      index :started_at
+      index %i[module_name status]
+    end
+
+    # Optimization Results table
+    create_table(:optimization_results) do
+      primary_key :id
+      String :module_name, null: false
+      String :optimizer_type, null: false
+      Float :score, null: false
+      Float :baseline_score
+      Integer :training_size
+      Integer :validation_size
+      String :parameters, text: true # JSON
+      String :metrics, text: true # JSON
+      String :best_prompts, text: true # JSON
+      DateTime :started_at
+      DateTime :finished_at
+      DateTime :created_at, null: false
+      DateTime :updated_at, null: false
+
+      index :module_name
+      index :optimizer_type
+      index :score
+      index %i[module_name optimizer_type]
+    end
+
+    # Training Examples table
+    create_table(:training_examples) do
+      primary_key :id
+      String :module_name, null: false
+      String :dataset_type, default: 'training'
+      String :inputs, text: true, null: false # JSON
+      String :expected_outputs, text: true # JSON
+      String :metadata, text: true # JSON
+      Integer :used_count, default: 0
+      DateTime :last_used_at
+      DateTime :created_at, null: false
+      DateTime :updated_at, null: false
+
+      index :module_name
+      index :dataset_type
+      index %i[module_name dataset_type]
+    end
+  end
+
+  down do
+    drop_table(:training_examples)
+    drop_table(:optimization_results)
+    drop_table(:module_executions)
+    drop_table(:api_requests)
+  end
+end

data/db/migrations/002_create_job_results.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+Sequel.migration do
+  up do
+    # Job Results table for persisting background job results
+    create_table(:job_results) do
+      primary_key :id
+      String :job_id, null: false, unique: true
+      String :job_class, null: false
+      String :queue, null: false
+      String :status, null: false, default: 'pending' # pending, processing, completed, failed
+      Integer :progress, default: 0
+      String :message
+      String :inputs, text: true # JSON
+      String :result, text: true # JSON
+      String :error_message
+      String :error_backtrace, text: true
+      Integer :retry_count, default: 0
+      DateTime :enqueued_at, null: false
+      DateTime :started_at
+      DateTime :finished_at
+      DateTime :expires_at
+      DateTime :created_at, null: false
+      DateTime :updated_at, null: false
+
+      index :job_id
+      index :job_class
+      index :status
+      index :queue
+      index :created_at
+      index :expires_at
+      index %i[job_class status]
+    end
+  end
+
+  down do
+    drop_table(:job_results)
+  end
+end

data/desiru.gemspec

@@ -13,7 +13,7 @@ Gem::Specification.new do |spec|
                      'enabling reliable, maintainable, and portable AI programming.'
   spec.homepage = 'https://github.com/obie/desiru'
   spec.license = 'MIT'
-  spec.required_ruby_version = '>= 3.4.2'
+  spec.required_ruby_version = '>= 3.3.0'
 
   spec.metadata['homepage_uri'] = spec.homepage
   spec.metadata['source_code_uri'] = 'https://github.com/obie/desiru'
@@ -36,9 +36,6 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'sidekiq', '~> 7.2'
   spec.add_dependency 'singleton', '~> 0.1'
 
-  # Development dependencies (basic ones, others in Gemfile)
-  spec.add_development_dependency 'bundler', '~> 2.0'
-  spec.add_development_dependency 'rake', '~> 13.0'
-  spec.add_development_dependency 'rspec', '~> 3.0'
+  # Development dependencies moved to Gemfile
   spec.metadata['rubygems_mfa_required'] = 'false'
 end

data/docs/background_processing_roadmap.md

@@ -0,0 +1,87 @@
+# Background Processing Roadmap for Desiru
+
+## Current State
+
+The Desiru background processing system now includes:
+- ✅ Sidekiq integration for job processing
+- ✅ Redis for fast job status tracking
+- ✅ Database persistence for long-term job result storage
+- ✅ Async module capabilities
+- ✅ Job status tracking and progress updates
+- ✅ Batch processing support
+
+## Proposed Enhancements
+
+### 1. Advanced Retry Strategies (Priority: High)
+Implement sophisticated retry mechanisms for failed jobs:
+- **Exponential backoff** with jitter to prevent thundering herd
+- **Circuit breaker pattern** for external service failures
+- **Dead letter queue** for jobs that exceed retry limits
+- **Custom retry policies** per job type
+
+### 2. Job Scheduling and Cron Support (Priority: Medium)
+Add support for scheduled and recurring jobs:
+- **Cron-style scheduling** using Sidekiq-cron or similar
+- **Delayed job execution** with precise timing
+- **Recurring optimization tasks** for model improvements
+- **Scheduled cleanup jobs** for expired data
+
+### 3. Workflow Orchestration (Priority: Medium)
+Enable complex multi-step workflows:
+- **Job dependencies** - jobs that wait for others to complete
+- **Parallel execution** with fan-out/fan-in patterns
+- **Conditional branching** based on job results
+- **Workflow visualization** and monitoring
+
+### 4. Enhanced Monitoring and Alerting (Priority: High)
+Improve visibility into job processing:
+- **Real-time dashboards** for job metrics
+- **Performance analytics** per job type
+- **Alert thresholds** for queue depth and processing time
+- **Integration with monitoring services** (Datadog, New Relic, etc.)
+
+### 5. Webhook and Callback System (Priority: Low)
+Notify external systems of job events:
+- **Configurable webhooks** for job completion/failure
+- **Event streaming** for real-time updates
+- **Retry logic** for failed webhook deliveries
+- **Security features** (HMAC signatures, etc.)
+
+### 6. Resource Management (Priority: Medium)
+Optimize resource usage:
+- **Dynamic worker scaling** based on queue depth
+- **Memory limits** per job type
+- **CPU throttling** for resource-intensive jobs
+- **Priority-based resource allocation**
+
+### 7. Testing Improvements (Priority: High)
+Enhance testing capabilities:
+- **Job testing helpers** for easier unit tests
+- **Performance benchmarking** framework
+- **Chaos engineering** tools for resilience testing
+- **Mock job execution** for integration tests
+
+## Implementation Priority
+
+1. **Phase 1** (Immediate):
+   - Advanced retry strategies
+   - Enhanced monitoring and alerting
+   - Testing improvements
+
+2. **Phase 2** (Near-term):
+   - Job scheduling and cron support
+   - Workflow orchestration basics
+   - Resource management
+
+3. **Phase 3** (Long-term):
+   - Full workflow orchestration
+   - Webhook and callback system
+   - Advanced resource optimization
+
+## Benefits
+
+- **Reliability**: Better retry strategies reduce job failures
+- **Scalability**: Resource management enables efficient scaling
+- **Visibility**: Enhanced monitoring provides operational insights
+- **Flexibility**: Workflow orchestration enables complex use cases
+- **Integration**: Webhooks allow seamless external system integration

data/docs/job_scheduling.md

@@ -0,0 +1,167 @@
+# Job Scheduling in Desiru
+
+Desiru provides a built-in job scheduling system that allows you to run background jobs periodically without adding external dependencies like sidekiq-cron.
+
+## Features
+
+- Simple cron-like scheduling expressions
+- Interval-based scheduling ("every 5 minutes")
+- Standard cron expressions support
+- Lightweight implementation with no external dependencies
+- Thread-safe singleton scheduler
+- Easy mixin for making jobs schedulable
+
+## Usage
+
+### Making a Job Schedulable
+
+Include the `Schedulable` mixin in your job class:
+
+```ruby
+class MyPeriodicJob < Desiru::Jobs::Base
+  include Desiru::Jobs::Schedulable
+
+  def perform(job_id = nil)
+    # Your job logic here
+    puts "Running periodic job: #{job_id}"
+    
+    # Store result if needed
+    store_result(job_id, { status: 'completed', timestamp: Time.now })
+  end
+end
+```
+
+### Scheduling Jobs
+
+#### Simple Interval Scheduling
+
+```ruby
+# Run every 60 seconds
+MyPeriodicJob.schedule(cron: '60')
+
+# Run every 5 minutes
+MyPeriodicJob.schedule(cron: 'every 5 minutes')
+
+# Run every 2 hours
+MyPeriodicJob.schedule(cron: 'every 2 hours')
+
+# Run daily
+MyPeriodicJob.schedule(cron: 'every 1 day')
+```
+
+#### Cron Expression Scheduling
+
+```ruby
+# Run every minute
+MyPeriodicJob.schedule(cron: '* * * * *')
+
+# Run every hour at minute 0
+MyPeriodicJob.schedule(cron: '0 * * * *')
+
+# Run daily at 9:30 AM
+MyPeriodicJob.schedule(cron: '30 9 * * *')
+```
+
+#### Advanced Scheduling Options
+
+```ruby
+# Schedule with custom name
+MyPeriodicJob.schedule(
+  name: 'custom_job_name',
+  cron: 'every 30 minutes'
+)
+
+# Schedule with arguments
+MyPeriodicJob.schedule(
+  cron: 'every 1 hour',
+  args: ['arg1', 'arg2']
+)
+
+# Schedule with additional options
+MyPeriodicJob.schedule(
+  name: 'important_job',
+  cron: '0 */6 * * *',  # Every 6 hours
+  args: ['production'],
+  priority: 'high'      # Additional options passed through
+)
+```
+
+### Managing the Scheduler
+
+```ruby
+scheduler = Desiru::Jobs::Scheduler.instance
+
+# Start the scheduler
+scheduler.start
+
+# Check if scheduler is running
+scheduler.running? # => true
+
+# Stop the scheduler
+scheduler.stop
+
+# Get information about a scheduled job
+info = scheduler.job_info('MyPeriodicJob')
+# => { job_class: MyPeriodicJob, cron: '60', next_run: Time, ... }
+
+# Clear all scheduled jobs
+scheduler.clear
+```
+
+### Checking Job Status
+
+```ruby
+# Check if a job is scheduled
+MyPeriodicJob.scheduled? # => true
+
+# Check by custom name
+MyPeriodicJob.scheduled?(name: 'custom_job_name') # => true
+
+# Unschedule a job
+MyPeriodicJob.unschedule
+
+# Unschedule by name
+MyPeriodicJob.unschedule(name: 'custom_job_name')
+```
+
+## Supported Cron Formats
+
+### Interval Expressions
+
+- Simple seconds: `"60"` (runs every 60 seconds)
+- Natural language: `"every N [second(s)|minute(s)|hour(s)|day(s)]"`
+
+### Cron Expressions
+
+Currently supports basic cron patterns:
+
+- `* * * * *` - Every minute
+- `0 * * * *` - Every hour at minute 0
+- `30 10 * * *` - Daily at 10:30 AM
+
+More complex cron patterns default to hourly execution.
+
+## Implementation Details
+
+The scheduler:
+- Runs in a background thread
+- Checks for jobs to run every second
+- Generates unique job IDs for each scheduled execution
+- Logs job execution and errors
+- Handles job execution errors gracefully without stopping the scheduler
+
+## Example
+
+See `examples/scheduled_job_example.rb` for a complete working example of scheduled jobs.
+
+## Best Practices
+
+1. **Idempotent Jobs**: Make your scheduled jobs idempotent since they may run multiple times
+2. **Error Handling**: Include proper error handling in your job's perform method
+3. **Logging**: Use Desiru.logger for consistent logging
+4. **Resource Cleanup**: Stop the scheduler gracefully when shutting down your application
+5. **Monitoring**: Monitor scheduled job execution through job results and logs
+
+## Integration with Sidekiq
+
+The scheduler integrates seamlessly with Sidekiq. When a scheduled job's time comes, the scheduler calls `perform_async` on the job class, which enqueues it into Sidekiq for processing.

data/dspy-analysis-swarm.yml

@@ -0,0 +1,60 @@
+version: 1
+swarm:
+  name: "DSPy Analysis & Documentation Team"
+  main: lead_analyst
+  instances:
+    lead_analyst:
+      description: "Lead analyst coordinating DSPy feature analysis and documentation strategy"
+      directory: .
+      model: opus
+      connections: [feature_researcher, integration_tester, documentation_writer]
+      prompt: "You are the lead analyst for a Ruby port of DSPy. Your role is to coordinate analysis of missing features compared to Python DSPy, oversee integration testing, and guide documentation preparation. Focus on strategic decisions and high-level architecture analysis. For maximum efficiency, whenever you need to perform multiple independent operations, invoke all relevant tools simultaneously rather than sequentially."
+      allowed_tools:
+        - Read
+        - Edit
+        - MultiEdit
+        - WebSearch
+        - WebFetch
+        - Bash
+
+    feature_researcher:
+      description: "DSPy expert researching Python DSPy features to identify gaps in Ruby implementation"
+      directory: .
+      model: opus
+      prompt: "You specialize in analyzing the Python DSPy library to identify missing features in this Ruby port. Research DSPy documentation, compare with current Ruby implementation, and document feature gaps. Focus on core DSPy concepts like modules, optimizers, retrievals, and signatures. For maximum efficiency, whenever you need to perform multiple independent operations, invoke all relevant tools simultaneously rather than sequentially."
+      allowed_tools:
+        - Read
+        - WebSearch
+        - WebFetch
+        - Edit
+        - MultiEdit
+        - Write
+        - Bash
+
+    integration_tester:
+      description: "Integration testing specialist ensuring all Ruby DSPy features work correctly through comprehensive tests"
+      directory: .
+      model: opus
+      connections: [feature_researcher]
+      prompt: "You are responsible for creating and running comprehensive integration tests to verify that all DSPy features work correctly in the Ruby implementation. Focus on end-to-end workflows, real API interactions, and complex module compositions. Use RSpec exclusively for all testing. For maximum efficiency, whenever you need to perform multiple independent operations, invoke all relevant tools simultaneously rather than sequentially."
+      allowed_tools:
+        - Read
+        - Edit
+        - MultiEdit
+        - Write
+        - Bash
+
+    documentation_writer:
+      description: "Technical writer preparing GitHub wiki documentation following DSPy documentation patterns"
+      directory: .
+      model: opus
+      connections: [feature_researcher]
+      prompt: "You create comprehensive GitHub wiki documentation for the Ruby DSPy port, following the structure and style of the original Python DSPy documentation. Focus on API references, usage examples, tutorials, and migration guides. Research the original DSPy docs for inspiration and maintain consistency with Ruby conventions. For maximum efficiency, whenever you need to perform multiple independent operations, invoke all relevant tools simultaneously rather than sequentially."
+      allowed_tools:
+        - Read
+        - Write
+        - Edit
+        - MultiEdit
+        - WebSearch
+        - WebFetch
+        - Bash

data/dspy-feature-analysis.md

@@ -0,0 +1,121 @@
+# DSPy Feature Analysis for Desiru Implementation
+
+This document provides a comprehensive analysis of the Python DSPy library's core features, modules, and components to guide the Ruby implementation of Desiru.
+
+## Core Concepts
+
+### 1. Programming Model
+- **Declarative Approach**: DSPy separates program flow (modules and logic) from parameters (prompts) that control LLM behavior
+- **Compositional**: Build complex systems by composing simple modules
+- **Self-Improving**: Programs can be automatically optimized through compilation
+
+### 2. Signatures
+- Function declarations that specify what a text transformation should do (not how)
+- Format: `"input1, input2 -> output1, output2"`
+- Examples:
+  - `"question -> answer"` for basic Q&A
+  - `"context, question -> answer"` for retrieval-augmented generation
+  - `"sentence -> sentiment: bool"` for classification
+- Include field names and optional metadata
+- Support type hints to shape LM behavior
+
+### 3. Modules
+Core building blocks inspired by PyTorch modules:
+- **dspy.Predict**: Basic predictor, handles instructions and demonstrations
+- **dspy.ChainOfThought**: Adds step-by-step reasoning before output
+- **dspy.ProgramOfThought**: Outputs executable code
+- **dspy.ReAct**: Agent that can use tools to implement signatures
+- **dspy.MultiChainComparison**: Compares multiple ChainOfThought outputs
+- **dspy.Retrieve**: Information retrieval module
+- **dspy.BestOfN**: Runs module N times, returns best result
+- **dspy.Refine**: Iterative refinement of outputs
+
+### 4. Data Handling
+- **Example**: Core data type, similar to Python dict with utilities
+- **Prediction**: Special subclass of Example returned by modules
+- Supports loading from HuggingFace datasets, CSV files
+- Built-in train/test split capabilities
+
+### 5. Metrics
+- Functions that take (example, prediction, optional trace) and return a score
+- Can be simple boolean checks or complex DSPy programs
+- Used for both evaluation and optimization
+- Support for LLM-as-Judge metrics
+
+### 6. Optimizers (Teleprompters)
+Automated prompt optimization strategies:
+- **LabeledFewShot**: Uses provided labeled examples
+- **BootstrapFewShot**: Generates demonstrations from program execution
+- **BootstrapFewShotWithRandomSearch**: Multiple runs with random search
+- **MIPROv2**: Advanced optimizer using Bayesian optimization
+- **BootstrapFinetune**: Generates data for finetuning
+- **COPRO**: Collaborative prompt optimization
+- **KNNFewShot**: K-nearest neighbor example selection
+- **Ensemble**: Combines multiple optimized programs
+
+### 7. Compilation Process
+1. **Bootstrapping**: Run program on training data to collect execution traces
+2. **Filtering**: Keep only traces that pass the metric
+3. **Demonstration Selection**: Choose best examples for few-shot prompts
+4. **Instruction Generation**: Create optimized instructions (some optimizers)
+5. **Parameter Updates**: Update module prompts and demonstrations
+
+### 8. Assertions and Constraints
+- **dspy.Assert**: Hard constraints that must be satisfied
+- **dspy.Suggest**: Soft constraints for guidance
+- **dspy.Refine**: Iterative refinement based on constraints
+- **dspy.BestOfN**: Sample multiple outputs, select best
+
+### 9. Retrieval and RAG
+- Built-in support for retrieval-augmented generation
+- **ColBERTv2** integration for semantic search
+- Composable retrieval modules
+- Support for various vector databases
+
+### 10. Agent Capabilities
+- **ReAct** module for tool use and multi-step reasoning
+- Support for building complex agent loops
+- Integration with external tools and APIs
+
+## Key Architectural Patterns
+
+1. **Separation of Concerns**: Program logic separate from LM parameters
+2. **Modular Composition**: Build complex systems from simple modules
+3. **Automatic Optimization**: Compile programs to improve performance
+4. **Trace-Based Learning**: Learn from execution traces, not just outputs
+5. **Metric-Driven Development**: Define success metrics, let DSPy optimize
+
+## Implementation Priorities for Desiru
+
+### Phase 1: Core Foundation
+1. Signature parsing and representation
+2. Basic Predict module
+3. Example and Prediction data structures
+4. Simple metrics system
+
+### Phase 2: Essential Modules
+1. ChainOfThought module
+2. Basic optimizer (BootstrapFewShot)
+3. Compilation infrastructure
+4. Trace collection system
+
+### Phase 3: Advanced Features
+1. ReAct agent module
+2. Retrieval modules
+3. Advanced optimizers (MIPROv2)
+4. Assertion system
+
+### Phase 4: Ecosystem
+1. Data loaders
+2. Integration with Ruby ML libraries
+3. Performance optimizations
+4. Documentation and examples
+
+## Design Considerations for Ruby
+
+1. **Module System**: Leverage Ruby's module system for composability
+2. **DSL**: Create Ruby-idiomatic DSL for signatures
+3. **Blocks**: Use blocks for metric definitions
+4. **Method Missing**: Consider for dynamic module composition
+5. **Lazy Evaluation**: For efficient trace collection
+6. **Concurrent Processing**: For parallel optimization runs

data/examples/README.md

@@ -39,6 +39,75 @@ Shows how to use the BootstrapFewShot optimizer to improve module performance wi
 ruby examples/few_shot_learning.rb
 ```
 
+### rest_api.rb
+Creates a REST API server using Grape integration, exposing Desiru modules as HTTP endpoints.
+
+```bash
+ruby examples/rest_api.rb
+# Visit http://localhost:9292 for API documentation
+```
+
+### rest_api_advanced.rb
+Advanced REST API with authentication, rate limiting, and tool-using AI agents.
+
+```bash
+ruby examples/rest_api_advanced.rb
+# API keys: demo-key-123, test-key-456
+```
+
+### sinatra_api.rb
+Lightweight REST API using Sinatra integration as an alternative to Grape.
+
+```bash
+ruby examples/sinatra_api.rb
+# Visit http://localhost:9293 for simpler API endpoints
+```
+
+### graphql_integration.rb
+GraphQL API server with automatic schema generation from Desiru signatures.
+
+```bash
+ruby examples/graphql_integration.rb
+# GraphiQL interface at http://localhost:9292/graphiql
+```
+
+### react_agent.rb
+Demonstrates the ReAct module for building tool-using AI agents.
+
+```bash
+ruby examples/react_agent.rb
+```
+
+### async_processing.rb
+Shows how to use background job processing with Sidekiq for long-running operations.
+
+```bash
+# Start Redis first
+redis-server
+
+# In another terminal, start Sidekiq workers
+bundle exec sidekiq
+
+# Run the example
+ruby examples/async_processing.rb
+```
+
+### persistence_example.rb
+Demonstrates Sequel-based persistence for tracking module executions, API requests, and training data.
+
+```bash
+ruby examples/persistence_example.rb
+# Creates a SQLite database with execution history
+```
+
+### api_with_persistence.rb
+REST API server with automatic request tracking and analytics dashboard.
+
+```bash
+ruby examples/api_with_persistence.rb
+# Visit http://localhost:9294 for the dashboard
+```
+
 ## Creating Your Own Examples
 
 When creating new examples: