desiru 0.1.0 → 0.1.1

data/lib/desiru/persistence/models/module_execution.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Desiru
+  module Persistence
+    module Models
+      # Tracks module execution history
+      class ModuleExecution < Base
+        set_dataset :module_executions
+        many_to_one :api_request
+
+        json_column :inputs
+        json_column :outputs
+        json_column :metadata
+
+        def validate
+          super
+          validates_presence %i[module_name status started_at]
+          validates_includes %w[pending running completed failed], :status
+        end
+
+        def duration
+          return nil unless started_at && finished_at
+
+          finished_at - started_at
+        end
+
+        def success?
+          status == 'completed'
+        end
+
+        def failed?
+          status == 'failed'
+        end
+      end
+    end
+  end
+end

data/lib/desiru/persistence/models/optimization_result.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Desiru
+  module Persistence
+    module Models
+      # Stores optimization results and metrics
+      class OptimizationResult < Base
+        set_dataset :optimization_results
+        json_column :parameters
+        json_column :metrics
+        json_column :best_prompts
+
+        def validate
+          super
+          validates_presence %i[module_name optimizer_type score]
+          validates_numeric :score
+          validates_min_length 1, :training_size if training_size
+        end
+
+        def improvement_percentage
+          return nil unless baseline_score && score.positive?
+
+          ((score - baseline_score) / baseline_score * 100).round(2)
+        end
+      end
+    end
+  end
+end

data/lib/desiru/persistence/models/training_example.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Desiru
+  module Persistence
+    module Models
+      # Stores training examples for modules
+      class TrainingExample < Base
+        set_dataset :training_examples
+        json_column :inputs
+        json_column :expected_outputs
+        json_column :metadata
+
+        def validate
+          super
+          validates_presence %i[module_name inputs]
+          validates_includes %w[training validation test], :dataset_type if dataset_type
+        end
+
+        def used?
+          used_count&.positive?
+        end
+      end
+    end
+  end
+end

data/lib/desiru/persistence/models.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Desiru
+  module Persistence
+    # Namespace for Sequel models
+    module Models
+      # Base class will be defined during setup
+      Base = nil # rubocop:disable Naming/ConstantName
+    end
+  end
+end

data/lib/desiru/persistence/repositories/api_request_repository.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+require_relative 'base_repository'
+
+module Desiru
+  module Persistence
+    module Repositories
+      # Repository for API request records
+      class ApiRequestRepository < BaseRepository
+        def initialize
+          super(Models::ApiRequest)
+        end
+
+        def find_by_path(path)
+          dataset.where(path: path).all
+        end
+
+        def recent(limit = 10)
+          dataset
+            .order(Sequel.desc(:created_at))
+            .limit(limit)
+            .all
+        end
+
+        def by_status_code_range(min, max)
+          dataset.where(status_code: min..max).all
+        end
+
+        def successful
+          by_status_code_range(200, 299)
+        end
+
+        def failed
+          dataset.where { status_code >= 400 }.all
+        end
+
+        def average_response_time(path = nil)
+          scope = dataset
+          scope = scope.where(path: path) if path
+          scope = scope.exclude(response_time: nil)
+
+          avg = scope.avg(:response_time)
+          avg&.round(3)
+        end
+
+        def requests_per_minute(minutes_ago = 60)
+          since = Time.now - (minutes_ago * 60)
+          count = dataset.where { created_at >= since }.count
+
+          (count.to_f / minutes_ago).round(2)
+        end
+
+        def top_paths(limit = 10)
+          dataset
+            .group_and_count(:path)
+            .order(Sequel.desc(:count))
+            .limit(limit)
+            .map { |row| { path: row[:path], count: row[:count] } }
+        end
+
+        def create_from_rack_request(request, response)
+          create(
+            method: request.request_method,
+            path: request.path_info,
+            remote_ip: request.ip,
+            headers: extract_headers(request),
+            params: request.params,
+            status_code: response.status,
+            response_body: extract_response_body(response),
+            response_time: response.headers['X-Runtime']&.to_f
+          )
+        end
+
+        private
+
+        def extract_headers(request)
+          headers = {}
+          request.each_header do |key, value|
+            next unless key.start_with?('HTTP_')
+
+            header_name = key.sub(/^HTTP_/, '').split('_').map(&:capitalize).join('-')
+            headers[header_name] = value
+          end
+          headers
+        end
+
+        def extract_response_body(response)
+          return nil unless response.body.respond_to?(:each)
+
+          body = response.body.map { |part| part }
+          body.join
+        rescue StandardError
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/desiru/persistence/repositories/base_repository.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module Desiru
+  module Persistence
+    module Repositories
+      # Base repository with common CRUD operations
+      class BaseRepository
+        attr_reader :model_class
+
+        def initialize(model_class)
+          @model_class = model_class
+        end
+
+        def all
+          dataset.all
+        end
+
+        def find(id)
+          dataset.first(id: id)
+        end
+
+        def find_by(conditions)
+          dataset.where(conditions).first
+        end
+
+        def where(conditions)
+          dataset.where(conditions).all
+        end
+
+        def create(attributes)
+          model_class.create(attributes)
+        end
+
+        def update(id, attributes)
+          record = find(id)
+          return nil unless record
+
+          record.update(attributes)
+          record
+        end
+
+        def delete?(id)
+          record = find(id)
+          return false unless record
+
+          record.destroy
+          true
+        end
+
+        def count
+          dataset.count
+        end
+
+        def exists?(conditions)
+          dataset.where(conditions).count.positive?
+        end
+
+        def paginate(page: 1, per_page: 20)
+          dataset
+            .limit(per_page)
+            .offset((page - 1) * per_page)
+            .all
+        end
+
+        protected
+
+        def dataset
+          model_class.dataset
+        end
+
+        def transaction(&)
+          Database.transaction(&)
+        end
+      end
+    end
+  end
+end

data/lib/desiru/persistence/repositories/job_result_repository.rb

@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+require_relative 'base_repository'
+
+module Desiru
+  module Persistence
+    module Repositories
+      # Repository for job result persistence
+      class JobResultRepository < BaseRepository
+        def initialize
+          super(Models::JobResult)
+        end
+
+        def create_for_job(job_id, job_class, queue, inputs: nil, expires_at: nil)
+          create(
+            job_id: job_id,
+            job_class: job_class,
+            queue: queue,
+            status: Models::JobResult::STATUS_PENDING,
+            inputs: inputs&.to_json,
+            enqueued_at: Time.now,
+            expires_at: expires_at
+          )
+        end
+
+        def find_by_job_id(job_id)
+          find_by(job_id: job_id)
+        end
+
+        def mark_processing(job_id)
+          job_result = find_by_job_id(job_id)
+          return nil unless job_result
+
+          job_result.mark_as_processing!
+          job_result
+        end
+
+        def mark_completed(job_id, result, message: nil)
+          job_result = find_by_job_id(job_id)
+          return nil unless job_result
+
+          job_result.mark_as_completed!(result, message: message)
+          job_result
+        end
+
+        def mark_failed(job_id, error, backtrace: nil, increment_retry: true)
+          job_result = find_by_job_id(job_id)
+          return nil unless job_result
+
+          updates = {
+            status: Models::JobResult::STATUS_FAILED,
+            finished_at: Time.now,
+            error_message: error.to_s,
+            error_backtrace: backtrace&.join("\n")
+          }
+
+          updates[:retry_count] = job_result.retry_count + 1 if increment_retry
+
+          job_result.update(updates)
+          job_result
+        end
+
+        def update_progress(job_id, progress, message: nil)
+          job_result = find_by_job_id(job_id)
+          return nil unless job_result
+
+          job_result.update_progress(progress, message: message)
+          job_result
+        end
+
+        def cleanup_expired
+          dataset.expired.delete
+        end
+
+        def recent_by_class(job_class, limit: 10)
+          dataset.by_job_class(job_class).recent(limit).all
+        end
+
+        def statistics(job_class: nil, since: nil)
+          scope = dataset
+          scope = scope.by_job_class(job_class) if job_class
+          scope = scope.where { created_at >= since } if since
+
+          {
+            total: scope.count,
+            pending: scope.pending.count,
+            processing: scope.processing.count,
+            completed: scope.completed.count,
+            failed: scope.failed.count,
+            average_duration: calculate_average_duration(scope)
+          }
+        end
+
+        private
+
+        def calculate_average_duration(dataset)
+          completed = dataset.completed.where(Sequel.~(started_at: nil)).where(Sequel.~(finished_at: nil))
+          return 0 if completed.empty?
+
+          total_duration = 0
+          count = 0
+
+          completed.each do |job|
+            next unless job.started_at && job.finished_at
+
+            duration = job.finished_at - job.started_at
+            total_duration += duration
+            count += 1
+          end
+
+          count.positive? ? total_duration / count : 0
+        end
+      end
+    end
+  end
+end

data/lib/desiru/persistence/repositories/module_execution_repository.rb

@@ -0,0 +1,85 @@
+# frozen_string_literal: true
+
+require_relative 'base_repository'
+
+module Desiru
+  module Persistence
+    module Repositories
+      # Repository for module execution records
+      class ModuleExecutionRepository < BaseRepository
+        def initialize
+          super(Models::ModuleExecution)
+        end
+
+        def find_by_module(module_name)
+          dataset.where(module_name: module_name).all
+        end
+
+        def recent(limit = 10)
+          dataset
+            .order(Sequel.desc(:started_at))
+            .limit(limit)
+            .all
+        end
+
+        def by_status(status)
+          dataset.where(status: status).all
+        end
+
+        def average_duration(module_name = nil)
+          scope = dataset
+          scope = scope.where(module_name: module_name) if module_name
+          scope = scope.where(status: 'completed')
+                       .exclude(finished_at: nil)
+
+          records = scope.all
+          return nil if records.empty?
+
+          durations = records.map(&:duration).compact
+          return nil if durations.empty?
+
+          durations.sum.to_f / durations.length
+        end
+
+        def success_rate(module_name = nil)
+          scope = dataset
+          scope = scope.where(module_name: module_name) if module_name
+
+          total = scope.count
+          return 0.0 if total.zero?
+
+          successful = scope.where(status: 'completed').count
+          (successful.to_f / total * 100).round(2)
+        end
+
+        def create_for_module(module_name, inputs, api_request_id: nil)
+          create(
+            module_name: module_name,
+            inputs: inputs,
+            status: 'pending',
+            started_at: Time.now,
+            api_request_id: api_request_id
+          )
+        end
+
+        def complete(id, outputs, metadata = {})
+          update(id, {
+                   outputs: outputs,
+                   metadata: metadata,
+                   status: 'completed',
+                   finished_at: Time.now
+                 })
+        end
+
+        def fail(id, error_message, error_backtrace = nil)
+          update(id, {
+                   error_message: error_message,
+                   error_backtrace: error_backtrace,
+                   status: 'failed',
+                   finished_at: Time.now
+                 })
+        end
+      end
+    end
+  end
+end

data/lib/desiru/persistence/repositories/optimization_result_repository.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require_relative 'base_repository'
+
+module Desiru
+  module Persistence
+    module Repositories
+      # Repository for optimization result records
+      class OptimizationResultRepository < BaseRepository
+        def initialize
+          super(Models::OptimizationResult)
+        end
+
+        def find_by_module(module_name)
+          dataset.where(module_name: module_name).all
+        end
+
+        def find_best_for_module(module_name)
+          dataset
+            .where(module_name: module_name)
+            .order(Sequel.desc(:score))
+            .first
+        end
+
+        def recent(limit = 10)
+          dataset
+            .order(Sequel.desc(:created_at))
+            .limit(limit)
+            .all
+        end
+
+        def by_optimizer_type(type)
+          dataset.where(optimizer_type: type).all
+        end
+
+        def average_improvement(module_name = nil)
+          scope = dataset.exclude(baseline_score: nil)
+          scope = scope.where(module_name: module_name) if module_name
+
+          improvements = scope.select_map(&:improvement_percentage).compact
+
+          return nil if improvements.empty?
+
+          improvements.sum / improvements.length
+        end
+
+        def top_performers(limit = 5)
+          dataset
+            .exclude(baseline_score: nil)
+            .order(Sequel.desc { (score - baseline_score) / baseline_score })
+            .limit(limit)
+            .all
+        end
+
+        def create_result(module_name:, optimizer_type:, score:, **attributes)
+          create(
+            module_name: module_name,
+            optimizer_type: optimizer_type,
+            score: score,
+            started_at: attributes[:started_at] || Time.now,
+            **attributes
+          )
+        end
+      end
+    end
+  end
+end

data/lib/desiru/persistence/repositories/training_example_repository.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+require_relative 'base_repository'
+
+module Desiru
+  module Persistence
+    module Repositories
+      # Repository for training example records
+      class TrainingExampleRepository < BaseRepository
+        def initialize
+          super(Models::TrainingExample)
+        end
+
+        def find_by_module(module_name, dataset_type: nil)
+          scope = dataset.where(module_name: module_name)
+          scope = scope.where(dataset_type: dataset_type) if dataset_type
+          scope.all
+        end
+
+        def find_unused(module_name, limit = 10)
+          dataset
+            .where(module_name: module_name, used_count: 0)
+            .limit(limit)
+            .all
+        end
+
+        def find_least_used(module_name, limit = 10)
+          dataset
+            .where(module_name: module_name)
+            .order(:used_count, :last_used_at)
+            .limit(limit)
+            .all
+        end
+
+        def mark_as_used?(id)
+          record = find(id)
+          return false unless record
+
+          record.update(
+            used_count: record.used_count + 1,
+            last_used_at: Time.now
+          )
+          true
+        end
+
+        def bulk_create(module_name, examples, dataset_type: 'training')
+          transaction do
+            examples.map do |example|
+              create(
+                module_name: module_name,
+                dataset_type: dataset_type,
+                inputs: example[:inputs],
+                expected_outputs: example[:outputs],
+                metadata: example[:metadata]
+              )
+            end
+          end
+        end
+
+        def split_dataset(module_name, train_ratio: 0.8, val_ratio: 0.1)
+          all_examples = find_by_module(module_name)
+          total = all_examples.length
+
+          train_size = (total * train_ratio).floor
+          val_size = (total * val_ratio).floor
+
+          shuffled = all_examples.shuffle
+
+          {
+            training: shuffled[0...train_size],
+            validation: shuffled[train_size...(train_size + val_size)],
+            test: shuffled[(train_size + val_size)..]
+          }
+        end
+
+        def export_for_training(module_name, format: :dspy)
+          examples = find_by_module(module_name, dataset_type: 'training')
+
+          case format
+          when :dspy
+            examples.map do |ex|
+              {
+                inputs: ex.inputs,
+                outputs: ex.expected_outputs
+              }
+            end
+          when :jsonl
+            examples.map do |ex|
+              JSON.generate({
+                              inputs: ex.inputs,
+                              outputs: ex.expected_outputs,
+                              metadata: ex.metadata
+                            })
+            end.join("\n")
+          else
+            raise ArgumentError, "Unknown format: #{format}"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/desiru/persistence/repository.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+require_relative 'repositories/base_repository'
+require_relative 'repositories/module_execution_repository'
+require_relative 'repositories/api_request_repository'
+require_relative 'repositories/optimization_result_repository'
+require_relative 'repositories/training_example_repository'
+require_relative 'repositories/job_result_repository'
+
+module Desiru
+  module Persistence
+    # Repository pattern for data access
+    module Repository
+      def self.setup!
+        # Register all repositories
+        Persistence.register_repository(:module_executions,
+                                        Repositories::ModuleExecutionRepository.new)
+        Persistence.register_repository(:api_requests,
+                                        Repositories::ApiRequestRepository.new)
+        Persistence.register_repository(:optimization_results,
+                                        Repositories::OptimizationResultRepository.new)
+        Persistence.register_repository(:training_examples,
+                                        Repositories::TrainingExampleRepository.new)
+        Persistence.register_repository(:job_results,
+                                        Repositories::JobResultRepository.new)
+      end
+    end
+  end
+end