desiru 0.1.0 → 0.2.0

data/lib/desiru/persistence/database.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+require 'sequel'
+require 'logger'
+
+module Desiru
+  module Persistence
+    # Database connection and migration management
+    module Database
+      class << self
+        attr_reader :connection
+
+        def connect(database_url = nil)
+          url = database_url || Persistence.database_url
+
+          @connection = Sequel.connect(
+            url,
+            logger: logger,
+            max_connections: max_connections
+          )
+
+          # Enable foreign keys for SQLite
+          @connection.run('PRAGMA foreign_keys = ON') if sqlite?
+
+          @connection
+        end
+
+        def disconnect
+          @connection&.disconnect
+          @connection = nil
+        end
+
+        def migrate!
+          raise 'Not connected to database' unless @connection
+
+          Sequel.extension :migration
+          migrations_path = File.expand_path('../../../db/migrations', __dir__)
+          Sequel::Migrator.run(@connection, migrations_path)
+
+          # Initialize persistence layer after migrations
+          require_relative 'setup'
+          Setup.initialize!(@connection)
+        end
+
+        def transaction(&)
+          raise 'Not connected to database' unless @connection
+
+          @connection.transaction(&)
+        end
+
+        private
+
+        def logger
+          return nil unless ENV['DESIRU_DEBUG'] || ENV['DEBUG']
+
+          Logger.new($stdout).tap do |logger|
+            logger.level = Logger::INFO
+          end
+        end
+
+        def max_connections
+          ENV['DESIRU_DB_MAX_CONNECTIONS']&.to_i || 10
+        end
+
+        def sqlite?
+          @connection&.adapter_scheme == :sqlite
+        end
+      end
+    end
+  end
+end

data/lib/desiru/persistence/models/api_request.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Desiru
+  module Persistence
+    module Models
+      # Tracks REST API requests
+      class ApiRequest < Base
+        set_dataset :api_requests
+        one_to_many :module_executions
+
+        json_column :headers
+        json_column :params
+        json_column :response_body
+
+        def validate
+          super
+          # Validate method column separately due to name conflict with Ruby's method method
+          if self[:method].nil? || self[:method].to_s.empty?
+            errors.add(:method, 'is required')
+          elsif !%w[GET POST PUT PATCH DELETE].include?(self[:method])
+            errors.add(:method, 'must be GET, POST, PUT, PATCH, or DELETE')
+          end
+          validates_presence %i[path status_code]
+        end
+
+        def success?
+          status_code >= 200 && status_code < 300
+        end
+
+        def duration_ms
+          return nil unless response_time
+
+          (response_time * 1000).round
+        end
+      end
+    end
+  end
+end

data/lib/desiru/persistence/models/job_result.rb

@@ -0,0 +1,138 @@
+# frozen_string_literal: true
+
+module Desiru
+  module Persistence
+    module Models
+      # Model for storing background job results
+      class JobResult < Base
+        set_dataset :job_results
+
+        # Status constants
+        STATUS_PENDING = 'pending'
+        STATUS_PROCESSING = 'processing'
+        STATUS_COMPLETED = 'completed'
+        STATUS_FAILED = 'failed'
+
+        # Validations
+        def validate
+          super
+          validates_presence %i[job_id job_class queue status enqueued_at]
+          validates_unique :job_id if db&.table_exists?(:job_results)
+          validates_includes %w[pending processing completed failed], :status
+        end
+
+        # Scopes
+        dataset_module do
+          def pending
+            where(status: STATUS_PENDING)
+          end
+
+          def processing
+            where(status: STATUS_PROCESSING)
+          end
+
+          def completed
+            where(status: STATUS_COMPLETED)
+          end
+
+          def failed
+            where(status: STATUS_FAILED)
+          end
+
+          def expired
+            where { expires_at < Time.now }
+          end
+
+          def active
+            where { (expires_at > Time.now) | (expires_at =~ nil) }
+          end
+
+          def by_job_class(job_class)
+            where(job_class: job_class)
+          end
+
+          def recent(limit = 10)
+            order(Sequel.desc(:created_at)).limit(limit)
+          end
+        end
+
+        # Instance methods
+        def pending?
+          status == STATUS_PENDING
+        end
+
+        def processing?
+          status == STATUS_PROCESSING
+        end
+
+        def completed?
+          status == STATUS_COMPLETED
+        end
+
+        def failed?
+          status == STATUS_FAILED
+        end
+
+        def expired?
+          expires_at && expires_at < Time.now
+        end
+
+        def duration
+          return nil unless started_at && finished_at
+
+          finished_at - started_at
+        end
+
+        # JSON field accessors
+        def inputs_data
+          return {} unless inputs
+
+          JSON.parse(inputs, symbolize_names: true)
+        rescue JSON::ParserError
+          {}
+        end
+
+        def result_data
+          return {} unless result
+
+          JSON.parse(result, symbolize_names: true)
+        rescue JSON::ParserError
+          {}
+        end
+
+        def mark_as_processing!
+          update(
+            status: STATUS_PROCESSING,
+            started_at: Time.now,
+            progress: 0
+          )
+        end
+
+        def mark_as_completed!(result_data, message: nil)
+          update(
+            status: STATUS_COMPLETED,
+            finished_at: Time.now,
+            progress: 100,
+            result: result_data.to_json,
+            message: message
+          )
+        end
+
+        def mark_as_failed!(error, backtrace: nil)
+          update(
+            status: STATUS_FAILED,
+            finished_at: Time.now,
+            error_message: error.to_s,
+            error_backtrace: backtrace&.join("\n")
+          )
+        end
+
+        def update_progress(progress, message: nil)
+          updates = { progress: progress }
+          updates[:message] = message if message
+          update(updates)
+        end
+      end
+    end
+  end
+end

data/lib/desiru/persistence/models/module_execution.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Desiru
+  module Persistence
+    module Models
+      # Tracks module execution history
+      class ModuleExecution < Base
+        set_dataset :module_executions
+        many_to_one :api_request
+
+        json_column :inputs
+        json_column :outputs
+        json_column :metadata
+
+        def validate
+          super
+          validates_presence %i[module_name status started_at]
+          validates_includes %w[pending running completed failed], :status
+        end
+
+        def duration
+          return nil unless started_at && finished_at
+
+          finished_at - started_at
+        end
+
+        def success?
+          status == 'completed'
+        end
+
+        def failed?
+          status == 'failed'
+        end
+      end
+    end
+  end
+end

data/lib/desiru/persistence/models/optimization_result.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Desiru
+  module Persistence
+    module Models
+      # Stores optimization results and metrics
+      class OptimizationResult < Base
+        set_dataset :optimization_results
+        json_column :parameters
+        json_column :metrics
+        json_column :best_prompts
+
+        def validate
+          super
+          validates_presence %i[module_name optimizer_type score]
+          validates_numeric :score
+          validates_min_length 1, :training_size if training_size
+        end
+
+        def improvement_percentage
+          return nil unless baseline_score && score.positive?
+
+          ((score - baseline_score) / baseline_score * 100).round(2)
+        end
+      end
+    end
+  end
+end

data/lib/desiru/persistence/models/training_example.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Desiru
+  module Persistence
+    module Models
+      # Stores training examples for modules
+      class TrainingExample < Base
+        set_dataset :training_examples
+        json_column :inputs
+        json_column :expected_outputs
+        json_column :metadata
+
+        def validate
+          super
+          validates_presence %i[module_name inputs]
+          validates_includes %w[training validation test], :dataset_type if dataset_type
+        end
+
+        def used?
+          used_count&.positive?
+        end
+      end
+    end
+  end
+end

data/lib/desiru/persistence/models.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Desiru
+  module Persistence
+    # Namespace for Sequel models
+    module Models
+      # Base class will be defined during setup
+      Base = nil # rubocop:disable Naming/ConstantName
+    end
+  end
+end

data/lib/desiru/persistence/repositories/api_request_repository.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+require_relative 'base_repository'
+
+module Desiru
+  module Persistence
+    module Repositories
+      # Repository for API request records
+      class ApiRequestRepository < BaseRepository
+        def initialize
+          super(Models::ApiRequest)
+        end
+
+        def find_by_path(path)
+          dataset.where(path: path).all
+        end
+
+        def recent(limit = 10)
+          dataset
+            .order(Sequel.desc(:created_at))
+            .limit(limit)
+            .all
+        end
+
+        def by_status_code_range(min, max)
+          dataset.where(status_code: min..max).all
+        end
+
+        def successful
+          by_status_code_range(200, 299)
+        end
+
+        def failed
+          dataset.where { status_code >= 400 }.all
+        end
+
+        def average_response_time(path = nil)
+          scope = dataset
+          scope = scope.where(path: path) if path
+          scope = scope.exclude(response_time: nil)
+
+          avg = scope.avg(:response_time)
+          avg&.round(3)
+        end
+
+        def requests_per_minute(minutes_ago = 60)
+          since = Time.now - (minutes_ago * 60)
+          count = dataset.where { created_at >= since }.count
+
+          (count.to_f / minutes_ago).round(2)
+        end
+
+        def top_paths(limit = 10)
+          dataset
+            .group_and_count(:path)
+            .order(Sequel.desc(:count))
+            .limit(limit)
+            .map { |row| { path: row[:path], count: row[:count] } }
+        end
+
+        def create_from_rack_request(request, response)
+          create(
+            method: request.request_method,
+            path: request.path_info,
+            remote_ip: request.ip,
+            headers: extract_headers(request),
+            params: request.params,
+            status_code: response.status,
+            response_body: extract_response_body(response),
+            response_time: response.headers['X-Runtime']&.to_f
+          )
+        end
+
+        private
+
+        def extract_headers(request)
+          headers = {}
+          request.each_header do |key, value|
+            next unless key.start_with?('HTTP_')
+
+            header_name = key.sub(/^HTTP_/, '').split('_').map(&:capitalize).join('-')
+            headers[header_name] = value
+          end
+          headers
+        end
+
+        def extract_response_body(response)
+          return nil unless response.body.respond_to?(:each)
+
+          body = response.body.map { |part| part }
+          body.join
+        rescue StandardError
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/desiru/persistence/repositories/base_repository.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module Desiru
+  module Persistence
+    module Repositories
+      # Base repository with common CRUD operations
+      class BaseRepository
+        attr_reader :model_class
+
+        def initialize(model_class)
+          @model_class = model_class
+        end
+
+        def all
+          dataset.all
+        end
+
+        def find(id)
+          dataset.first(id: id)
+        end
+
+        def find_by(conditions)
+          dataset.where(conditions).first
+        end
+
+        def where(conditions)
+          dataset.where(conditions).all
+        end
+
+        def create(attributes)
+          model_class.create(attributes)
+        end
+
+        def update(id, attributes)
+          record = find(id)
+          return nil unless record
+
+          record.update(attributes)
+          record
+        end
+
+        def delete?(id)
+          record = find(id)
+          return false unless record
+
+          record.destroy
+          true
+        end
+
+        def count
+          dataset.count
+        end
+
+        def exists?(conditions)
+          dataset.where(conditions).any?
+        end
+
+        def paginate(page: 1, per_page: 20)
+          dataset
+            .limit(per_page)
+            .offset((page - 1) * per_page)
+            .all
+        end
+
+        protected
+
+        def dataset
+          model_class.dataset
+        end
+
+        def transaction(&)
+          Database.transaction(&)
+        end
+      end
+    end
+  end
+end

data/lib/desiru/persistence/repositories/job_result_repository.rb

@@ -0,0 +1,116 @@
+# frozen_string_literal: true
+
+require_relative 'base_repository'
+
+module Desiru
+  module Persistence
+    module Repositories
+      # Repository for job result persistence
+      class JobResultRepository < BaseRepository
+        def initialize
+          super(Models::JobResult)
+        end
+
+        def create_for_job(job_id, job_class, queue, inputs: nil, expires_at: nil)
+          create(
+            job_id: job_id,
+            job_class: job_class,
+            queue: queue,
+            status: Models::JobResult::STATUS_PENDING,
+            inputs: inputs&.to_json,
+            enqueued_at: Time.now,
+            expires_at: expires_at
+          )
+        end
+
+        def find_by_job_id(job_id)
+          find_by(job_id: job_id)
+        end
+
+        def mark_processing(job_id)
+          job_result = find_by_job_id(job_id)
+          return nil unless job_result
+
+          job_result.mark_as_processing!
+          job_result
+        end
+
+        def mark_completed(job_id, result, message: nil)
+          job_result = find_by_job_id(job_id)
+          return nil unless job_result
+
+          job_result.mark_as_completed!(result, message: message)
+          job_result
+        end
+
+        def mark_failed(job_id, error, backtrace: nil, increment_retry: true)
+          job_result = find_by_job_id(job_id)
+          return nil unless job_result
+
+          updates = {
+            status: Models::JobResult::STATUS_FAILED,
+            finished_at: Time.now,
+            error_message: error.to_s,
+            error_backtrace: backtrace&.join("\n")
+          }
+
+          updates[:retry_count] = job_result.retry_count + 1 if increment_retry
+
+          job_result.update(updates)
+          job_result
+        end
+
+        def update_progress(job_id, progress, message: nil)
+          job_result = find_by_job_id(job_id)
+          return nil unless job_result
+
+          job_result.update_progress(progress, message: message)
+          job_result
+        end
+
+        def cleanup_expired
+          dataset.expired.delete
+        end
+
+        def recent_by_class(job_class, limit: 10)
+          dataset.by_job_class(job_class).recent(limit).all
+        end
+
+        def statistics(job_class: nil, since: nil)
+          scope = dataset
+          scope = scope.by_job_class(job_class) if job_class
+          scope = scope.where { created_at >= since } if since
+
+          {
+            total: scope.count,
+            pending: scope.pending.count,
+            processing: scope.processing.count,
+            completed: scope.completed.count,
+            failed: scope.failed.count,
+            average_duration: calculate_average_duration(scope)
+          }
+        end
+
+        private
+
+        def calculate_average_duration(dataset)
+          completed = dataset.completed.where(Sequel.~(started_at: nil)).where(Sequel.~(finished_at: nil))
+          return 0 if completed.empty?
+
+          total_duration = 0
+          count = 0
+
+          completed.each do |job|
+            next unless job.started_at && job.finished_at
+
+            duration = job.finished_at - job.started_at
+            total_duration += duration
+            count += 1
+          end
+
+          count.positive? ? total_duration / count : 0
+        end
+      end
+    end
+  end
+end