desiru 0.1.0 → 0.1.1

data/lib/desiru/graphql/schema_generator.rb

@@ -2,6 +2,8 @@
 
 require 'graphql'
 require_relative 'data_loader'
+require_relative 'batch_loader'
+require_relative 'type_builder'
 
 module Desiru
   module GraphQL
@@ -12,11 +14,16 @@ module Desiru
       def initialize
         @signatures = {}
         @modules = {}
-        @type_cache = {}
+        @type_cache = {} # Instance cache for schema-specific types
         @schema_class = nil
         @data_loader = DataLoader.new
       end
 
+      # Clear the global type cache (useful for testing or reloading)
+      def self.clear_type_cache!
+        TypeBuilder.clear_type_cache!
+      end
+
       # Register a signature with a name for GraphQL query/mutation
       def register_signature(name, signature)
         @signatures[name] = signature
@@ -36,47 +43,68 @@ module Desiru
 
       # Generate a GraphQL schema from registered signatures
       def generate_schema
-        return @schema_class if @schema_class && @signatures.empty?
+        # Always rebuild if signatures have changed
+        return @schema_class if @schema_class && @last_signature_count == @signatures.size
 
+        @last_signature_count = @signatures.size
         query_class = build_query_type
 
         @schema_class = Class.new(::GraphQL::Schema) do
           query(query_class) if query_class
+
+          # Enable GraphQL's built-in dataloader
+          use ::GraphQL::Dataloader
+
+          # Enable lazy execution for batch loading
+          lazy_resolve(::GraphQL::Execution::Lazy, :value)
         end
 
         @schema_class
       end
 
+      def add_query_field(query_class, field_name, field_def)
+        # Add field directly without resolver class
+        query_class.field field_name, field_def[:type],
+                          null: false,
+                          description: field_def[:description] do
+          # Add arguments
+          field_def[:arguments].each do |arg_name, arg_def|
+            argument arg_name, arg_def[:type], required: arg_def[:required]
+          end
+        end
+
+        # Define the resolver method for this field
+        query_class.define_method field_name do |**args|
+          if field_def[:module_instance].respond_to?(:batch_forward)
+            # Get the dataloader for this request
+            dataloader = context.dataloader
+            # Load through the dataloader
+            dataloader.with(
+              Desiru::GraphQL::ModuleLoader,
+              field_name,
+              field_def[:modules]
+            ).load(args)
+          else
+            # Direct execution
+            field_def[:resolver].call(args, context)
+          end
+        end
+      end
+
       private
 
       def build_query_type
         return nil if @signatures.empty?
 
         query_fields = build_query_fields
+        query_class_builder = self
 
         Class.new(::GraphQL::Schema::Object) do
           graphql_name 'Query'
           description 'Desiru query operations'
 
           query_fields.each do |field_name, field_def|
-            # Create a resolver class for each field
-            resolver_class = Class.new(::GraphQL::Schema::Resolver) do
-              # Set the return type
-              type field_def[:type], null: false
-
-              # Add arguments
-              field_def[:arguments].each do |arg_name, arg_def|
-                argument arg_name, arg_def[:type], required: arg_def[:required]
-              end
-
-              # Define resolve method
-              define_method :resolve do |**args|
-                field_def[:resolver].call(args)
-              end
-            end
-
-            # Add field with resolver
-            field field_name, resolver: resolver_class, description: field_def[:description]
+            query_class_builder.add_query_field(self, field_name, field_def)
           end
         end
       end
@@ -85,12 +113,12 @@ module Desiru
         fields = {}
 
         @signatures.each do |operation_name, signature|
-          output_type = build_output_type(signature)
+          output_type = TypeBuilder.build_output_type(signature)
 
           arguments = {}
           signature.input_fields.each do |field_name, field|
             arguments[camelcase_field_name(field_name)] = {
-              type: graphql_type_for_field(field),
+              type: TypeBuilder.graphql_type_for_field(field),
               required: !field.optional
             }
           end
@@ -99,7 +127,9 @@ module Desiru
             type: output_type,
             description: "Generated from signature: #{signature.raw_signature}",
             arguments: arguments,
-            resolver: ->(args) { execute_signature(operation_name, signature, args) }
+            resolver: ->(args, context) { execute_signature(operation_name, signature, args, context) },
+            module_instance: @modules[operation_name],
+            modules: @modules
           }
         end
 
@@ -111,151 +141,25 @@ module Desiru
         nil
       end
 
-      def build_output_type(signature)
-        type_name = "Output#{signature.object_id}"
-        return @type_cache[type_name] if @type_cache[type_name]
-
-        output_field_defs = {}
-        signature.output_fields.each do |field_name, field|
-          output_field_defs[camelcase_field_name(field_name)] = {
-            type: graphql_type_for_field(field),
-            null: field.optional,
-            description: field.description
-          }
-        end
-
-        output_type = Class.new(::GraphQL::Schema::Object) do
-          graphql_name type_name
-          description 'Generated output type'
-
-          output_field_defs.each do |field_name, field_def|
-            field field_name, field_def[:type],
-                  null: field_def[:null],
-                  description: field_def[:description]
-          end
-        end
-
-        @type_cache[type_name] = output_type
-      end
-
-      def graphql_type_for_field(field)
-        base_type = case field.type
-                    when :string
-                      ::GraphQL::Types::String
-                    when :int, :integer
-                      ::GraphQL::Types::Int
-                    when :float
-                      ::GraphQL::Types::Float
-                    when :bool, :boolean
-                      ::GraphQL::Types::Boolean
-                    when :list
-                      # Handle list types
-                      element_type = graphql_type_for_element(field.element_type)
-                      [element_type]
-                    when :literal
-                      # Create enum type for literal values
-                      create_enum_type(field)
-                    else
-                      ::GraphQL::Types::String
-                    end
-
-        if field.optional
-          base_type
-        else
-          # Arrays are already wrapped, so handle them differently
-          base_type.is_a?(Array) ? [base_type.first, { null: false }] : base_type.to_non_null_type
-        end
-      end
-
-      def graphql_type_for_element(element_type)
-        case element_type
-        when Hash
-          # Handle typed arrays like List[Literal['yes', 'no']]
-          if element_type[:type] == :literal
-            create_enum_type_from_values(element_type[:values])
-          else
-            ::GraphQL::Types::String
-          end
-        else
-          # Simple types
-          case element_type
-          when :string then ::GraphQL::Types::String
-          when :int, :integer then ::GraphQL::Types::Int
-          when :float then ::GraphQL::Types::Float
-          when :bool, :boolean then ::GraphQL::Types::Boolean
-          else ::GraphQL::Types::String
-          end
-        end
-      end
-
-      def create_enum_type(field)
-        enum_name = "#{field.name.to_s.capitalize}Enum"
-        return @type_cache[enum_name] if @type_cache[enum_name]
-
-        # Extract literal values from the field's validator
-        values = extract_literal_values(field)
-
-        enum_type = Class.new(::GraphQL::Schema::Enum) do
-          graphql_name enum_name
-          description "Enum for #{field.name}"
-
-          values.each do |val|
-            value val.upcase.gsub(/[^A-Z0-9_]/, '_'), value: val
-          end
-        end
-
-        @type_cache[enum_name] = enum_type
-      end
-
-      def create_enum_type_from_values(values)
-        enum_name = "Literal#{values.map(&:capitalize).join}Enum"
-        return @type_cache[enum_name] if @type_cache[enum_name]
-
-        enum_type = Class.new(::GraphQL::Schema::Enum) do
-          graphql_name enum_name
-
-          values.each do |val|
-            value val.upcase.gsub(/[^A-Z0-9_]/, '_'), value: val
-          end
-        end
-
-        @type_cache[enum_name] = enum_type
-      end
-
-      def extract_literal_values(field)
-        # Try to extract values from the field's validator
-        if field.respond_to?(:validator) && field.validator.respond_to?(:instance_variable_get)
-          field.validator.instance_variable_get(:@values) || []
-        elsif field.respond_to?(:element_type) && field.element_type.is_a?(Hash)
-          field.element_type[:values] || []
-        else
-          []
-        end
-      end
-
-      def execute_signature(operation_name, signature, args)
+      def execute_signature(operation_name, signature, args, context = {})
         # Convert GraphQL arguments from camelCase to snake_case
         inputs = transform_graphql_args(args)
 
+        # Get data loader from context if available
+        context[:data_loader] || @data_loader
+
         # Check if we have a registered module for this operation
         if @modules[operation_name]
-          # Use DataLoader for batch optimization
-          loader = @data_loader.for(@modules[operation_name].class)
-          promise = loader.load(inputs)
-
-          # In a real GraphQL implementation, this would be handled by the executor
-          # For now, we'll resolve immediately
-          result = promise.value
+          module_instance = @modules[operation_name]
 
-          # Transform module result to GraphQL response format
-          transform_module_result(result)
+          # Direct execution - batching will be handled by the executor
         else
           # Fallback: create a module instance on the fly
           module_class = infer_module_class(signature)
           module_instance = module_class.new(signature)
-          result = module_instance.call(inputs)
-          transform_module_result(result)
         end
+        result = module_instance.call(inputs)
+        transform_module_result(result)
       end
 
       def transform_graphql_args(args)
@@ -294,7 +198,7 @@ module Desiru
         # Remove trailing '?' for optional fields
         clean_name = field_name.to_s.gsub('?', '')
         parts = clean_name.split('_')
-        parts[0] + parts[1..-1].map(&:capitalize).join
+        parts[0] + parts[1..].map(&:capitalize).join
       end
     end
   end

data/lib/desiru/graphql/type_builder.rb

@@ -0,0 +1,138 @@
+# frozen_string_literal: true
+
+require 'graphql'
+require_relative 'enum_builder'
+require_relative 'type_cache_warmer'
+
+module Desiru
+  module GraphQL
+    # Handles GraphQL type generation and caching
+    module TypeBuilder
+      extend self
+
+      @type_cache = {}
+      @cache_mutex = Mutex.new
+
+      class << self
+        attr_accessor :type_cache, :cache_mutex
+      end
+
+      def clear_type_cache!
+        @cache_mutex.synchronize do
+          @type_cache.clear
+        end
+      end
+
+      def build_output_type(signature)
+        # Create a stable cache key based on signature structure
+        cache_key = generate_type_cache_key('Output', signature.output_fields)
+
+        # Check cache first
+        @cache_mutex.synchronize do
+          return @type_cache[cache_key] if @type_cache[cache_key]
+        end
+
+        output_field_defs = build_field_definitions(signature.output_fields)
+
+        output_type = Class.new(::GraphQL::Schema::Object) do
+          graphql_name "Output#{cache_key.hash.abs}"
+          description 'Generated output type'
+
+          output_field_defs.each do |field_name, field_def|
+            field field_name, field_def[:type],
+                  null: field_def[:null],
+                  description: field_def[:description]
+          end
+        end
+
+        # Store in cache
+        @cache_mutex.synchronize do
+          @type_cache[cache_key] = output_type
+        end
+
+        output_type
+      end
+
+      def graphql_type_for_field(field)
+        case field.type
+        when :int, :integer
+          ::GraphQL::Types::Int
+        when :float
+          ::GraphQL::Types::Float
+        when :bool, :boolean
+          ::GraphQL::Types::Boolean
+        when :list
+          # Handle list types
+          element_type = graphql_type_for_element(field.element_type)
+          [element_type]
+        when :literal
+          # Create enum type for literal values
+          EnumBuilder.create_enum_type(field, @type_cache, @cache_mutex)
+        else
+          ::GraphQL::Types::String
+        end
+      end
+
+      private
+
+      def build_field_definitions(fields)
+        field_defs = {}
+        fields.each do |field_name, field|
+          field_defs[camelcase_field_name(field_name)] = {
+            type: graphql_type_for_field(field),
+            null: field.optional,
+            description: field.description
+          }
+        end
+        field_defs
+      end
+
+      def graphql_type_for_element(element_type)
+        case element_type
+        when Hash
+          # Handle typed arrays like List[Literal['yes', 'no']]
+          if element_type[:type] == :literal
+            EnumBuilder.create_enum_from_values(element_type[:values], @type_cache, @cache_mutex)
+          else
+            ::GraphQL::Types::String
+          end
+        else
+          # Simple types
+          element_to_graphql_type(element_type)
+        end
+      end
+
+      def element_to_graphql_type(element_type)
+        case element_type
+        when :int, :integer then ::GraphQL::Types::Int
+        when :float then ::GraphQL::Types::Float
+        when :bool, :boolean then ::GraphQL::Types::Boolean
+        else ::GraphQL::Types::String
+        end
+      end
+
+      def camelcase_field_name(field_name)
+        # Convert snake_case to camelCase for GraphQL conventions
+        # Remove trailing '?' for optional fields
+        clean_name = field_name.to_s.gsub('?', '')
+        parts = clean_name.split('_')
+        parts[0] + parts[1..].map(&:capitalize).join
+      end
+
+      def generate_type_cache_key(prefix, fields)
+        # Generate a stable cache key based on field structure
+        # Use a more efficient approach with less string concatenation
+        field_data = fields.map do |name, field|
+          components = [name.to_s, field.type.to_s]
+          components << (field.optional ? 'T' : 'F')
+          components << field.element_type.to_s if field.respond_to?(:element_type)
+          components
+        end.sort
+
+        # Use hash for more compact cache keys
+        content_hash = field_data.hash.abs
+        "#{prefix}:#{content_hash}"
+      end
+    end
+  end
+end

data/lib/desiru/graphql/type_cache_warmer.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+module Desiru
+  module GraphQL
+    # Utility for warming the GraphQL type cache with common types
+    module TypeCacheWarmer
+      extend self
+
+      # Pre-generate commonly used types to improve cold-start performance
+      def warm_common_types!
+        warm_common_output_types!
+        warm_common_enums!
+      end
+
+      # Get statistics about the type cache
+      def cache_stats
+        cache = TypeBuilder.instance_variable_get(:@type_cache)
+        mutex = TypeBuilder.instance_variable_get(:@cache_mutex)
+
+        mutex.synchronize do
+          {
+            total_types: cache.size,
+            output_types: cache.keys.count { |k| k.start_with?('Output:') },
+            enum_types: cache.keys.count { |k| k.include?('Enum') },
+            cache_keys: cache.keys
+          }
+        end
+      end
+
+      private
+
+      def warm_common_output_types!
+        common_field_sets.each do |fields|
+          signature = create_mock_signature(fields)
+          TypeBuilder.build_output_type(signature)
+        end
+      end
+
+      def common_field_sets
+        [
+          # Single field types
+          { id: create_field(:string, false) },
+          { result: create_field(:string, true) },
+          { output: create_field(:string, true) },
+          { message: create_field(:string, true) },
+          {
+            id: create_field(:string, false),
+            result: create_field(:string, true),
+            timestamp: create_field(:float, true)
+          },
+          {
+            output: create_field(:string, true),
+            confidence: create_field(:float, true),
+            reasoning: create_field(:string, true)
+          },
+          {
+            success: create_field(:bool, false),
+            message: create_field(:string, true),
+            data: create_field(:string, true)
+          }
+        ]
+      end
+
+      def create_field(type, optional)
+        Struct.new(:type, :optional, :description).new(type, optional, nil)
+      end
+
+      def create_mock_signature(fields)
+        Struct.new(:output_fields).new(fields)
+      end
+
+      def warm_common_enums!
+        # Common enum value sets
+        common_enum_values = [
+          %w[pending processing completed failed],
+          %w[low medium high critical],
+          %w[yes no maybe],
+          %w[active inactive suspended],
+          %w[draft published archived]
+        ]
+
+        cache = TypeBuilder.instance_variable_get(:@type_cache)
+        mutex = TypeBuilder.instance_variable_get(:@cache_mutex)
+
+        common_enum_values.each do |values|
+          EnumBuilder.create_enum_from_values(values, cache, mutex)
+        end
+      end
+    end
+  end
+end

data/lib/desiru/jobs/async_predict.rb

@@ -9,7 +9,7 @@ module Desiru
 
       def perform(job_id, module_class_name, signature_str, inputs, options = {})
         update_status(job_id, 'running', message: 'Initializing module')
-        
+
         module_class = Object.const_get(module_class_name)
 
         # Extract module initialization parameters

data/lib/desiru/jobs/base.rb

@@ -18,7 +18,11 @@ module Desiru
       protected
 
       def store_result(job_id, result, ttl: 3600)
+        # Store in Redis for fast access
         redis.setex(result_key(job_id), ttl, result.to_json)
+
+        # Also persist to database for long-term storage
+        persist_result_to_db(job_id, result)
       end
 
       def fetch_result(job_id)
@@ -43,11 +47,74 @@ module Desiru
         status_data[:message] = message if message
 
         redis.setex(status_key(job_id), 86_400, status_data.to_json)
+
+        # Also persist to database
+        persist_status_to_db(job_id, status, progress: progress, message: message)
       end
 
       def status_key(job_id)
         "desiru:status:#{job_id}"
       end
+
+      # Database persistence methods
+      def persist_result_to_db(job_id, result)
+        return unless persistence_enabled?
+
+        job_repo.mark_completed(job_id, result)
+      rescue StandardError => e
+        Desiru.logger.warn("Failed to persist job result to database: #{e.message}")
+      end
+
+      def persist_error_to_db(job_id, error, backtrace = nil)
+        return unless persistence_enabled?
+
+        job_repo.mark_failed(job_id, error, backtrace: backtrace)
+      rescue StandardError => e
+        Desiru.logger.warn("Failed to persist job error to database: #{e.message}")
+      end
+
+      def persist_status_to_db(job_id, status, progress: nil, message: nil)
+        return unless persistence_enabled?
+
+        case status
+        when 'processing'
+          job_repo.mark_processing(job_id)
+          # Also update progress if provided
+          job_repo.update_progress(job_id, progress, message: message) if progress
+        when 'completed'
+          # Already handled by persist_result_to_db
+        when 'failed'
+          # Already handled by persist_error_to_db
+        else
+          job_repo.update_progress(job_id, progress, message: message) if progress
+        end
+      rescue StandardError => e
+        Desiru.logger.warn("Failed to persist job status to database: #{e.message}")
+      end
+
+      def create_job_record(job_id, inputs: nil, expires_at: nil)
+        return unless persistence_enabled?
+
+        job_repo.create_for_job(
+          job_id,
+          self.class.name,
+          self.class.get_sidekiq_options['queue'] || 'default',
+          inputs: inputs,
+          expires_at: expires_at
+        )
+      rescue StandardError => e
+        Desiru.logger.warn("Failed to create job record in database: #{e.message}")
+      end
+
+      def persistence_enabled?
+        Desiru::Persistence.enabled?
+      rescue StandardError
+        false
+      end
+
+      def job_repo
+        @job_repo ||= Desiru::Persistence.repositories[:job_results]
+      end
     end
   end
 end

data/lib/desiru/jobs/batch_processor.rb

@@ -10,7 +10,7 @@ module Desiru
       def perform(batch_id, module_class_name, signature_str, inputs_array, options = {})
         total_items = inputs_array.size
         update_status(batch_id, 'running', progress: 0, message: "Processing #{total_items} items")
-        
+
         module_class = Object.const_get(module_class_name)
 
         # Extract module initialization parameters
@@ -34,9 +34,9 @@ module Desiru
 
         inputs_array.each_with_index do |inputs, index|
           progress = ((index + 1).to_f / total_items * 100).round
-          update_status(batch_id, 'running', progress: progress, 
-                       message: "Processing item #{index + 1} of #{total_items}")
-          
+          update_status(batch_id, 'running', progress: progress,
+                                             message: "Processing item #{index + 1} of #{total_items}")
+
           result = module_instance.call(**inputs)
           results << {
             index: index,
@@ -53,8 +53,8 @@ module Desiru
         end
 
         final_status = errors.empty? ? 'completed' : 'completed_with_errors'
-        update_status(batch_id, final_status, progress: 100, 
-                     message: "Processed #{results.size} successfully, #{errors.size} failed")
+        update_status(batch_id, final_status, progress: 100,
+                                              message: "Processed #{results.size} successfully, #{errors.size} failed")
 
         store_result(batch_id, {
                        success: errors.empty?,