conductor_ruby 0.1.0

data/lib/conductor/worker/task_definition_registrar.rb

@@ -0,0 +1,322 @@
+# frozen_string_literal: true
+
+require 'json'
+require_relative '../http/models/task_def'
+require_relative '../client/metadata_client'
+
+module Conductor
+  module Worker
+    # TaskDefinitionRegistrar - Handles automatic task definition registration
+    # Generates JSON schemas from worker function signatures and registers
+    # task definitions with the Conductor server.
+    class TaskDefinitionRegistrar
+      # @param configuration [Configuration] Conductor configuration
+      # @param logger [Logger] Logger instance
+      def initialize(configuration, logger: nil)
+        @configuration = configuration
+        @metadata_client = Client::MetadataClient.new(configuration)
+        @logger = logger || Logger.new($stdout)
+      end
+
+      # Register a task definition for a worker
+      # @param worker [Worker] The worker instance
+      # @return [Boolean] True if registration succeeded
+      def register(worker)
+        return false unless worker.register_task_def
+
+        task_def = build_task_definition(worker)
+
+        # Generate schemas if worker has typed parameters
+        input_schema = generate_input_schema(worker)
+        output_schema = generate_output_schema(worker)
+
+        # Register schemas if available
+        register_schemas(worker.task_definition_name, input_schema, output_schema) if input_schema || output_schema
+
+        # Register or update task definition
+        if worker.overwrite_task_def
+          register_or_update_task_def(task_def)
+        else
+          register_if_not_exists(task_def)
+        end
+
+        @logger.info("Registered task definition: #{worker.task_definition_name}")
+        true
+      rescue StandardError => e
+        @logger.warn("Failed to register task definition '#{worker.task_definition_name}': #{e.message}")
+        false
+      end
+
+      private
+
+      # Build a TaskDef from worker configuration
+      # @param worker [Worker] The worker instance
+      # @return [TaskDef]
+      def build_task_definition(worker)
+        task_def = worker.task_def_template&.dup || Http::Models::TaskDef.new
+
+        task_def.name = worker.task_definition_name
+
+        # Set reasonable defaults if not provided
+        task_def.retry_count ||= 3
+        task_def.retry_logic ||= Http::Models::TaskDef::RetryLogic::FIXED
+        task_def.timeout_policy ||= Http::Models::TaskDef::TaskTimeoutPolicy::TIME_OUT_WF
+        task_def.timeout_seconds ||= 60
+        task_def.response_timeout_seconds ||= 60
+
+        task_def
+      end
+
+      # Generate JSON Schema for worker input parameters
+      # @param worker [Worker] The worker instance
+      # @return [Hash, nil] JSON Schema or nil
+      def generate_input_schema(worker)
+        return nil unless worker.execute_function.respond_to?(:parameters)
+
+        params = worker.execute_function.parameters
+        return nil if params.empty?
+
+        # Skip if first param is a positional arg (takes full Task object)
+        first_type = params.first&.first
+        return nil if %i[req opt rest].include?(first_type)
+
+        properties = {}
+        required = []
+
+        params.each do |type, name|
+          next unless name
+
+          prop_name = name.to_s
+
+          case type
+          when :keyreq # Required keyword argument
+            properties[prop_name] = infer_property_schema(name)
+            required << prop_name
+          when :key # Optional keyword argument
+            properties[prop_name] = infer_property_schema(name)
+          when :keyrest # **kwargs
+            # Can't generate schema for **kwargs
+            return nil
+          end
+        end
+
+        return nil if properties.empty?
+
+        schema = {
+          '$schema' => 'http://json-schema.org/draft-07/schema#',
+          'type' => 'object',
+          'title' => "#{worker.task_definition_name}_input",
+          'properties' => properties
+        }
+
+        schema['required'] = required unless required.empty?
+        schema['additionalProperties'] = !worker.strict_schema
+
+        schema
+      end
+
+      # Generate JSON Schema for worker output
+      # @param worker [Worker] The worker instance
+      # @return [Hash, nil] JSON Schema or nil
+      def generate_output_schema(_worker)
+        # Output schema is harder to infer without return type annotations
+        # In Ruby, we'd need Sorbet/RBS type annotations
+        # For now, return nil (no output schema)
+        nil
+      end
+
+      # Infer property schema from parameter name
+      # Uses naming conventions to guess types
+      # @param name [Symbol] Parameter name
+      # @return [Hash] Property schema
+      def infer_property_schema(name)
+        name_str = name.to_s.downcase
+
+        # Infer type from naming conventions
+        type = if name_str.end_with?('_id', 'id', '_count', 'count', '_num', 'num', '_index', 'index')
+                 'integer'
+               elsif name_str.end_with?('_at', '_time', '_date')
+                 'string' # ISO8601 date string
+               elsif name_str.start_with?('is_', 'has_', 'can_', 'should_', 'enable')
+                 'boolean'
+               elsif name_str.end_with?('_amount', '_price', '_rate', '_percent')
+                 'number'
+               elsif name_str.end_with?('_list', '_items', '_array', '_ids')
+                 'array'
+               elsif name_str.end_with?('_data', '_config', '_options', '_params', '_payload')
+                 'object'
+               else
+                 'string' # Default to string
+               end
+
+        schema = { 'type' => type }
+
+        # Add format hints for certain types
+        case name_str
+        when /email/
+          schema['format'] = 'email'
+        when /url/, /uri/, /href/
+          schema['format'] = 'uri'
+        when /_at$/, /_time$/
+          schema['format'] = 'date-time'
+        when /_date$/
+          schema['format'] = 'date'
+        when /uuid/, /guid/
+          schema['format'] = 'uuid'
+        end
+
+        schema
+      end
+
+      # Register schemas with the server
+      # @param task_name [String] Task definition name
+      # @param input_schema [Hash, nil] Input schema
+      # @param output_schema [Hash, nil] Output schema
+      def register_schemas(task_name, input_schema, output_schema)
+        # NOTE: Schema registration requires Orkes Conductor
+        # OSS Conductor may not have this endpoint
+
+        if input_schema
+          begin
+            register_schema("#{task_name}_input", input_schema)
+          rescue ApiError => e
+            @logger.debug("Schema registration not available: #{e.message}") if e.status == 404
+          end
+        end
+
+        return unless output_schema
+
+        begin
+          register_schema("#{task_name}_output", output_schema)
+        rescue ApiError => e
+          @logger.debug("Schema registration not available: #{e.message}") if e.status == 404
+        end
+      end
+
+      # Register a single schema
+      # @param name [String] Schema name
+      # @param schema [Hash] JSON Schema
+      def register_schema(name, _schema)
+        # This would call the schema API if available
+        # For now, just log
+        @logger.debug("Would register schema: #{name}")
+      end
+
+      # Register task def, update if already exists
+      # @param task_def [TaskDef]
+      def register_or_update_task_def(task_def)
+        @metadata_client.update_task_def(task_def)
+      rescue ApiError => e
+        raise unless e.status == 404
+
+        # Task def doesn't exist, create it
+        @metadata_client.register_task_def([task_def])
+      end
+
+      # Register task def only if it doesn't exist
+      # @param task_def [TaskDef]
+      def register_if_not_exists(task_def)
+        existing = @metadata_client.get_task_def(task_def.name)
+        @logger.info("Task definition '#{task_def.name}' already exists, skipping")
+      rescue ApiError => e
+        raise unless e.status == 404
+
+        # Task def doesn't exist, create it
+        @metadata_client.register_task_def([task_def])
+      end
+    end
+
+    # JsonSchemaGenerator - Utility for generating JSON Schema from Ruby types
+    # Can be extended to work with Sorbet types or RBS annotations
+    class JsonSchemaGenerator
+      # Ruby type to JSON Schema type mapping
+      TYPE_MAP = {
+        'String' => 'string',
+        'Integer' => 'integer',
+        'Float' => 'number',
+        'Numeric' => 'number',
+        'TrueClass' => 'boolean',
+        'FalseClass' => 'boolean',
+        'Array' => 'array',
+        'Hash' => 'object',
+        'NilClass' => 'null',
+        'Time' => 'string',
+        'Date' => 'string',
+        'DateTime' => 'string'
+      }.freeze
+
+      # Generate JSON Schema from a Ruby value (for inference)
+      # @param value [Object] A sample value
+      # @return [Hash] JSON Schema
+      def self.from_value(value)
+        case value
+        when String
+          { 'type' => 'string' }
+        when Integer
+          { 'type' => 'integer' }
+        when Float
+          { 'type' => 'number' }
+        when TrueClass, FalseClass
+          { 'type' => 'boolean' }
+        when Array
+          if value.empty?
+            { 'type' => 'array' }
+          else
+            { 'type' => 'array', 'items' => from_value(value.first) }
+          end
+        when Hash
+          generate_object_schema(value)
+        when Time, DateTime
+          { 'type' => 'string', 'format' => 'date-time' }
+        when Date
+          { 'type' => 'string', 'format' => 'date' }
+        when NilClass
+          { 'type' => 'null' }
+        else
+          { 'type' => 'object' }
+        end
+      end
+
+      # Generate schema from a hash with sample values
+      # @param hash [Hash] Hash with sample values
+      # @return [Hash] JSON Schema
+      def self.generate_object_schema(hash)
+        properties = {}
+        hash.each do |key, value|
+          properties[key.to_s] = from_value(value)
+        end
+
+        {
+          'type' => 'object',
+          'properties' => properties
+        }
+      end
+
+      # Generate a schema from a Ruby class definition
+      # Works with Struct, Data (Ruby 3.2+), or classes with attr_accessor
+      # @param klass [Class] Ruby class
+      # @return [Hash] JSON Schema
+      def self.from_class(klass)
+        properties = {}
+
+        # Try to get attribute names
+        if klass.respond_to?(:members)
+          # Struct or Data
+          klass.members.each do |attr|
+            properties[attr.to_s] = { 'type' => 'string' }
+          end
+        elsif klass.instance_methods.include?(:to_h)
+          # Has to_h, try to instantiate and inspect
+          # Skip this for safety
+        end
+
+        {
+          '$schema' => 'http://json-schema.org/draft-07/schema#',
+          'type' => 'object',
+          'title' => klass.name,
+          'properties' => properties
+        }
+      end
+    end
+  end
+end

data/lib/conductor/worker/task_handler.rb

@@ -0,0 +1,360 @@
+# frozen_string_literal: true
+
+require 'logger'
+require_relative 'worker'
+require_relative 'worker_registry'
+require_relative 'worker_config'
+require_relative 'task_runner'
+require_relative 'task_definition_registrar'
+require_relative 'events/sync_event_dispatcher'
+require_relative 'events/listener_registry'
+
+module Conductor
+  module Worker
+    # TaskHandler - The top-level orchestrator that manages all workers
+    # Creates one Thread per worker, each running a TaskRunner
+    #
+    # Supports multiple execution modes based on worker configuration:
+    # - :thread (default) - Thread-based with ThreadPoolExecutor
+    # - :ractor - Ractor-based for true parallelism (Ruby 3.1+)
+    # - :fiber - Fiber-based with async gem for high I/O concurrency
+    class TaskHandler
+      attr_reader :workers, :configuration, :event_dispatcher
+
+      # Initialize TaskHandler
+      # @param workers [Array<Worker>, nil] Pre-created worker instances
+      # @param configuration [Configuration, nil] Conductor configuration
+      # @param scan_for_annotated_workers [Boolean] Auto-discover workers from registry
+      # @param import_modules [Array<String>, nil] Ruby files to require (triggers registration)
+      # @param event_listeners [Array<Object>, nil] Custom event listeners
+      # @param logger [Logger, nil] Logger instance
+      # @param register_task_definitions [Boolean] Auto-register task definitions on start
+      def initialize(
+        workers: nil,
+        configuration: nil,
+        scan_for_annotated_workers: true,
+        import_modules: nil,
+        event_listeners: nil,
+        logger: nil,
+        register_task_definitions: false
+      )
+        @configuration = configuration || Configuration.new
+        @logger = logger || create_default_logger
+        @event_dispatcher = Events::SyncEventDispatcher.new
+        @workers = []
+        @threads = []
+        @runners = []
+        @ractors = [] # For Ractor-based workers
+        @running = false
+        @mutex = Mutex.new
+        @register_task_definitions = register_task_definitions
+        @event_listeners = []
+
+        # Register event listeners
+        register_listeners(event_listeners) if event_listeners
+
+        # Import modules (triggers worker_task registrations)
+        import_worker_modules(import_modules) if import_modules
+
+        # Discover workers from registry
+        discover_registered_workers if scan_for_annotated_workers
+
+        # Add provided workers
+        add_workers(workers) if workers
+      end
+
+      # Add workers to the handler
+      # @param workers [Array<Worker>] Workers to add
+      # @return [self]
+      def add_workers(workers)
+        workers.each { |w| add_worker(w) }
+        self
+      end
+
+      # Add a single worker
+      # @param worker [Worker] Worker to add
+      # @return [self]
+      def add_worker(worker)
+        @mutex.synchronize do
+          @workers << worker
+        end
+        self
+      end
+
+      # Start all worker threads
+      # @return [self]
+      def start
+        @mutex.synchronize do
+          return self if @running
+
+          @running = true
+
+          # Register task definitions if enabled
+          register_all_task_definitions if @register_task_definitions
+
+          @workers.each do |worker|
+            start_worker(worker)
+          end
+
+          @logger.info("TaskHandler started with #{@workers.size} workers")
+        end
+
+        self
+      end
+
+      private
+
+      # Start a single worker with appropriate runner type
+      # @param worker [Worker] The worker to start
+      def start_worker(worker)
+        # Determine execution mode from worker configuration
+        isolation = worker.respond_to?(:isolation) ? worker.isolation : :thread
+        executor = worker.respond_to?(:executor) ? worker.executor : :thread_pool
+
+        case isolation
+        when :ractor
+          start_ractor_worker(worker)
+        else
+          # Thread-based execution (default)
+          case executor
+          when :fiber
+            start_fiber_worker(worker)
+          else
+            start_thread_worker(worker)
+          end
+        end
+      end
+
+      # Start a thread-based worker (default mode)
+      # @param worker [Worker] The worker to start
+      def start_thread_worker(worker)
+        runner = TaskRunner.new(
+          worker,
+          configuration: @configuration,
+          event_dispatcher: @event_dispatcher,
+          logger: @logger
+        )
+        @runners << runner
+
+        thread = Thread.new(runner) do |r|
+          Thread.current.name = "conductor-worker-#{r.worker.task_definition_name}"
+          r.run
+        rescue StandardError => e
+          @logger.fatal("Fatal error in worker '#{r.worker.task_definition_name}': #{e.message}")
+          @logger.debug(e.backtrace.join("\n")) if e.backtrace
+        end
+
+        @threads << thread
+      end
+
+      # Start a Ractor-based worker for true parallelism
+      # @param worker [Worker] The worker to start
+      def start_ractor_worker(worker)
+        require_relative 'ractor_task_runner'
+
+        RactorSupport.require_ractors!
+
+        thread_count = worker.respond_to?(:thread_count) ? worker.thread_count : 1
+
+        # Create event receiver Ractor to collect events from worker Ractors
+        event_receiver = create_event_receiver_ractor(worker.task_definition_name)
+
+        # Create multiple Ractors for parallelism
+        thread_count.times do |i|
+          ractor = Ractor.new(worker, @configuration, i, event_receiver) do |w, config, ractor_id, evt_queue|
+            runner = RactorTaskRunner.new(
+              w,
+              configuration: config,
+              ractor_id: ractor_id,
+              event_queue: evt_queue
+            )
+            runner.run
+          end
+          @ractors << ractor
+        end
+
+        @logger.info("Started #{thread_count} Ractor(s) for '#{worker.task_definition_name}'")
+      end
+
+      # Start a fiber-based worker for high I/O concurrency
+      # @param worker [Worker] The worker to start
+      def start_fiber_worker(worker)
+        require_relative 'fiber_executor'
+
+        AsyncSupport.require_async!
+
+        runner = FiberTaskRunner.new(
+          worker,
+          configuration: @configuration,
+          event_dispatcher: @event_dispatcher,
+          logger: @logger
+        )
+        @runners << runner
+
+        thread = Thread.new(runner) do |r|
+          Thread.current.name = "conductor-fiber-#{r.worker.task_definition_name}"
+          r.run
+        rescue StandardError => e
+          @logger.fatal("Fatal error in fiber worker '#{r.worker.task_definition_name}': #{e.message}")
+          @logger.debug(e.backtrace.join("\n")) if e.backtrace
+        end
+
+        @threads << thread
+      end
+
+      # Create event receiver Ractor to forward events to dispatcher
+      # @param task_name [String] Task name for logging
+      # @return [Ractor] Event receiver Ractor
+      def create_event_receiver_ractor(task_name)
+        dispatcher = @event_dispatcher
+        logger = @logger
+
+        Thread.new do
+          Thread.current.name = "conductor-event-receiver-#{task_name}"
+          # NOTE: In production, this would need proper Ractor communication
+          # For now, events from Ractors are logged but not dispatched
+          # due to Ractor isolation constraints
+          logger.debug("Event receiver started for #{task_name}")
+        end
+
+        # Return nil for now - Ractor event communication needs more work
+        nil
+      end
+
+      # Register all task definitions
+      def register_all_task_definitions
+        registrar = TaskDefinitionRegistrar.new(@configuration, logger: @logger)
+        @workers.each do |worker|
+          registrar.register(worker)
+        end
+      end
+
+      public
+
+      # Stop all workers gracefully
+      # @param timeout [Integer] Seconds to wait before force-killing threads
+      # @return [self]
+      def stop(timeout: 5)
+        @mutex.synchronize do
+          return self unless @running
+
+          @logger.info('Stopping TaskHandler...')
+
+          # Signal all runners to shutdown
+          @runners.each(&:shutdown)
+
+          # Wait for threads to finish
+          @threads.each do |thread|
+            thread.join(timeout)
+            thread.kill if thread.alive?
+          end
+
+          # Shutdown Ractors
+          @ractors.each do |ractor|
+            # Ractors don't have a clean shutdown mechanism
+            # They'll be GC'd when no longer referenced
+            ractor.take if ractor.respond_to?(:take)
+          rescue Ractor::ClosedError, Ractor::RemoteError
+            # Ractor already finished
+          end
+
+          @runners.clear
+          @threads.clear
+          @ractors.clear
+
+          stop_event_listeners
+          @running = false
+
+          @logger.info('TaskHandler stopped')
+        end
+
+        self
+      end
+
+      # Wait for all worker threads to complete (blocking)
+      # @return [self]
+      def join
+        @threads.each(&:join)
+        self
+      end
+
+      # Check if handler is running
+      # @return [Boolean]
+      def running?
+        @running
+      end
+
+      # Get list of worker names
+      # @return [Array<String>]
+      def worker_names
+        @workers.map(&:task_definition_name)
+      end
+
+      # Context manager pattern - execute block and stop on exit
+      # @yield [self]
+      # @return [Object] Block return value
+      def self.run(workers: nil, configuration: nil, **options)
+        handler = new(workers: workers, configuration: configuration, **options)
+        begin
+          yield handler if block_given?
+        ensure
+          handler.stop
+        end
+      end
+
+      private
+
+      # Create default logger
+      # @return [Logger]
+      def create_default_logger
+        logger = Logger.new($stdout)
+        logger.level = Logger::INFO
+        logger.formatter = proc do |severity, datetime, _progname, msg|
+          "[#{datetime.strftime('%Y-%m-%d %H:%M:%S')}] #{severity} -- #{msg}\n"
+        end
+        logger
+      end
+
+      # Register event listeners
+      # @param listeners [Array<Object>] Listeners to register
+      def register_listeners(listeners)
+        listeners.each do |listener|
+          @event_listeners << listener
+          Events::ListenerRegistry.register_task_runner_listener(listener, @event_dispatcher)
+        end
+      end
+
+      def stop_event_listeners
+        @event_listeners.each do |listener|
+          listener.stop if listener.respond_to?(:stop)
+        rescue StandardError => e
+          @logger.debug { "Error stopping listener: #{e.class}: #{e.message}" }
+        end
+      end
+
+      # Import worker modules from file paths
+      # @param modules [Array<String>] File paths or module names to require
+      def import_worker_modules(modules)
+        modules.each do |mod|
+          require mod
+        rescue LoadError => e
+          @logger.warn("Failed to load module '#{mod}': #{e.message}")
+        end
+      end
+
+      # Discover workers from the global registry
+      def discover_registered_workers
+        WorkerRegistry.all.each do |definition|
+          worker = Worker.new(
+            definition[:task_definition_name],
+            definition[:execute_function],
+            **definition[:options]
+          )
+          @workers << worker
+        end
+
+        @logger.info("Discovered #{WorkerRegistry.count} workers from registry") if WorkerRegistry.count.positive?
+      end
+    end
+  end
+end