busybee 0.1.0 → 0.3.0

data/lib/busybee/worker/configuration.rb

@@ -0,0 +1,287 @@
+# frozen_string_literal: true
+
+require "active_support/core_ext/string/inflections"
+
+module Busybee
+  class Worker
+    # Stores all DSL-declared metadata for a Worker subclass.
+    # Lazily instantiated per worker class via Worker.configuration.
+    class Configuration # rubocop:disable Metrics/ClassLength
+      VALID_TYPES = %w[string integer decimal boolean datetime duration uuid null].freeze
+      VALID_SOURCES = %i[variable header].freeze
+      VALID_WORKER_MODES = %i[polling streaming hybrid].freeze
+      VALID_POLLING_KWARGS = %i[max_jobs request_timeout].freeze
+      VALID_STREAMING_KWARGS = %i[buffer buffer_throttle].freeze
+
+      # Represents a declared input (from variable, header, or both).
+      Input = Struct.new(:name, :source, :required, :type, :description, :default,
+                         :accessor_name, :define_accessor, keyword_init: true)
+
+      # Represents a declared output returned from perform.
+      Output = Struct.new(:name, :required, :type, :description, keyword_init: true)
+
+      attr_accessor :description
+      attr_reader :inputs, :outputs, :worker_mode, :polling_config, :streaming_config,
+                  :job_timeout, :backoff, :backpressure_delay,
+                  :complete_job_on_success, :fail_job_on_error, :shutdown_on
+
+      def initialize(worker_class)
+        @worker_class = worker_class
+        @job_type = nil
+        @description = nil
+        @inputs = []
+        @outputs = []
+        @worker_mode = nil
+        @polling_config = {}
+        @streaming_config = {}
+        @job_timeout = nil
+        @backoff = nil
+        @backpressure_delay = nil
+        @complete_job_on_success = true
+        @fail_job_on_error = true
+        @shutdown_on = []
+      end
+
+      def job_type
+        @job_type || derive_default_job_type
+      end
+
+      def job_type=(value)
+        @job_type = value.to_s
+      end
+
+      def worker_mode=(value)
+        sym = value.to_sym
+        unless VALID_WORKER_MODES.include?(sym)
+          raise InvalidWorkerDefinition,
+                "Invalid worker mode #{value.inspect}. Valid: #{VALID_WORKER_MODES.map(&:inspect).join(', ')}"
+        end
+
+        @worker_mode = sym
+      end
+
+      def polling_config=(kwargs)
+        unknown = kwargs.keys - VALID_POLLING_KWARGS
+        if unknown.any?
+          raise InvalidWorkerDefinition,
+                "Unknown polling config: #{unknown.map(&:inspect).join(', ')}. " \
+                "Valid: #{VALID_POLLING_KWARGS.map(&:inspect).join(', ')}"
+        end
+
+        @polling_config = kwargs
+      end
+
+      def streaming_config=(kwargs)
+        unknown = kwargs.keys - VALID_STREAMING_KWARGS
+        if unknown.any?
+          raise InvalidWorkerDefinition,
+                "Unknown streaming config: #{unknown.map(&:inspect).join(', ')}. " \
+                "Valid: #{VALID_STREAMING_KWARGS.map(&:inspect).join(', ')}"
+        end
+
+        validate_buffer_option!(kwargs) if kwargs.key?(:buffer)
+        validate_no_throttle_without_buffer!(kwargs)
+        validate_buffer_throttle!(kwargs) if kwargs.key?(:buffer_throttle)
+
+        @streaming_config = kwargs
+      end
+
+      def job_timeout=(value)
+        validate_duration!(:job_timeout, value)
+        @job_timeout = value
+      end
+
+      def backoff=(value)
+        validate_duration!(:backoff, value)
+        @backoff = value
+      end
+
+      def backpressure_delay=(value)
+        validate_duration!(:backpressure_delay, value)
+        @backpressure_delay = value
+      end
+
+      def complete_job_on_success=(value)
+        unless [true, false].include?(value)
+          raise InvalidWorkerDefinition, "`complete_job_on_success` requires a boolean, got #{value.inspect}"
+        end
+
+        @complete_job_on_success = value
+      end
+
+      def fail_job_on_error=(value)
+        unless [true, false].include?(value)
+          raise InvalidWorkerDefinition, "`fail_job_on_error` requires a boolean, got #{value.inspect}"
+        end
+
+        @fail_job_on_error = value
+      end
+
+      def add_shutdown_on(*exception_classes)
+        exception_classes.each do |klass|
+          unless klass.is_a?(Class) && klass <= Exception
+            raise InvalidWorkerDefinition,
+                  "`shutdown_on` expects exception classes, got #{klass.inspect}"
+          end
+        end
+
+        @shutdown_on |= exception_classes
+      end
+
+      def add_input(input)
+        validate_name!(input.name, "input")
+        validate_source!(input)
+        validate_type!(input.name, input.type, "input") if input.type
+        validate_accessor_options!(input)
+        validate_unique_input!(input.name)
+
+        @inputs << input
+      end
+
+      def add_output(output)
+        validate_name!(output.name, "output")
+        validate_type!(output.name, output.type, "output") if output.type
+        validate_unique_output!(output.name)
+
+        @outputs << output
+      end
+
+      # Resolved buffer throttle for the streaming pump thread.
+      # Returns false (no throttling), 0 (minimal throttle), or a positive Numeric (ms).
+      def buffer_throttle
+        streaming_config.fetch(:buffer_throttle, Busybee.default_buffer_throttle)
+      end
+
+      # Whether this worker uses a pump thread + buffer for streaming.
+      # Default: true. Set to false via `streaming buffer: false` for inline stream processing.
+      def buffer?
+        streaming_config.fetch(:buffer, Busybee::Defaults::DEFAULT_STREAMING_BUFFER)
+      end
+
+      # Returns resolved polling options for client.with_each_job, merging
+      # DSL overrides with gem-level defaults.
+      def polling_options
+        {
+          max_jobs: polling_config[:max_jobs] || Busybee::Defaults::DEFAULT_MAX_JOBS,
+          request_timeout: polling_config[:request_timeout] || Busybee.default_job_request_timeout,
+          job_timeout: job_timeout || Busybee.default_job_lock_timeout
+        }
+      end
+
+      # Returns resolved streaming options for client.open_job_stream.
+      def streaming_options
+        { job_timeout: job_timeout || Busybee.default_job_lock_timeout }
+      end
+
+      def to_h
+        {
+          job_type: job_type,
+          description: description,
+          inputs: inputs.map(&:to_h),
+          outputs: outputs.map(&:to_h),
+          worker_mode: worker_mode,
+          polling_config: polling_config,
+          streaming_config: streaming_config,
+          job_timeout: job_timeout,
+          backoff: backoff,
+          backpressure_delay: backpressure_delay,
+          complete_job_on_success: complete_job_on_success,
+          fail_job_on_error: fail_job_on_error,
+          shutdown_on: shutdown_on
+        }
+      end
+
+      private
+
+      def derive_default_job_type
+        class_name = @worker_class.name
+        return "worker" if class_name.nil?
+
+        last_segment = class_name.split("::").last
+        last_segment = last_segment.delete_suffix("Worker") if last_segment != "Worker"
+        last_segment.underscore
+      end
+
+      def validate_name!(name, kind)
+        raise InvalidWorkerDefinition, "Name is required for all #{kind}s" if name.nil? || name.to_s.strip.empty?
+      end
+
+      def validate_source!(input)
+        sources = Array(input.source)
+        raise InvalidWorkerDefinition, "`source:` is required for input :#{input.name}" if sources.empty?
+
+        invalid = sources - VALID_SOURCES
+        unless invalid.empty?
+          raise InvalidWorkerDefinition,
+                "Invalid source #{invalid.map(&:inspect).join(', ')} for input :#{input.name}. " \
+                "Valid: #{VALID_SOURCES.map(&:inspect).join(', ')}"
+        end
+
+        # Deduplicate sources (e.g., [:variable, :variable] → [:variable])
+        input.source = sources.uniq
+      end
+
+      def validate_type!(name, type, kind)
+        return if VALID_TYPES.include?(type.to_s)
+
+        raise InvalidWorkerDefinition,
+              "Invalid type #{type.inspect} for #{kind} :#{name}. " \
+              "Valid: #{VALID_TYPES.join(', ')}"
+      end
+
+      def validate_accessor_options!(input)
+        return unless !input.define_accessor && input.accessor_name
+
+        raise InvalidWorkerDefinition,
+              "`define_accessor: false` and `accessor_name:` are mutually exclusive on input :#{input.name} " \
+              "— no accessor will be defined, so naming it is meaningless"
+      end
+
+      def validate_unique_input!(name)
+        return unless @inputs.any? { |i| i.name == name }
+
+        raise InvalidWorkerDefinition, "Input :#{name} is already declared"
+      end
+
+      def validate_unique_output!(name)
+        return unless @outputs.any? { |o| o.name == name }
+
+        raise InvalidWorkerDefinition, "Output :#{name} is already declared"
+      end
+
+      def validate_buffer_option!(kwargs)
+        return if [true, false].include?(kwargs[:buffer])
+
+        raise InvalidWorkerDefinition, "`buffer:` requires a boolean, got #{kwargs[:buffer].inspect}"
+      end
+
+      def validate_no_throttle_without_buffer!(kwargs)
+        return unless kwargs[:buffer] == false && kwargs.key?(:buffer_throttle)
+
+        raise InvalidWorkerDefinition,
+              "`buffer_throttle:` cannot be set when `buffer: false` — there is no buffer to throttle"
+      end
+
+      def validate_buffer_throttle!(kwargs)
+        # Coerce: true → 0 ("enable at minimal setting"), nil → false ("no throttling")
+        kwargs[:buffer_throttle] = 0 if kwargs[:buffer_throttle] == true
+        kwargs[:buffer_throttle] = false if kwargs[:buffer_throttle].nil?
+
+        value = kwargs[:buffer_throttle]
+        return if value == false
+        return if value.is_a?(Numeric) && value >= 0
+
+        raise InvalidWorkerDefinition,
+              "`buffer_throttle:` must be a non-negative Numeric, got #{value.inspect}"
+      end
+
+      def validate_duration!(attr, value)
+        return if value.is_a?(Integer)
+        return if defined?(ActiveSupport::Duration) && value.is_a?(ActiveSupport::Duration)
+
+        raise InvalidWorkerDefinition,
+              "`#{attr}` accepts Integer (milliseconds) or ActiveSupport::Duration, got #{value.class}"
+      end
+    end
+  end
+end

data/lib/busybee/worker/dsl.rb

@@ -0,0 +1,187 @@
+# frozen_string_literal: true
+
+require "busybee/worker/configuration"
+
+module Busybee
+  class Worker
+    # Class-level DSL for declaring worker metadata and configuration.
+    # Extended into Worker subclasses.
+    module DSL
+      # Sentinel for distinguishing "not passed" from "passed as nil".
+      NOT_SET = Object.new.freeze
+      private_constant :NOT_SET
+
+      # Ruby keywords that can't be used as bare method calls (obj.keyword is a syntax error).
+      RUBY_KEYWORDS = %w[
+        __FILE__ __LINE__ __ENCODING__
+        BEGIN END
+        alias and begin break case class def defined? do
+        else elsif end ensure false for if in module next
+        nil not or redo rescue retry return self super
+        then true undef unless until when while yield
+      ].freeze
+      private_constant :RUBY_KEYWORDS
+
+      def configuration
+        @configuration ||= Configuration.new(self)
+      end
+
+      def job_type(value = nil)
+        if value.nil?
+          configuration.job_type
+        else
+          configuration.job_type = value
+        end
+      end
+
+      def description(value = nil)
+        if value.nil?
+          configuration.description
+        else
+          configuration.description = value
+        end
+      end
+
+      def input(name, source:, required: NOT_SET, type: nil, description: nil, # rubocop:disable Metrics/ParameterLists
+                default: NOT_SET, accessor_name: nil, define_accessor: true)
+        validate_required_default_exclusivity!(name, required, default)
+
+        input_struct = Configuration::Input.new(
+          name: name.to_sym,
+          source: normalize_source(source),
+          required: resolve_required(required, default),
+          type: type,
+          description: description,
+          default: default == NOT_SET ? nil : default,
+          accessor_name: accessor_name&.to_sym,
+          define_accessor: define_accessor
+        )
+        configuration.add_input(input_struct)
+        define_input_accessor(input_struct) if define_accessor
+      end
+
+      def variable(name, **)
+        input(name, source: :variable, **)
+      end
+
+      def header(name, **)
+        input(name, source: :header, **)
+      end
+
+      def output(name, required: NOT_SET, type: nil, description: nil)
+        output_struct = Configuration::Output.new(
+          name: name.to_sym,
+          required: required == NOT_SET ? Busybee.default_output_required : !!required,
+          type: type,
+          description: description
+        )
+        configuration.add_output(output_struct)
+      end
+
+      def worker_mode(value)
+        configuration.worker_mode = value
+      end
+
+      def polling(**kwargs)
+        configuration.polling_config = kwargs
+      end
+
+      def streaming(**kwargs)
+        configuration.streaming_config = kwargs
+      end
+
+      def job_timeout(value)
+        configuration.job_timeout = value
+      end
+
+      def backoff(value)
+        configuration.backoff = value
+      end
+
+      def backpressure_delay(value = nil)
+        if value.nil?
+          configuration.backpressure_delay
+        else
+          configuration.backpressure_delay = value
+        end
+      end
+
+      def complete_job_on_success(value = nil)
+        if value.nil?
+          configuration.complete_job_on_success
+        else
+          configuration.complete_job_on_success = value
+        end
+      end
+
+      def fail_job_on_error(value = nil)
+        if value.nil?
+          configuration.fail_job_on_error
+        else
+          configuration.fail_job_on_error = value
+        end
+      end
+
+      def shutdown_on(*exception_classes)
+        if exception_classes.empty?
+          configuration.shutdown_on
+        else
+          configuration.add_shutdown_on(*exception_classes)
+        end
+      end
+
+      private
+
+      def normalize_source(source)
+        Array(source).map(&:to_sym)
+      end
+
+      def resolve_required(required, default)
+        # If default is provided and required wasn't explicitly set, input is not required
+        return false if required == NOT_SET && default != NOT_SET
+
+        required == NOT_SET ? Busybee.default_input_required : !!required
+      end
+
+      def validate_required_default_exclusivity!(name, required, default)
+        return unless required != NOT_SET && default != NOT_SET
+
+        raise InvalidWorkerDefinition,
+              "`required:` and `default:` are mutually exclusive on input :#{name} " \
+              "— a default makes the input always available, so `required:` is meaningless"
+      end
+
+      def define_input_accessor(input)
+        method_name = input.accessor_name || input.name
+        validate_accessor_name!(method_name)
+
+        sources = Array(input.source)
+        default_value = input.default
+
+        define_method(method_name) do
+          sources.each do |src|
+            v = public_send(:"#{src}s")[input.name.to_s]
+            return v unless v.nil?
+          end
+          default_value&.clone
+        end
+      end
+
+      def validate_accessor_name!(name)
+        str_name = name.to_s
+
+        if RUBY_KEYWORDS.include?(str_name)
+          raise InvalidWorkerDefinition,
+                "Cannot define accessor :#{name} — it's a Ruby keyword. " \
+                "Use `accessor_name:` or `define_accessor: false`"
+        end
+
+        return unless method_defined?(name) || private_method_defined?(name)
+
+        raise InvalidWorkerDefinition,
+              "A method :#{name} already exists on #{self.name || 'this worker'}. " \
+              "Use `accessor_name:` or `define_accessor: false`"
+      end
+    end
+  end
+end

data/lib/busybee/worker/shutdown.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+require "busybee/error"
+
+module Busybee
+  class Worker
+    # Raised when a `shutdown_on` exception is caught during perform_job.
+    # Signals to the Runner that the worker process should shut down.
+    # The original exception is available via `cause` (set by Ruby at raise time).
+    class Shutdown < Busybee::Error
+      attr_reader :worker_class
+
+      def initialize(message = "Shutting down worker #{Busybee.worker_name}", worker:)
+        @worker_class = worker
+        super(message)
+      end
+
+      def message
+        super.dup.tap do |msg|
+          msg << " due to #{cause&.class&.name || 'error'}"
+          msg << " in #{worker_class.name}" if worker_class&.name
+          msg << ": \"#{cause.message}\"" if cause
+        end
+      end
+    end
+  end
+end

data/lib/busybee/worker.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+require "active_support/core_ext/module/delegation"
+require "busybee/worker/configuration"
+require "busybee/worker/dsl"
+require "busybee/worker/shutdown"
+
+module Busybee
+  # Base class for defining job workers.
+  #
+  # Subclass and implement `perform` to handle jobs from the workflow engine.
+  # Runners call `perform_job(job)` as the entry point (internal use only).
+  #
+  # @example Minimal worker
+  #   class ProcessOrderWorker < Busybee::Worker
+  #     job_type "process-order"
+  #
+  #     def perform
+  #       order = Order.find(variables[:order_id])
+  #       order.process!
+  #       complete!(status: order.status)
+  #     end
+  #   end
+  #
+  class Worker
+    extend DSL
+
+    attr_reader :job
+
+    delegate :variables, :headers, :complete!, :fail!, :throw_bpmn_error!,
+             :update_retries, :update_timeout, to: :job
+
+    def initialize(job)
+      @job = job
+    end
+
+    def perform
+      raise NotImplementedError,
+            "#{self.class.name} must implement `perform`"
+    end
+
+    class << self
+      # Entry point called by Runners. Instantiates the worker, validates inputs,
+      # calls perform, and applies lifecycle behaviors (auto-complete, auto-fail, shutdown).
+      #
+      # Contract with Runners:
+      # - Returns normally: job was handled, runner continues.
+      # - Raises Shutdown: worker is unhealthy, runner should shut down.
+      #
+      # @param job [Busybee::Job] An activated job from the workflow engine
+      # @return [Object] The return value of perform (useful for testing)
+      # @raise [Busybee::Worker::Shutdown] if a shutdown_on exception is caught
+      def perform_job(job)
+        config = configuration
+        instance = new(job)
+        # [hook: job.started]
+        begin
+          validate_inputs!(instance, config)
+          result = instance.perform
+          handle_success(job, result, config)
+          result
+        rescue StandardError => e
+          handle_failure(job, e, config)
+          raise if e.is_a?(Shutdown)
+          raise Shutdown.new(worker: self) if shutdown_error?(e, config)
+
+          log_unhandled_error(job, e) unless config.fail_job_on_error
+        ensure # rubocop:disable Lint/EmptyEnsure
+          # [hook: job.finished]
+        end
+      end
+
+      private
+
+      def validate_inputs!(instance, config)
+        missing = config.inputs.select(&:required).reject { |input| input_present?(instance, input) }
+        return if missing.empty?
+
+        names = missing.map { |i| ":#{i.name}" }.join(", ")
+        raise Busybee::MissingInput, "Missing required inputs for #{configuration.job_type} worker: #{names}"
+      end
+
+      def input_present?(instance, input)
+        Array(input.source).any? { |src| !instance.public_send(:"#{src}s")[input.name.to_s].nil? }
+      end
+
+      def handle_success(job, result, config)
+        return unless config.complete_job_on_success && job.ready?
+
+        new_vars = result.is_a?(Hash) ? result : {}
+        validate_outputs!(new_vars, config)
+        begin
+          job.complete!(new_vars)
+        rescue StandardError => e
+          Busybee.logger&.warn("Failed to complete job #{job.key}: #{e.message}. Job will timeout and retry.")
+        end
+      end
+
+      def validate_outputs!(result, config)
+        missing = config.outputs.select(&:required).reject { |o| result.key?(o.name) || result.key?(o.name.to_s) }
+        return if missing.empty?
+
+        names = missing.map { |o| ":#{o.name}" }.join(", ")
+        raise Busybee::MissingOutput, "Missing required outputs for #{configuration.job_type} worker: #{names}"
+      end
+
+      def handle_failure(job, error, config)
+        return unless config.fail_job_on_error && job.ready?
+
+        fail_with = error.is_a?(Shutdown) ? (error.cause || error) : error
+        begin
+          job.fail!(fail_with, backoff: config.backoff)
+        rescue StandardError => e
+          Busybee.logger&.warn("Failed to fail job #{job.key}: #{e.message}. Job will timeout and retry.")
+        end
+      end
+
+      def shutdown_error?(error, config)
+        (config.shutdown_on + Busybee.shutdown_on_errors).any? { |klass| error.is_a?(klass) }
+      end
+
+      def log_unhandled_error(job, error)
+        Busybee.logger&.warn(
+          "Unhandled error in #{configuration.job_type} worker for job #{job.key} " \
+          "(fail_job_on_error is off): [#{error.class}] #{error.message}. Job will timeout and retry."
+        )
+      end
+    end
+  end
+end