temporalio 0.2.0 → 0.4.0

data/lib/temporalio/worker/tuner.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'temporalio/internal/bridge/worker'
+
 module Temporalio
   class Worker
     # Worker tuner that allows for dynamic customization of some aspects of worker configuration.
@@ -18,6 +20,14 @@ module Temporalio
           def initialize(slots) # rubocop:disable Lint/MissingSuper
             @slots = slots
           end
+
+          # @!visibility private
+          def _to_bridge_options
+            Internal::Bridge::Worker::TunerSlotSupplierOptions.new(
+              fixed_size: slots,
+              resource_based: nil
+            )
+          end
         end
 
         # A slot supplier that will dynamically adjust the number of slots based on resource usage.
@@ -34,6 +44,25 @@ module Temporalio
             @tuner_options = tuner_options
             @slot_options = slot_options
           end
+
+          # @!visibility private
+          def _to_bridge_options
+            Internal::Bridge::Worker::TunerSlotSupplierOptions.new(
+              fixed_size: nil,
+              resource_based: Internal::Bridge::Worker::TunerResourceBasedSlotSupplierOptions.new(
+                target_mem_usage: tuner_options.target_memory_usage,
+                target_cpu_usage: tuner_options.target_cpu_usage,
+                min_slots: slot_options.min_slots,
+                max_slots: slot_options.max_slots,
+                ramp_throttle: slot_options.ramp_throttle
+              )
+            )
+          end
+        end
+
+        # @!visibility private
+        def _to_bridge_options
+          raise ArgumentError, 'Tuner slot suppliers must be instances of Fixed or ResourceBased'
         end
       end
 
@@ -146,6 +175,15 @@ module Temporalio
         @activity_slot_supplier = activity_slot_supplier
         @local_activity_slot_supplier = local_activity_slot_supplier
       end
+
+      # @!visibility private
+      def _to_bridge_options
+        Internal::Bridge::Worker::TunerOptions.new(
+          workflow_slot_supplier: workflow_slot_supplier._to_bridge_options,
+          activity_slot_supplier: activity_slot_supplier._to_bridge_options,
+          local_activity_slot_supplier: local_activity_slot_supplier._to_bridge_options
+        )
+      end
     end
   end
 end

data/lib/temporalio/worker/workflow_executor/thread_pool.rb

@@ -0,0 +1,235 @@
+# frozen_string_literal: true
+
+require 'etc'
+require 'temporalio/internal/bridge/api'
+require 'temporalio/internal/proto_utils'
+require 'temporalio/internal/worker/workflow_instance'
+require 'temporalio/scoped_logger'
+require 'temporalio/worker/thread_pool'
+require 'temporalio/worker/workflow_executor'
+require 'temporalio/workflow'
+require 'temporalio/workflow/definition'
+require 'timeout'
+
+module Temporalio
+  class Worker
+    class WorkflowExecutor
+      # Thread pool implementation of {WorkflowExecutor}.
+      #
+      # Users should use {default} unless they have specific needs to change the thread pool or max threads.
+      class ThreadPool < WorkflowExecutor
+        # @return [ThreadPool] Default executor that lazily constructs an instance with default values.
+        def self.default
+          @default ||= ThreadPool.new
+        end
+
+        # Create a thread pool executor. Most users may prefer {default}.
+        #
+        # @param max_threads [Integer] Maximum number of threads to use concurrently.
+        # @param thread_pool [Worker::ThreadPool] Thread pool to use.
+        def initialize(max_threads: [4, Etc.nprocessors].max, thread_pool: Temporalio::Worker::ThreadPool.default) # rubocop:disable Lint/MissingSuper
+          @max_threads = max_threads
+          @thread_pool = thread_pool
+          @workers_mutex = Mutex.new
+          @workers = []
+          @workers_by_worker_state_and_run_id = {}
+        end
+
+        # @!visibility private
+        def _validate_worker(workflow_worker, worker_state)
+          # Do nothing
+        end
+
+        # @!visibility private
+        def _activate(activation, worker_state, &)
+          # Get applicable worker
+          worker = @workers_mutex.synchronize do
+            run_key = [worker_state, activation.run_id]
+            @workers_by_worker_state_and_run_id.fetch(run_key) do
+              # If not found, get a new one either by creating if not enough or find the one with the fewest.
+              new_worker = if @workers.size < @max_threads
+                             created_worker = Worker.new(self)
+                             @workers << Worker.new(self)
+                             created_worker
+                           else
+                             @workers.min_by(&:workflow_count)
+                           end
+              @workers_by_worker_state_and_run_id[run_key] = new_worker
+              new_worker.workflow_count += 1
+              new_worker
+            end
+          end
+          raise "No worker for run ID #{activation.run_id}" unless worker
+
+          # Enqueue activation
+          worker.enqueue_activation(activation, worker_state, &)
+        end
+
+        # @!visibility private
+        def _thread_pool
+          @thread_pool
+        end
+
+        # @!visibility private
+        def _remove_workflow(worker_state, run_id)
+          @workers_mutex.synchronize do
+            worker = @workers_by_worker_state_and_run_id.delete([worker_state, run_id])
+            if worker
+              worker.workflow_count -= 1
+              # Remove worker from array if done. The array should be small enough that the delete being O(N) is not
+              # worth using a set or a map.
+              if worker.workflow_count.zero?
+                @workers.delete(worker)
+                worker.shutdown
+              end
+            end
+          end
+        end
+
+        # @!visibility private
+        class Worker
+          LOG_ACTIVATIONS = false
+
+          attr_accessor :workflow_count
+
+          def initialize(executor)
+            @executor = executor
+            @workflow_count = 0
+            @queue = Queue.new
+            executor._thread_pool.execute { run }
+          end
+
+          # @!visibility private
+          def enqueue_activation(activation, worker_state, &completion_block)
+            @queue << [:activate, activation, worker_state, completion_block]
+          end
+
+          # @!visibility private
+          def shutdown
+            @queue << [:shutdown]
+          end
+
+          private
+
+          def run
+            loop do
+              work = @queue.pop
+              if work.is_a?(Exception)
+                Warning.warn("Failed activation: #{work}")
+              elsif work.is_a?(Array)
+                case work.first
+                when :shutdown
+                  return
+                when :activate
+                  activate(work[1], work[2], &work[3])
+                end
+              end
+            rescue Exception => e # rubocop:disable Lint/RescueException
+              Warning.warn("Unexpected failure during run: #{e.full_message}")
+            end
+          end
+
+          def activate(activation, worker_state, &)
+            worker_state.logger.debug("Received workflow activation: #{activation}") if LOG_ACTIVATIONS
+
+            # Check whether it has eviction
+            cache_remove_job = activation.jobs.find { |j| !j.remove_from_cache.nil? }&.remove_from_cache
+
+            # If it's eviction only, just evict inline and do nothing else
+            if cache_remove_job && activation.jobs.size == 1
+              evict(worker_state, activation.run_id, cache_remove_job)
+              worker_state.logger.debug('Sending empty workflow completion') if LOG_ACTIVATIONS
+              yield Internal::Bridge::Api::WorkflowCompletion::WorkflowActivationCompletion.new(
+                run_id: activation.run_id,
+                successful: Internal::Bridge::Api::WorkflowCompletion::Success.new
+              )
+              return
+            end
+
+            completion = Timeout.timeout(
+              worker_state.deadlock_timeout,
+              DeadlockError,
+              # TODO(cretz): Document that this affects all running workflows on this worker
+              # and maybe test to see how that is mitigated
+              "[TMPRL1101] Potential deadlock detected: workflow didn't yield " \
+              "within #{worker_state.deadlock_timeout} second(s)."
+            ) do
+              # Get or create workflow
+              instance = worker_state.get_or_create_running_workflow(activation.run_id) do
+                create_instance(activation, worker_state)
+              end
+
+              # Activate. We expect most errors in here to have been captured inside.
+              instance.activate(activation)
+            rescue Exception => e # rubocop:disable Lint/RescueException
+              worker_state.logger.error("Failed activation on workflow run ID: #{activation.run_id}")
+              worker_state.logger.error(e)
+              Internal::Worker::WorkflowInstance.new_completion_with_failure(
+                run_id: activation.run_id,
+                error: e,
+                failure_converter: worker_state.data_converter.failure_converter,
+                payload_converter: worker_state.data_converter.payload_converter
+              )
+            end
+
+            # Go ahead and evict if there is an eviction job
+            evict(worker_state, activation.run_id, cache_remove_job) if cache_remove_job
+
+            # Complete the activation
+            worker_state.logger.debug("Sending workflow completion: #{completion}") if LOG_ACTIVATIONS
+            yield completion
+          end
+
+          def create_instance(initial_activation, worker_state)
+            # Extract start job
+            init_job = initial_activation.jobs.find { |j| !j.initialize_workflow.nil? }&.initialize_workflow
+            raise 'Missing initialize job in initial activation' unless init_job
+
+            # Obtain definition
+            definition = worker_state.workflow_definitions[init_job.workflow_type]
+            # If not present and not reserved, try dynamic
+            if !definition && !Internal::ProtoUtils.reserved_name?(init_job.workflow_type)
+              definition = worker_state.workflow_definitions[nil]
+            end
+
+            unless definition
+              raise Error::ApplicationError.new(
+                "Workflow type #{init_job.workflow_type} is not registered on this worker, available workflows: " +
+                  worker_state.workflow_definitions.keys.compact.sort.join(', '),
+                type: 'NotFoundError'
+              )
+            end
+
+            Internal::Worker::WorkflowInstance.new(
+              Internal::Worker::WorkflowInstance::Details.new(
+                namespace: worker_state.namespace,
+                task_queue: worker_state.task_queue,
+                definition:,
+                initial_activation:,
+                logger: worker_state.logger,
+                metric_meter: worker_state.metric_meter,
+                payload_converter: worker_state.data_converter.payload_converter,
+                failure_converter: worker_state.data_converter.failure_converter,
+                interceptors: worker_state.workflow_interceptors,
+                disable_eager_activity_execution: worker_state.disable_eager_activity_execution,
+                illegal_calls: worker_state.illegal_calls,
+                workflow_failure_exception_types: worker_state.workflow_failure_exception_types,
+                unsafe_workflow_io_enabled: worker_state.unsafe_workflow_io_enabled
+              )
+            )
+          end
+
+          def evict(worker_state, run_id, cache_remove_job)
+            worker_state.evict_running_workflow(run_id, cache_remove_job)
+            @executor._remove_workflow(worker_state, run_id)
+          end
+        end
+
+        private_constant :Worker
+
+        # Error raised when a processing a workflow task takes more than the expected amount of time.
+        class DeadlockError < Exception; end # rubocop:disable Lint/InheritException
+      end
+    end
+  end
+end

data/lib/temporalio/worker/workflow_executor.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require 'temporalio/worker/workflow_executor/thread_pool'
+
+module Temporalio
+  class Worker
+    # Workflow executor that executes workflow tasks. Unlike {ActivityExecutor}, this class is not meant for user
+    # implementation. The only implementation that is currently accepted is {WorkflowExecutor::ThreadPool}.
+    class WorkflowExecutor
+      # @!visibility private
+      def initialize
+        raise 'Cannot create custom executors'
+      end
+
+      # @!visibility private
+      def _validate_worker(workflow_worker, worker_state)
+        raise NotImplementedError
+      end
+
+      # @!visibility private
+      def _activate(activation, worker_state, &)
+        raise NotImplementedError
+      end
+    end
+  end
+end

data/lib/temporalio/worker/workflow_replayer.rb

@@ -0,0 +1,350 @@
+# frozen_string_literal: true
+
+require 'temporalio/api'
+require 'temporalio/converters'
+require 'temporalio/internal/bridge'
+require 'temporalio/internal/bridge/worker'
+require 'temporalio/internal/worker/multi_runner'
+require 'temporalio/internal/worker/workflow_worker'
+require 'temporalio/worker/interceptor'
+require 'temporalio/worker/thread_pool'
+require 'temporalio/worker/tuner'
+require 'temporalio/worker/workflow_executor'
+require 'temporalio/workflow'
+require 'temporalio/workflow_history'
+
+module Temporalio
+  class Worker
+    # Replayer to replay workflows from existing history.
+    class WorkflowReplayer
+      Options = Data.define(
+        :workflows,
+        :namespace,
+        :task_queue,
+        :data_converter,
+        :workflow_executor,
+        :interceptors,
+        :build_id,
+        :identity,
+        :logger,
+        :illegal_workflow_calls,
+        :workflow_failure_exception_types,
+        :workflow_payload_codec_thread_pool,
+        :unsafe_workflow_io_enabled,
+        :debug_mode,
+        :runtime
+      )
+
+      # Options as returned from {options} representing the options passed to the constructor.
+      class Options; end # rubocop:disable Lint/EmptyClass
+
+      # @return [Options] Options for this replayer which has the same attributes as {initialize}.
+      attr_reader :options
+
+      # Create a new replayer. This combines some options from both {Worker.initialize} and {Client.initialize}.
+      #
+      # @param workflows [Array<Class<Workflow::Definition>>] Workflows for this replayer.
+      # @param namespace [String] Namespace as set in the workflow info.
+      # @param task_queue [String] Task queue as set in the workflow info.
+      # @param data_converter [Converters::DataConverter] Data converter to use for all data conversions to/from
+      #   payloads.
+      # @param workflow_executor [WorkflowExecutor] Workflow executor that workflow tasks run within. This must be a
+      #   {WorkflowExecutor::ThreadPool} currently.
+      # @param interceptors [Array<Interceptor::Workflow>] Workflow interceptors.
+      # @param build_id [String] Unique identifier for the current runtime. This is best set as a unique value
+      #   representing all code and should change only when code does. This can be something like a git commit hash. If
+      #   unset, default is hash of known Ruby code.
+      # @param identity [String, nil] Override the identity for this replater.
+      # @param logger [Logger] Logger to use. Defaults to stdout with warn level. Callers setting this logger are
+      #   responsible for closing it.
+      # @param illegal_workflow_calls [Hash<String, [:all, Array<Symbol>]>] Set of illegal workflow calls that are
+      #   considered unsafe/non-deterministic and will raise if seen. The key of the hash is the fully qualified string
+      #   class name (no leading `::`). The value is either `:all` which means any use of the class, or an array of
+      #   symbols for methods on the class that cannot be used. The methods refer to either instance or class methods,
+      #   there is no way to differentiate at this time.
+      # @param workflow_failure_exception_types [Array<Class<Exception>>] Workflow failure exception types. This is the
+      #   set of exception types that, if a workflow-thrown exception extends, will cause the workflow/update to fail
+      #   instead of suspending the workflow via task failure. These are applied in addition to the
+      #   `workflow_failure_exception_type` on the workflow definition class itself. If {::Exception} is set, it
+      #   effectively will fail a workflow/update in all user exception cases.
+      # @param workflow_payload_codec_thread_pool [ThreadPool, nil] Thread pool to run payload codec encode/decode
+      #   within. This is required if a payload codec exists and the worker is not fiber based. Codecs can potentially
+      #   block execution which is why they need to be run in the background.
+      # @param unsafe_workflow_io_enabled [Boolean] If false, the default, workflow code that invokes io_wait on the
+      #   fiber scheduler will fail. Instead of setting this to true, users are encouraged to use
+      #   {Workflow::Unsafe.io_enabled} with a block for narrower enabling of IO.
+      # @param debug_mode [Boolean] If true, deadlock detection is disabled. Deadlock detection will fail workflow tasks
+      #   if they block the thread for too long. This defaults to true if the `TEMPORAL_DEBUG` environment variable is
+      #   `true` or `1`.
+      # @param runtime [Runtime] Runtime for this replayer.
+      #
+      # @yield If a block is present, this is the equivalent of calling {with_replay_worker} with the block and
+      #   discarding the result.
+      def initialize(
+        workflows:,
+        namespace: 'ReplayNamespace',
+        task_queue: 'ReplayTaskQueue',
+        data_converter: Converters::DataConverter.default,
+        workflow_executor: WorkflowExecutor::ThreadPool.default,
+        interceptors: [],
+        build_id: Worker.default_build_id,
+        identity: nil,
+        logger: Logger.new($stdout, level: Logger::WARN),
+        illegal_workflow_calls: Worker.default_illegal_workflow_calls,
+        workflow_failure_exception_types: [],
+        workflow_payload_codec_thread_pool: nil,
+        unsafe_workflow_io_enabled: false,
+        debug_mode: %w[true 1].include?(ENV['TEMPORAL_DEBUG'].to_s.downcase),
+        runtime: Runtime.default,
+        &
+      )
+        @options = Options.new(
+          workflows:,
+          namespace:,
+          task_queue:,
+          data_converter:,
+          workflow_executor:,
+          interceptors:,
+          build_id:,
+          identity:,
+          logger:,
+          illegal_workflow_calls:,
+          workflow_failure_exception_types:,
+          workflow_payload_codec_thread_pool:,
+          unsafe_workflow_io_enabled:,
+          debug_mode:,
+          runtime:
+        ).freeze
+        # Preload definitions and other settings
+        @workflow_definitions = Internal::Worker::WorkflowWorker.workflow_definitions(workflows)
+        @nondeterminism_as_workflow_fail, @nondeterminism_as_workflow_fail_for_types =
+          Internal::Worker::WorkflowWorker.bridge_workflow_failure_exception_type_options(
+            workflow_failure_exception_types:, workflow_definitions: @workflow_definitions
+          )
+        # If there is a block, we'll go ahead and assume it's for with_replay_worker
+        with_replay_worker(&) if block_given? # steep:ignore
+      end
+
+      # Replay a workflow history.
+      #
+      # If doing multiple histories, it is better to use {replay_workflows} or {with_replay_worker} since they create
+      # a replay worker just once instead of each time like this call does.
+      #
+      # @param history [WorkflowHistory] History to replay.
+      # @param raise_on_replay_failure [Boolean] If true, the default, this will raise an exception on any replay
+      #   failure. If false and the replay fails, the failure will be available in {ReplayResult.replay_failure}.
+      #
+      # @return [ReplayResult] Result of the replay.
+      def replay_workflow(history, raise_on_replay_failure: true)
+        with_replay_worker { |worker| worker.replay_workflow(history, raise_on_replay_failure:) }
+      end
+
+      # Replay multiple workflow histories.
+      #
+      # @param histories [Enumerable<WorkflowHistory>] Histories to replay.
+      # @param raise_on_replay_failure [Boolean] If true, this will raise an exception on any replay failure. If false,
+      #   the default, and the replay fails, the failure will be available in {ReplayResult.replay_failure}.
+      #
+      # @return [Array<ReplayResult>] Results of the replay.
+      def replay_workflows(histories, raise_on_replay_failure: false)
+        with_replay_worker do |worker|
+          histories.map { |h| worker.replay_workflow(h, raise_on_replay_failure:) }
+        end
+      end
+
+      # Run a block of code with a {ReplayWorker} to execute replays.
+      #
+      # @yield Block of code to run with a replay worker.
+      # @yieldparam [ReplayWorker] Worker to run replays on. Note, only one workflow can replay at a time.
+      # @yieldreturn [Object] Result of the block.
+      def with_replay_worker(&)
+        worker = ReplayWorker.new(
+          options:,
+          workflow_definitions: @workflow_definitions,
+          nondeterminism_as_workflow_fail: @nondeterminism_as_workflow_fail,
+          nondeterminism_as_workflow_fail_for_types: @nondeterminism_as_workflow_fail_for_types
+        )
+        begin
+          yield worker
+        ensure
+          worker._shutdown
+        end
+      end
+
+      # Result of a single workflow replay run.
+      class ReplayResult
+        # @return [WorkflowHistory] History originally passed in to the replayer.
+        attr_reader :history
+
+        # @return [Exception, nil] Failure during replay if any.
+        attr_reader :replay_failure
+
+        # @!visibility private
+        def initialize(history:, replay_failure:)
+          @history = history
+          @replay_failure = replay_failure
+        end
+      end
+
+      # Replay worker that can be used to replay individual workflow runs. Only one call to {replay_workflow} can be
+      # made at a time.
+      class ReplayWorker
+        # @!visibility private
+        def initialize(
+          options:,
+          workflow_definitions:,
+          nondeterminism_as_workflow_fail:,
+          nondeterminism_as_workflow_fail_for_types:
+        )
+          # Create the bridge worker and the replayer
+          @bridge_replayer, @bridge_worker = Internal::Bridge::Worker::WorkflowReplayer.new(
+            options.runtime._core_runtime,
+            Internal::Bridge::Worker::Options.new(
+              activity: false,
+              workflow: true,
+              namespace: options.namespace,
+              task_queue: options.task_queue,
+              tuner: Tuner.create_fixed(
+                workflow_slots: 2, activity_slots: 1, local_activity_slots: 1
+              )._to_bridge_options,
+              build_id: options.build_id,
+              identity_override: options.identity,
+              max_cached_workflows: 2,
+              max_concurrent_workflow_task_polls: 1,
+              nonsticky_to_sticky_poll_ratio: 1.0,
+              max_concurrent_activity_task_polls: 1,
+              no_remote_activities: true,
+              sticky_queue_schedule_to_start_timeout: 1.0,
+              max_heartbeat_throttle_interval: 1.0,
+              default_heartbeat_throttle_interval: 1.0,
+              max_worker_activities_per_second: nil,
+              max_task_queue_activities_per_second: nil,
+              graceful_shutdown_period: 0.0,
+              use_worker_versioning: false,
+              nondeterminism_as_workflow_fail:,
+              nondeterminism_as_workflow_fail_for_types:
+            )
+          )
+
+          # Create the workflow worker
+          @workflow_worker = Internal::Worker::WorkflowWorker.new(
+            bridge_worker: @bridge_worker,
+            namespace: options.namespace,
+            task_queue: options.task_queue,
+            workflow_definitions:,
+            workflow_executor: options.workflow_executor,
+            logger: options.logger,
+            data_converter: options.data_converter,
+            metric_meter: options.runtime.metric_meter,
+            workflow_interceptors: options.interceptors.select do |i|
+              i.is_a?(Interceptor::Workflow)
+            end,
+            disable_eager_activity_execution: false,
+            illegal_workflow_calls: options.illegal_workflow_calls,
+            workflow_failure_exception_types: options.workflow_failure_exception_types,
+            workflow_payload_codec_thread_pool: options.workflow_payload_codec_thread_pool,
+            unsafe_workflow_io_enabled: options.unsafe_workflow_io_enabled,
+            debug_mode: options.debug_mode,
+            on_eviction: proc { |_, remove_job| @last_workflow_remove_job = remove_job } # steep:ignore
+          )
+
+          # Create the runner
+          @runner = Internal::Worker::MultiRunner.new(workers: [self], shutdown_signals: [])
+        end
+
+        # Replay a workflow history.
+        #
+        # @param history [WorkflowHistory] History to replay.
+        # @param raise_on_replay_failure [Boolean] If true, the default, this will raise an exception on any replay
+        #   failure. If false and the replay fails, the failure will be available in {ReplayResult.replay_failure}.
+        #
+        # @return [ReplayResult] Result of the replay.
+        def replay_workflow(history, raise_on_replay_failure: true)
+          raise ArgumentError, 'Expected history as WorkflowHistory' unless history.is_a?(WorkflowHistory)
+          # Due to our event processing model, only one can run at a time
+          raise 'Already running' if @running
+          raise 'Replayer shutdown' if @shutdown
+
+          # Push history proto
+          # TODO(cretz): Unset this
+          @running = true
+          @last_workflow_remove_job = nil
+          begin
+            @bridge_replayer.push_history(
+              history.workflow_id, Api::History::V1::History.new(events: history.events).to_proto
+            )
+
+            # Process events until workflow complete
+            until @last_workflow_remove_job
+              event = @runner.next_event
+              case event
+              when Internal::Worker::MultiRunner::Event::PollSuccess
+                @workflow_worker.handle_activation(
+                  runner: @runner,
+                  activation: Internal::Bridge::Api::WorkflowActivation::WorkflowActivation.decode(event.bytes),
+                  decoded: false
+                )
+              when Internal::Worker::MultiRunner::Event::WorkflowActivationDecoded
+                @workflow_worker.handle_activation(runner: @runner, activation: event.activation, decoded: true)
+              when Internal::Worker::MultiRunner::Event::WorkflowActivationComplete
+                @workflow_worker.handle_activation_complete(
+                  runner: @runner,
+                  activation_completion: event.activation_completion,
+                  encoded: event.encoded,
+                  completion_complete_queue: event.completion_complete_queue
+                )
+              when Internal::Worker::MultiRunner::Event::WorkflowActivationCompletionComplete
+              # Ignore
+              else
+                raise "Unexpected event: #{event}"
+              end
+            end
+
+            # Create exception if removal is due to error
+            err = if @last_workflow_remove_job.reason == :NONDETERMINISM
+                    Workflow::NondeterminismError.new(
+                      "#{@last_workflow_remove_job.reason}: #{@last_workflow_remove_job.message}"
+                    )
+                  elsif !%i[CACHE_FULL LANG_REQUESTED].include?(@last_workflow_remove_job.reason)
+                    Workflow::InvalidWorkflowStateError.new(
+                      "#{@last_workflow_remove_job.reason}: #{@last_workflow_remove_job.message}"
+                    )
+                  end
+            # Raise if wanting to raise, otherwise return result
+            raise err if raise_on_replay_failure && err
+
+            ReplayResult.new(history:, replay_failure: err)
+          ensure
+            @running = false
+          end
+        end
+
+        # @!visibility private
+        def _shutdown
+          @shutdown = true
+          @runner.initiate_shutdown
+          # Wait for all-pollers-shutdown before finalizing
+          until @runner.next_event.is_a?(Internal::Worker::MultiRunner::Event::AllPollersShutDown); end
+          @runner.wait_complete_and_finalize_shutdown
+          @workflow_worker.on_shutdown_complete
+          @workflow_worker = nil
+        end
+
+        # @!visibility private
+        def _bridge_worker
+          @bridge_worker
+        end
+
+        # @!visibility private
+        def _initiate_shutdown
+          _bridge_worker.initiate_shutdown
+        end
+
+        # @!visibility private
+        def _wait_all_complete
+          # Do nothing
+        end
+      end
+    end
+  end
+end