temporal-ruby 0.0.1.pre.pre1 → 0.0.1

data/lib/temporal/workflow/history/event_target.rb

@@ -69,6 +69,10 @@ module Temporal
         def hash
           [id, type].hash
         end
+
+        def to_s
+          "#{type} (#{id})"
+        end
       end
     end
   end

data/lib/temporal/workflow/history/window.rb

@@ -28,7 +28,7 @@ module Temporal
             @local_time = nil
           when 'WORKFLOW_TASK_COMPLETED'
             @replay = true
-          when 'WORKFLOW_TASK_SCHEDULED', 'WORKFLOW_TASK_FAILED'
+          when 'WORKFLOW_TASK_SCHEDULED'
             # no-op
           else
             events << event

data/lib/temporal/workflow/poller.rb

@@ -1,16 +1,23 @@
 require 'temporal/client'
+require 'temporal/thread_pool'
 require 'temporal/middleware/chain'
 require 'temporal/workflow/task_processor'
+require 'temporal/error_handler'
 
 module Temporal
   class Workflow
     class Poller
-      def initialize(namespace, task_queue, workflow_lookup, middleware = [])
+      DEFAULT_OPTIONS = {
+        thread_pool_size: 10
+      }.freeze
+
+      def initialize(namespace, task_queue, workflow_lookup, middleware = [], options = {})
         @namespace = namespace
         @task_queue = task_queue
         @workflow_lookup = workflow_lookup
         @middleware = middleware
         @shutting_down = false
+        @options = DEFAULT_OPTIONS.merge(options)
       end
 
       def start
@@ -18,50 +25,71 @@ module Temporal
         @thread = Thread.new(&method(:poll_loop))
       end
 
-      def stop
+      def stop_polling
         @shutting_down = true
-        Thread.new { Temporal.logger.info('Shutting down a workflow poller') }.join
+        Temporal.logger.info('Shutting down a workflow poller')
+      end
+
+      def cancel_pending_requests
+        client.cancel_polling_request
       end
 
       def wait
-        @thread.join
+        thread.join
+        thread_pool.shutdown
       end
 
       private
 
-      attr_reader :namespace, :task_queue, :client, :workflow_lookup, :middleware
+      attr_reader :namespace, :task_queue, :workflow_lookup, :middleware, :options, :thread
 
       def client
         @client ||= Temporal::Client.generate
       end
 
-      def middleware_chain
-        @middleware_chain ||= Middleware::Chain.new(middleware)
-      end
-
       def shutting_down?
         @shutting_down
       end
 
       def poll_loop
-        while !shutting_down? do
-          Temporal.logger.debug("Polling worklow task queue (#{namespace} / #{task_queue})")
+        last_poll_time = Time.now
+        metrics_tags = { namespace: namespace, task_queue: task_queue }.freeze
+
+        loop do
+          thread_pool.wait_for_available_threads
+
+          return if shutting_down?
+
+          time_diff_ms = ((Time.now - last_poll_time) * 1000).round
+          Temporal.metrics.timing('workflow_poller.time_since_last_poll', time_diff_ms, metrics_tags)
+          Temporal.logger.debug("Polling Worklow task queue", { namespace: namespace, task_queue: task_queue })
 
           task = poll_for_task
-          process(task) if task&.workflow_type
+          last_poll_time = Time.now
+          next unless task&.workflow_type
+
+          thread_pool.schedule { process(task) }
         end
       end
 
       def poll_for_task
         client.poll_workflow_task_queue(namespace: namespace, task_queue: task_queue)
       rescue StandardError => error
-        Temporal.logger.error("Unable to poll workflow task queue: #{error.inspect}")
+        Temporal.logger.error("Unable to poll Workflow task queue", { namespace: namespace, task_queue: task_queue, error: error.inspect })
+        Temporal::ErrorHandler.handle(error)
+
         nil
       end
 
       def process(task)
+        middleware_chain = Middleware::Chain.new(middleware)
+
         TaskProcessor.new(task, namespace, workflow_lookup, client, middleware_chain).process
       end
+
+      def thread_pool
+        @thread_pool ||= ThreadPool.new(options[:thread_pool_size])
+      end
     end
   end
 end

data/lib/temporal/workflow/replay_aware_logger.rb

@@ -11,17 +11,17 @@ module Temporal
       end
 
       SEVERITIES.each do |severity|
-        define_method severity do |message|
+        define_method severity do |message, data = {}|
           return if replay?
 
-          main_logger.public_send(severity, message)
+          main_logger.public_send(severity, message, data)
         end
       end
 
-      def log(severity, message)
+      def log(severity, message, data = {})
         return if replay?
 
-        main_logger.log(severity, message)
+        main_logger.log(severity, message, data)
       end
 
       private

data/lib/temporal/workflow/state_manager.rb

@@ -1,13 +1,17 @@
-require 'temporal/json'
+require 'set'
 require 'temporal/errors'
 require 'temporal/workflow/command'
 require 'temporal/workflow/command_state_machine'
 require 'temporal/workflow/history/event_target'
 require 'temporal/metadata'
+require 'temporal/concerns/payloads'
+require 'temporal/workflow/errors'
 
 module Temporal
   class Workflow
     class StateManager
+      include Concerns::Payloads
+
       SIDE_EFFECT_MARKER = 'SIDE_EFFECT'.freeze
       RELEASE_MARKER = 'RELEASE'.freeze
 
@@ -101,7 +105,7 @@ module Temporal
           dispatch(
             History::EventTarget.workflow,
             'started',
-            parse_payload(event.attributes.input),
+            from_payloads(event.attributes.input),
             Metadata.generate(Metadata::WORKFLOW_TYPE, event.attributes)
           )
 
@@ -131,26 +135,26 @@ module Temporal
 
         when 'ACTIVITY_TASK_SCHEDULED'
           state_machine.schedule
-          discard_command(event.originating_event_id)
+          discard_command(target)
 
         when 'ACTIVITY_TASK_STARTED'
           state_machine.start
 
         when 'ACTIVITY_TASK_COMPLETED'
           state_machine.complete
-          dispatch(target, 'completed', parse_payload(event.attributes.result))
+          dispatch(target, 'completed', from_result_payloads(event.attributes.result))
 
         when 'ACTIVITY_TASK_FAILED'
           state_machine.fail
-          dispatch(target, 'failed', parse_failure(event.attributes.failure, ActivityException))
+          dispatch(target, 'failed', Temporal::Workflow::Errors.generate_error(event.attributes.failure, ActivityException))
 
         when 'ACTIVITY_TASK_TIMED_OUT'
           state_machine.time_out
-          dispatch(target, 'failed', parse_failure(event.attributes.failure))
+          dispatch(target, 'failed', Temporal::Workflow::Errors.generate_error(event.attributes.failure))
 
         when 'ACTIVITY_TASK_CANCEL_REQUESTED'
           state_machine.requested
-          discard_command(event.originating_event_id)
+          discard_command(target)
 
         when 'REQUEST_CANCEL_ACTIVITY_TASK_FAILED'
           state_machine.fail
@@ -158,11 +162,11 @@ module Temporal
 
         when 'ACTIVITY_TASK_CANCELED'
           state_machine.cancel
-          dispatch(target, 'failed', parse_failure(event.attributes.failure))
+          dispatch(target, 'failed', Temporal::Workflow::Errors.generate_error(event.attributes.failure))
 
         when 'TIMER_STARTED'
           state_machine.start
-          discard_command(event.originating_event_id)
+          discard_command(target)
 
         when 'TIMER_FIRED'
           state_machine.complete
@@ -193,10 +197,10 @@ module Temporal
 
         when 'MARKER_RECORDED'
           state_machine.complete
-          handle_marker(event.id, event.attributes.marker_name, parse_payload(event.attributes.details['data']))
+          handle_marker(event.id, event.attributes.marker_name, from_details_payloads(event.attributes.details['data']))
 
         when 'WORKFLOW_EXECUTION_SIGNALED'
-          dispatch(target, 'signaled', event.attributes.signal_name, parse_payload(event.attributes.input))
+          dispatch(target, 'signaled', event.attributes.signal_name, from_signal_payloads(event.attributes.input))
 
         when 'WORKFLOW_EXECUTION_TERMINATED'
           # todo
@@ -206,30 +210,30 @@ module Temporal
 
         when 'START_CHILD_WORKFLOW_EXECUTION_INITIATED'
           state_machine.schedule
-          discard_command(event.originating_event_id)
+          discard_command(target)
 
         when 'START_CHILD_WORKFLOW_EXECUTION_FAILED'
           state_machine.fail
-          dispatch(target, 'failed', 'StandardError', parse_payload(event.attributes.cause))
+          dispatch(target, 'failed', 'StandardError', from_payloads(event.attributes.cause))
 
         when 'CHILD_WORKFLOW_EXECUTION_STARTED'
           state_machine.start
 
         when 'CHILD_WORKFLOW_EXECUTION_COMPLETED'
           state_machine.complete
-          dispatch(target, 'completed', parse_payload(event.attributes.result))
+          dispatch(target, 'completed', from_result_payloads(event.attributes.result))
 
         when 'CHILD_WORKFLOW_EXECUTION_FAILED'
           state_machine.fail
-          dispatch(target, 'failed', parse_failure(event.attributes.failure))
+          dispatch(target, 'failed', Temporal::Workflow::Errors.generate_error(event.attributes.failure))
 
         when 'CHILD_WORKFLOW_EXECUTION_CANCELED'
           state_machine.cancel
-          dispatch(target, 'failed', parse_failure(event.attributes.failure))
+          dispatch(target, 'failed', Temporal::Workflow::Errors.generate_error(event.attributes.failure))
 
         when 'CHILD_WORKFLOW_EXECUTION_TIMED_OUT'
           state_machine.time_out
-          dispatch(target, 'failed', parse_failure(event.attributes.failure))
+          dispatch(target, 'failed', Temporal::Workflow::Errors.generate_error(event.attributes.failure))
 
         when 'CHILD_WORKFLOW_EXECUTION_TERMINATED'
           # todo
@@ -277,8 +281,18 @@ module Temporal
         dispatcher.dispatch(target, name, attributes)
       end
 
-      def discard_command(command_id)
-        commands.delete_if { |(id, _)| id == command_id }
+      def discard_command(target)
+        # Pop the first command from the list, it is expected to match
+        existing_command_id, existing_command = commands.shift
+
+        if !existing_command_id
+          raise NonDeterministicWorkflowError, "A command #{target} was not scheduled upon replay"
+        end
+
+        existing_target = event_target_from(existing_command_id, existing_command)
+        if target != existing_target
+          raise NonDeterministicWorkflowError, "Unexpected command #{existing_target} (expected #{target})"
+        end
       end
 
       def handle_marker(id, type, details)
@@ -304,39 +318,6 @@ module Temporal
         end
       end
 
-      def parse_payload(payload)
-        return if payload.nil? || payload.payloads.empty?
-
-        binary = payload.payloads.first.data
-        JSON.deserialize(binary)
-      end
-
-      def parse_failure(failure, default_exception_class = StandardError)
-        case failure.failure_info
-        when :application_failure_info
-          exception_class = safe_constantize(failure.application_failure_info.type)
-          exception_class ||= default_exception_class
-          details = parse_payload(failure.application_failure_info.details)
-          backtrace = failure.stack_trace.split("\n")
-
-          exception_class.new(details).tap do |exception|
-            exception.set_backtrace(backtrace) if !backtrace.empty?
-          end
-        when :timeout_failure_info
-          TimeoutError.new("Timeout type: #{failure.timeout_failure_info.timeout_type.to_s}")
-        when :canceled_failure_info
-          # TODO: Distinguish between different entity cancellations
-          StandardError.new(parse_payload(failure.canceled_failure_info.details))
-        else
-          StandardError.new(failure.message)
-        end
-      end
-
-      def safe_constantize(const)
-        Object.const_get(const) if Object.const_defined?(const)
-      rescue NameError
-        nil
-      end
     end
   end
 end

data/lib/temporal/workflow/task_processor.rb

@@ -1,14 +1,18 @@
 require 'temporal/workflow/executor'
 require 'temporal/workflow/history'
 require 'temporal/metadata'
+require 'temporal/error_handler'
 require 'temporal/errors'
 
 module Temporal
   class Workflow
     class TaskProcessor
+      MAX_FAILED_ATTEMPTS = 1
+
       def initialize(task, namespace, workflow_lookup, client, middleware_chain)
         @task = task
         @namespace = namespace
+        @metadata = Metadata.generate(Metadata::WORKFLOW_TASK_TYPE, task, namespace)
         @task_token = task.task_token
         @workflow_name = task.workflow_type.name
         @workflow_class = workflow_lookup.find(workflow_name)
@@ -19,37 +23,35 @@ module Temporal
       def process
         start_time = Time.now
 
-        Temporal.logger.info("Processing a workflow task for #{workflow_name}")
+        Temporal.logger.debug("Processing Workflow task", metadata.to_h)
         Temporal.metrics.timing('workflow_task.queue_time', queue_time_ms, workflow: workflow_name)
 
         if !workflow_class
           raise Temporal::WorkflowNotRegistered, 'Workflow is not registered with this worker'
         end
 
-        history = Workflow::History.new(task.history.events)
+        history = fetch_full_history
         # TODO: For sticky workflows we need to cache the Executor instance
         executor = Workflow::Executor.new(workflow_class, history)
-        metadata = Metadata.generate(Metadata::WORKFLOW_TASK_TYPE, task, namespace)
 
         commands = middleware_chain.invoke(metadata) do
           executor.run
         end
 
         complete_task(commands)
-      rescue Temporal::ClientError => error
-        fail_task(error)
       rescue StandardError => error
-        Temporal.logger.error("Workflow task for #{workflow_name} failed with: #{error.inspect}")
-        Temporal.logger.debug(error.backtrace.join("\n"))
+        Temporal::ErrorHandler.handle(error, metadata: metadata)
+
+        fail_task(error)
       ensure
         time_diff_ms = ((Time.now - start_time) * 1000).round
         Temporal.metrics.timing('workflow_task.latency', time_diff_ms, workflow: workflow_name)
-        Temporal.logger.debug("Workflow task processed in #{time_diff_ms}ms")
+        Temporal.logger.debug("Workflow task processed", metadata.to_h.merge(execution_time: time_diff_ms))
       end
 
       private
 
-      attr_reader :task, :namespace, :task_token, :workflow_name, :workflow_class, :client, :middleware_chain
+      attr_reader :task, :namespace, :task_token, :workflow_name, :workflow_class, :client, :middleware_chain, :metadata
 
       def queue_time_ms
         scheduled = task.scheduled_time.to_f
@@ -57,21 +59,49 @@ module Temporal
         ((started - scheduled) * 1_000).round
       end
 
+      def fetch_full_history
+        events = task.history.events.to_a
+        next_page_token = task.next_page_token
+
+        while !next_page_token.empty? do
+          response = client.get_workflow_execution_history(
+            namespace: namespace,
+            workflow_id: task.workflow_execution.workflow_id,
+            run_id: task.workflow_execution.run_id,
+            next_page_token: next_page_token
+          )
+
+          events += response.history.events.to_a
+          next_page_token = response.next_page_token
+        end
+
+        Workflow::History.new(events)
+      end
+
       def complete_task(commands)
-        Temporal.logger.info("Workflow task for #{workflow_name} completed")
+        Temporal.logger.info("Workflow task completed", metadata.to_h)
 
         client.respond_workflow_task_completed(task_token: task_token, commands: commands)
       end
 
       def fail_task(error)
-        Temporal.logger.error("Workflow task for #{workflow_name} failed with: #{error.inspect}")
+        Temporal.logger.error("Workflow task failed", metadata.to_h.merge(error: error.inspect))
         Temporal.logger.debug(error.backtrace.join("\n"))
 
+        # Only fail the workflow task on the first attempt. Subsequent failures of the same workflow task
+        # should timeout. This is to avoid spinning on the failed workflow task as the service doesn't
+        # yet exponentially backoff on retries.
+        return if task.attempt > MAX_FAILED_ATTEMPTS
+
         client.respond_workflow_task_failed(
           task_token: task_token,
           cause: Temporal::Api::Enums::V1::WorkflowTaskFailedCause::WORKFLOW_TASK_FAILED_CAUSE_UNHANDLED_COMMAND,
           exception: error
         )
+      rescue StandardError => error
+        Temporal.logger.error("Unable to fail Workflow task", metadata.to_h.merge(error: error.inspect))
+
+        Temporal::ErrorHandler.handle(error, metadata: metadata)
       end
     end
   end