aws-flow 1.1.1 → 1.2.0

data/lib/aws/decider/task_handler.rb

@@ -17,9 +17,11 @@ module AWS
   module Flow
 
 
-    # A decision task handler to work with a {WorkflowTaskPoller}. Create a `DecisionTaskHandler` and pass it to
-    # {WorkflowTaskPoller} on {WorkflowTaskPoller#initialize construction}.
+    # A decision task handler to work with a {WorkflowTaskPoller}. Create a
+    # `DecisionTaskHandler` and pass it to {WorkflowTaskPoller} on
+    # {WorkflowTaskPoller#initialize construction}.
     class DecisionTaskHandler
+      attr_reader :workflow_definition_map
 
       # Creates a new `DecisionTaskHandler`.
       #
@@ -31,7 +33,7 @@ module AWS
       def initialize(workflow_definition_map, options=nil)
         @workflow_definition_map = workflow_definition_map
         @logger = options.logger if options
-        @logger ||= Utilities::LogFactory.make_logger(self, "debug")
+        @logger ||= Utilities::LogFactory.make_logger(self)
       end
 
 
@@ -41,11 +43,8 @@ module AWS
       #
       def handle_decision_task(decision_task_iterator)
         history_helper = HistoryHelper.new(decision_task_iterator)
-        @logger.debug "made history_helper"
         decider = create_async_decider(history_helper)
-        @logger.debug "made async_decider"
         decider.decide
-        @logger.debug "decided"
         decisions = decider.get_decisions
         response = {:task_token => decider.task_token}
         context_data = decider.decision_helper.workflow_context_data
@@ -72,5 +71,4 @@ module AWS
     end
 
   end
-
 end

data/lib/aws/decider/task_poller.rb

@@ -44,26 +44,30 @@ module AWS
         @domain = domain
         @task_list = task_list
         @logger = options.logger if options
-        @logger ||= Utilities::LogFactory.make_logger(self, "debug")
+        @logger ||= Utilities::LogFactory.make_logger(self)
       end
 
 
 
+      # @api private
       # Retrieves any decision tasks that are ready.
-      def get_decision_tasks
+      def get_decision_task
         @domain.decision_tasks.poll_for_single_task(@task_list)
       end
 
       def poll_and_process_single_task
         # TODO waitIfSuspended
         begin
-          @logger.debug "Starting a new task...\n\n\n"
-          tasks = get_decision_tasks
-          return false if tasks.nil?
-          @logger.debug "We have this many tasks #{tasks}"
-          @logger.debug "debugging on #{tasks}\n"
-          task_completed_request = @handler.handle_decision_task(tasks)
-          @logger.debug "task to be responded to with #{task_completed_request}\n"
+          @logger.debug "Polling for a new decision task of type #{@handler.workflow_definition_map.keys.map{ |x| "#{x.name} #{x.version}"} } on task_list: #{@task_list}"
+          task = get_decision_task
+          if task.nil?
+            @logger.debug "Didn't get a task on task_list: #{@task_list}"
+            return false
+          end
+          @logger.info Utilities.workflow_task_to_debug_string("Got decision task", task)
+
+          task_completed_request = @handler.handle_decision_task(task)
+          @logger.debug "Response to the task will be #{task_completed_request}"
           if !task_completed_request[:decisions].empty? && (task_completed_request[:decisions].first.keys.include?(:fail_workflow_execution_decision_attributes))
             fail_hash = task_completed_request[:decisions].first[:fail_workflow_execution_decision_attributes]
             reason = fail_hash[:reason]
@@ -71,11 +75,11 @@ module AWS
             @logger.debug "#{reason}, #{details}"
           end
           @service.respond_decision_task_completed(task_completed_request)
+          @logger.info Utilities.workflow_task_to_debug_string("Finished executing task", task)
         rescue AWS::SimpleWorkflow::Errors::UnknownResourceFault => e
-          @logger.error "Error in the poller, #{e}"
-          @logger.error "The error class in #{e.class}"
+          @logger.error "Error in the poller, #{e.class}, #{e}"
         rescue Exception => e
-          @logger.error "Error in the poller, #{e}"
+          @logger.error "Error in the poller, #{e.class}, #{e}"
         end
       end
     end
@@ -118,7 +122,7 @@ module AWS
         @task_list = task_list
         @activity_definition_map = activity_definition_map
         @logger = options.logger if options
-        @logger ||= Utilities::LogFactory.make_logger(self, "debug")
+        @logger ||= Utilities::LogFactory.make_logger(self)
         @executor = executor
       end
 
@@ -133,20 +137,35 @@ module AWS
         activity_type = task.activity_type
         begin
           context = ActivityExecutionContext.new(@service, @domain, task)
-          activity_implementation = @activity_definition_map[activity_type]
-          raise "This activity worker was told to work on activity type #{activity_type.name}, but this activity worker only knows how to work on #{@activity_definition_map.keys.map(&:name).join' '}" unless activity_implementation
+          unless activity_implementation = @activity_definition_map[activity_type]
+            raise "This activity worker was told to work on activity type #{activity_type.name}.#{activity_type.version}, but this activity worker only knows how to work on #{@activity_definition_map.keys.map(&:name).join' '}"
+          end
+
           output, original_result, too_large = activity_implementation.execute(task.input, context)
-          @logger.debug "Responding on task_token #{task.task_token} for task #{task}"
+
+           @logger.debug "Responding on task_token #{task.task_token} for task #{task}."
           if too_large
-            @logger.warn "The output of this activity was too large (greater than 2^15), and therefore aws-flow could not return it to SWF. aws-flow is now attempting to mark this activity as failed. For reference, the result was #{original_result}"
-            respond_activity_task_failed_with_retry(task.task_token, "An activity cannot send a response with a result larger than 32768 characters. Please reduce the response size. A truncated prefix output is included in the details field.", output)
+            @logger.error "Activity #{activity_type.name}.#{activity_type.version} failed: The output of this activity was too large (greater than 2^15), and therefore aws-flow could not return it to SWF. aws-flow is now attempting to mark this activity as failed. For reference, the result was #{original_result}"
+
+            respond_activity_task_failed_with_retry(
+              task.task_token,
+              "An activity cannot send a response with a result larger than 32768 characters. Please reduce the response size. A truncated prefix output is included in the details field.",
+              output
+            )
           elsif ! activity_implementation.execution_options.manual_completion
-            @service.respond_activity_task_completed(:task_token => task.task_token, :result => output)
+            @service.respond_activity_task_completed(
+              :task_token => task.task_token,
+              :result => output
+            )
           end
         rescue ActivityFailureException => e
-          @logger.error "The activity failed, with original output of #{original_result} and dataconverted result of #{output}. aws-flow will now attempt to fail it."
-          respond_activity_task_failed_with_retry(task.task_token, e.message, e.details)
+          @logger.error "Activity #{activity_type.name}.#{activity_type.version} with input #{task.input} failed with exception #{e}."
 
+          respond_activity_task_failed_with_retry(
+            task.task_token,
+            e.message,
+            e.details
+          )
         end
         #TODO all the completion stuffs
       end
@@ -267,10 +286,10 @@ module AWS
           begin
             execute(task)
           rescue CancellationException => e
+            @logger.error "Got an error, #{e.message}, while executing #{task.activity_type.name}."
             respond_activity_task_canceled_with_retry(task.task_token, e.message)
           rescue Exception => e
-            @logger.error "Got an error, #{e.message}, while executing #{task.activity_type.name}"
-            @logger.error "Full stack trace: #{e.backtrace}"
+            @logger.error "Got an error, #{e.message}, while executing #{task.activity_type.name}. Full stack trace: #{e.backtrace}"
             respond_activity_task_failed_with_retry(task.task_token, e.message, e.backtrace)
             #Do rescue stuff
           ensure
@@ -278,7 +297,7 @@ module AWS
           end
         rescue Exception => e
           semaphore_needs_release = true
-          @logger.error "Got into the other error mode"
+          @logger.error "Got into the other error mode: #{e}"
           raise e
         ensure
           @poll_semaphore.release if semaphore_needs_release
@@ -300,26 +319,23 @@ module AWS
         @poll_semaphore ||= SuspendableSemaphore.new
         @poll_semaphore.acquire
         semaphore_needs_release = true
-        @logger.debug "before the poll\n\n"
-        # This is to warm the lazily loaded clients in the @service, so we don't
-        # pay for their loading in every forked client
+        @logger.debug "Before the poll"
         begin
           if use_forking
             @executor.block_on_max_workers
           end
+          @logger.debug "Polling for a new activity task of type #{@activity_definition_map.keys.map{ |x| "#{x.name} #{x.version}"} } on task_list: #{@task_list}"
           task = @domain.activity_tasks.poll_for_single_task(@task_list)
           if task
-            @logger.info "got a task, #{task.activity_type.name}"
-            @logger.info "The task token I got was: #{task.task_token}"
+            @logger.info Utilities.activity_task_to_debug_string("Got activity task", task)
           end
         rescue Exception => e
-          @logger.error "I have not been able to poll successfully, and am now bailing out, with error #{e}"
+          @logger.error "Error in the poller, #{e.class}, #{e}"
           @poll_semaphore.release
           return false
         end
         if task.nil?
-          "Still polling at #{Time.now}, but didn't get anything"
-          @logger.debug "Still polling at #{Time.now}, but didn't get anything"
+          @logger.debug "Didn't get a task on task_list: #{@task_list}"
           @poll_semaphore.release
           return false
         end
@@ -329,8 +345,7 @@ module AWS
         else
           process_single_task(task)
         end
-        # process_single_task(task)
-        @logger.debug "finished executing the task"
+        @logger.info Utilities.activity_task_to_debug_string("Finished executing task", task)
         return true
       end
     end

data/lib/aws/decider/utilities.rb

@@ -21,15 +21,25 @@ module AWS
     module Utilities
       # @api private
       class LogFactory
-        def self.make_logger(klass, name)
-          logname = "#{Dir.tmpdir}/#{klass.class.to_s}_#{name}"
+        def self.make_logger(klass)
+          make_logger_with_level(klass, Logger::INFO)
+        end
+        def self.make_logger_with_level(klass, level)
+          logname = "#{Dir.tmpdir}/#{klass.class.to_s}"
           logname.gsub!(/::/, '-')
           log = Logger.new(logname)
-          log.level = Logger::DEBUG
+          log.level = level
           log
         end
       end
 
+      def self.workflow_task_to_debug_string(message, task)
+        return "#{message} #{task.workflow_type.name}.#{task.workflow_type.version} for execution with workflow_id: #{task.workflow_execution.workflow_id}, run_id: #{task.workflow_execution.run_id}, task_list: #{task.workflow_execution.task_list} with task_token: #{task.task_token}"
+      end
+      def self.activity_task_to_debug_string(message, task)
+        return "#{message} #{task.activity_type.name}.#{task.activity_type.version} with input: #{task.input} and task_token: #{task.task_token}"
+      end
+
 
       # @api private
       def self.drill_on_future(future)

data/lib/aws/decider/version.rb

@@ -16,7 +16,7 @@
 module AWS
   module Flow
     def self.version
-      "1.1.1"
+      "1.2.0"
     end
   end
 end

data/lib/aws/decider/worker.rb

@@ -41,7 +41,9 @@ module AWS
         @domain = domain
         @task_list = task_list_to_poll
         @options = Utilities::interpret_block_for_options(WorkerOptions, block)
-        args.each { |klass_or_instance| add_implementation(klass_or_instance) } if args
+        if args
+          args.each { |klass_or_instance| add_implementation(klass_or_instance) }
+        end
         @shutting_down = false
         %w{ TERM INT }.each do |signal|
           Signal.trap(signal) do
@@ -98,10 +100,11 @@ module AWS
     # Represents an activity type.
     class ActivityType < GenericType; end
 
-    # This worker class is intended for use by the workflow implementation. It is configured with
-    # a task list and a workflow implementation. The worker class polls for decision tasks in the
-    # specified task list. When a decision task is received, it creates an instance of the workflow implementation and
-    # calls the @ execute() decorated method to process the task.
+    # This worker class is intended for use by the workflow implementation. It
+    # is configured with a task list and a workflow implementation. The worker
+    # class polls for decision tasks in the specified task list. When a decision
+    # task is received, it creates an instance of the workflow implementation
+    # and calls the @ execute() decorated method to process the task.
     class WorkflowWorker < GenericWorker
 
       # The workflow type for this workflow worker.
@@ -121,12 +124,16 @@ module AWS
       # @param args
       #   The decisions to use.
       #
-      def initialize(service, domain, task_list, *args)
+      def initialize(service, domain, task_list, *args, &block)
         @workflow_definition_map = {}
         @workflow_type_options = []
-        super(service, domain, task_list, *args)
+        @options = Utilities::interpret_block_for_options(WorkerOptions, block)
 
+        @logger = @options.logger if @options
+        @logger ||= Utilities::LogFactory.make_logger(self)
+        @options.logger ||= @logger if @options
 
+        super(service, domain, task_list, *args)
       end
 
       def set_workflow_implementation_types(workflow_implementation_types)
@@ -151,15 +158,33 @@ module AWS
           implementation_options = nil
           get_state_method = workflow_class.get_state_method
           signals = workflow_class.signals
-          @workflow_definition_map[workflow_type] = WorkflowDefinitionFactory.new(workflow_class, workflow_type, registration_options, options, execution_method, signals, get_state_method)
-          # TODO should probably do something like GenericWorkflowWorker#registerWorkflowTypes
-          workflow_hash = options.get_options([:default_task_start_to_close_timeout, :default_execution_start_to_close_timeout, :default_child_policy], {
-                        :domain => @domain.name,
-                        :name => workflow_type.name,
-                        :version => version
-                      })
+
+          @workflow_definition_map[workflow_type] = WorkflowDefinitionFactory.new(
+            workflow_class,
+            workflow_type,
+            registration_options,
+            options,
+            execution_method,
+            signals,
+            get_state_method
+          )
+          # TODO should probably do something like
+          # GenericWorkflowWorker#registerWorkflowTypes
+          workflow_hash = options.get_options(
+            [
+              :default_task_start_to_close_timeout,
+              :default_execution_start_to_close_timeout,
+              :default_child_policy
+            ], {
+              :domain => @domain.name,
+              :name => workflow_type.name,
+              :version => version
+            }
+          )
           if options.default_task_list
-            workflow_hash.merge!({:default_task_list => {:name => options.default_task_list} })
+            workflow_hash.merge!(
+              {:default_task_list => {:name => options.default_task_list} }
+            )
           end
           @workflow_type_options << workflow_hash
         end
@@ -173,6 +198,7 @@ module AWS
           begin
             @service.register_workflow_type(workflow_type_options)
           rescue AWS::SimpleWorkflow::Errors::TypeAlreadyExistsFault => e
+            @logger.warn "#{e.class} while trying to register workflow #{e.message} with options #{workflow_type_options}"
             # Purposefully eaten up, the alternative is to check first, and who
             # wants to do two trips when one will do?
           end
@@ -183,7 +209,8 @@ module AWS
       # Starts the workflow with a {WorkflowTaskPoller}.
       #
       # @param [true,false] should_register
-      #   Indicates whether the workflow needs to be registered with Amazon SWF first. If {#register} was already called
+      #   Indicates whether the workflow needs to be registered with Amazon SWF
+      #   first. If {#register} was already called
       #   for this workflow worker, specify `false`.
       #
       def start(should_register = true)
@@ -193,14 +220,24 @@ module AWS
         # TODO Set up throttler
         # TODO Set up a timeout on the throttler correctly,
         # TODO Make this a generic poller, go to the right kind correctly
-        poller = WorkflowTaskPoller.new(@service, @domain, DecisionTaskHandler.new(@workflow_definition_map, @options), @task_list, @options)
+
+        poller = WorkflowTaskPoller.new(
+          @service,
+          @domain,
+          DecisionTaskHandler.new(@workflow_definition_map, @options),
+          @task_list,
+          @options
+        )
+
         register if should_register
+        @logger.debug "Starting an infinite loop to poll and process workflow tasks."
         loop do
           run_once(false, poller)
         end
       end
 
-      # Starts the workflow and runs it once, with an optional {WorkflowTaskPoller}.
+      # Starts the workflow and runs it once, with an optional
+      # {WorkflowTaskPoller}.
       #
       # @param should_register (see #start)
       #
@@ -209,21 +246,31 @@ module AWS
       #
       def run_once(should_register = false, poller = nil)
         register if should_register
-        poller = WorkflowTaskPoller.new(@service, @domain, DecisionTaskHandler.new(@workflow_definition_map, @options), @task_list, @options) if poller.nil?
+
+        poller = WorkflowTaskPoller.new(
+          @service,
+          @domain,
+          DecisionTaskHandler.new(@workflow_definition_map, @options),
+          @task_list,
+          @options
+        ) if poller.nil?
+
         Kernel.exit if @shutting_down
         poller.poll_and_process_single_task
       end
     end
 
 
-    # Used to implement an activity worker. You can use the `ActivityWorker` class to conveniently poll a task list for
-    # activity tasks.
-    #
-    # You configure the activity worker with activity implementation objects. This worker class then polls for activity
-    # tasks in the specified task list. When an activity task is received, it looks up the appropriate implementation
-    # that you provided, and calls the activity method to process the task. Unlike the {WorkflowWorker}, which creates a
-    # new instance for every decision task, the `ActivityWorker` simply uses the object you provided.
+    # Used to implement an activity worker. You can use the `ActivityWorker`
+    # class to conveniently poll a task list for activity tasks.
     #
+    # You configure the activity worker with activity implementation objects.
+    # This worker class then polls for activity tasks in the specified task
+    # list. When an activity task is received, it looks up the appropriate
+    # implementation that you provided, and calls the activity method to
+    # process the task. Unlike the {WorkflowWorker}, which creates a new
+    # instance for every decision task, the `ActivityWorker` simply uses the
+    # object you provided.
     class ActivityWorker < GenericWorker
 
       # Creates a new `ActivityWorker` instance.
@@ -245,11 +292,17 @@ module AWS
         @activity_definition_map = {}
         @activity_type_options = []
         @options = Utilities::interpret_block_for_options(WorkerOptions, block)
+
         @logger = @options.logger if @options
-        @logger ||= Utilities::LogFactory.make_logger(self, "debug")
+        @logger ||= Utilities::LogFactory.make_logger(self)
+        @options.logger ||= @logger if @options
+
         max_workers = @options.execution_workers if @options
         max_workers = 20 if (max_workers.nil? || max_workers.zero?)
-        @executor = ForkingExecutor.new(:max_workers => max_workers, :logger => @logger)
+        @executor = ForkingExecutor.new(
+          :max_workers => max_workers,
+          :logger => @logger
+        )
         @shutdown_first_time_function = lambda do
           @executor.shutdown Float::INFINITY
           Kernel.exit
@@ -273,8 +326,15 @@ module AWS
           begin
             @service.register_activity_type(activity_type_options)
           rescue AWS::SimpleWorkflow::Errors::TypeAlreadyExistsFault => e
-            previous_registration = @service.describe_activity_type(:domain => @domain.name, :activity_type => {:name => activity_type_options[:name], :version => activity_type_options[:version]})
-            default_options = activity_type_options.select {|key, val| key =~ /default/}
+            @logger.warn "#{e.class} while trying to register activity #{e.message} with options #{activity_type_options}"
+            previous_registration = @service.describe_activity_type(
+              :domain => @domain.name,
+              :activity_type => {
+                :name => activity_type_options[:name],
+                :version => activity_type_options[:version]
+              }
+            )
+            default_options = activity_type_options.select { |key, val| key =~ /default/}
             previous_keys = previous_registration["configuration"].keys.map {|x| camel_case_to_snake_case(x).to_sym}
 
             previous_registration = Hash[previous_keys.zip(previous_registration["configuration"].values)]
@@ -282,7 +342,10 @@ module AWS
               previous_registration[:default_task_list][:name] = previous_registration[:default_task_list].delete("name")
             end
             registration_difference =  default_options.sort.to_a - previous_registration.sort.to_a
-            raise "There is a difference between the types you have registered previously and the types you are currently registering, but you haven't changed the version. These new changes will not be picked up. In particular, these options are different #{Hash[registration_difference]}" unless registration_difference.empty?
+
+            unless registration_difference.empty?
+              raise "There is a difference between the types you have registered previously and the types you are currently registering, but you haven't changed the version. These new changes will not be picked up. In particular, these options are different #{Hash[registration_difference]}"
+            end
             # Purposefully eaten up, the alternative is to check first, and who
             # wants to do two trips when one will do?
           end
@@ -299,16 +362,28 @@ module AWS
         instance = (class_or_instance.class == Class) ? class_or_instance.new : class_or_instance
         klass.activities.each do |activity_type|
 
-          #TODO this should assign to an activityImplementation, so that we can call execute on it later
-          @activity_definition_map[activity_type] = ActivityDefinition.new(instance, activity_type.name.split(".").last, nil, activity_type.options, activity_type.options.data_converter)
+          # TODO this should assign to an activityImplementation, so that we can
+          # call execute on it later
+          @activity_definition_map[activity_type] = ActivityDefinition.new(
+            instance,
+            activity_type.name.split(".").last,
+            nil,
+            activity_type.options,
+            activity_type.options.data_converter
+          )
           options = activity_type.options
           option_hash = {
             :domain => @domain.name,
             :name => activity_type.name.to_s,
             :version => activity_type.version
           }
+
           option_hash.merge!(options.get_default_options)
-          option_hash.merge!(:default_task_list => {:name => options.default_task_list}) if options.default_task_list
+
+          option_hash.merge!(
+            :default_task_list => {:name => options.default_task_list}
+          ) if options.default_task_list
+
           @activity_type_options << option_hash
         end
       end
@@ -317,27 +392,49 @@ module AWS
       # Starts the activity that was added to the `ActivityWorker`.
       #
       # @param [true, false] should_register
-      #   Set to `false` if the activity should not register itself (it is already registered).
+      #   Set to `false` if the activity should not register itself (it is
+      #   already registered).
       #
       def start(should_register = true)
+
         register if should_register
-        poller = ActivityTaskPoller.new(@service, @domain, @task_list, @activity_definition_map, @executor, @options)
+        poller = ActivityTaskPoller.new(
+          @service,
+          @domain,
+          @task_list,
+          @activity_definition_map,
+          @executor,
+          @options
+        )
+
+        @logger.debug "Starting an infinite loop to poll and process activity tasks."
         loop do
           run_once(false, poller)
         end
       end
 
-      # Starts the activity that was added to the `ActivityWorker` and, optionally, sets the {ActivityTaskPoller}.
+      # Starts the activity that was added to the `ActivityWorker` and,
+      # optionally, sets the {ActivityTaskPoller}.
       #
       # @param [true, false] should_register
-      #   Set to `false` if the activity should not register itself (it is already registered).
+      #   Set to `false` if the activity should not register itself (it is
+      #   already registered).
       #
       # @param [ActivityTaskPoller] poller
-      #   The {ActivityTaskPoller} to use. If this is not set, a default {ActivityTaskPoller} will be created.
+      #   The {ActivityTaskPoller} to use. If this is not set, a default
+      #   {ActivityTaskPoller} will be created.
       #
       def run_once(should_register = true, poller = nil)
         register if should_register
-        poller = ActivityTaskPoller.new(@service, @domain, @task_list, @activity_definition_map, @executor, @options) if poller.nil?
+        poller = ActivityTaskPoller.new(
+          @service,
+          @domain,
+          @task_list,
+          @activity_definition_map,
+          @executor,
+          @options
+        ) if poller.nil?
+
         Kernel.exit if @shutting_down
         poller.poll_and_process_single_task(@options.use_forking)
       end