aws-flow 2.4.0 → 3.0.0

data/lib/aws/templates/result.rb

@@ -0,0 +1,183 @@
+module AWS
+  module Flow
+    module Templates
+
+      # ResultWorker is responsible for processing the results of the background
+      # jobs. It starts an ActivityWorker to process the ActivityTasks for
+      # FlowDefaultResultActivityRuby.run activity. It either returns futures or
+      # or actual results themselves back to the user
+      class ResultWorker
+
+        # Wrapper around a ruby {Hash} to provide synchronization around making
+        # changes to the encapsulated hash.
+        class SynchronizedHash
+          attr_reader :hash
+
+          def initialize
+            @semaphore = Mutex.new
+            @hash = {}
+          end
+
+          def method_missing(method, *args)
+            # Not very efficient but ruby structures are not thread
+            # safe in MRI.
+            @semaphore.synchronize{ return @hash.send(method, *args) }
+          end
+        end
+
+        class << self
+          attr_reader :results
+        end
+
+        # Controls synchronization around creation of the ActivityWorker to
+        # ensure singleton
+        @semaphore = Mutex.new
+
+        # Starts ResultWorker and ensures that a single ActivityWorker is
+        # started for this process. Initializes all class instance variables.
+        def self.start(domain)
+
+          # If already initiated, return
+          return @task_list if @task_list
+
+          # Acquire the lock to ensure only 1 copy of the worker is created
+          @semaphore.synchronize do
+            # If multiple threads were waiting on the lock, then we should
+            # return if the worker was created by the previous thread
+            return @task_list if @task_list
+            # Initiate all class instance variables. @semaphore around this
+            # block ensures a singleton
+            self.init
+          end
+
+          # Create pipes for IPC
+          reader, writer = IO.pipe
+
+          # Start the ForkingExecutor with the ActivityWorker
+          self.start_executor(reader, writer, domain)
+
+          # Close one end of the writer pipe
+          writer.close
+
+          # Start the listener thread
+          self.start_listener(reader)
+
+          # Register signal handlers for this process
+          self.handle_signals
+
+          return @task_list
+
+        end
+
+        private
+
+        # Initiates the class instance variables
+        # @api private
+        def self.init
+          # We want the result to be sent to a specific tasklist so that no other
+          # worker gets the result of this workflow.
+          @task_list ||= "#{Socket.gethostname}:#{Process.pid}:#{SecureRandom.uuid}"
+          # Results will be stored in this hash
+          @results ||= SynchronizedHash.new
+          # Create a new forking executor
+          @executor ||= ForkingExecutor.new
+        end
+
+        # Start the ActivityWorker using the ForkingExecutor
+        # @api private
+        def self.start_executor(reader, writer, domain)
+          # Create a child process and start an ActivityWorker
+          @executor.execute do
+            $0 = 'result-worker'
+            # Close one end of the reader pipe
+            reader.close
+
+            # Create a new instance of the FlowDefaultResultActivityRuby
+            # class and add it to the ActivityWorker. We instantiate the
+            # activity with the writer pipe so that the activity instance
+            # can report results back to the parent process.
+            activity = AWS::Flow::Templates.result_activity.new(writer)
+
+            # Start the activity worker. In case of UnknownResourceFault,
+            # register the types and start it again.
+            AWS::Flow::Templates::Utils.register_on_failure(domain) do |x|
+              swf = AWS::SimpleWorkflow.new
+              x = swf.domains[x]
+              AWS::Flow::ActivityWorker.new(x.client, x, @task_list, activity).start(false)
+            end
+          end
+        end
+
+        # Starts a listener thread that reads data from a reader pipe and
+        # updates the result hash
+        # @api private
+        def self.start_listener(reader)
+          @listener_t = Thread.new do
+            Thread.current[:name] = "listener_t"
+            while true
+              data = reader.gets
+              result = Marshal.load(data)
+              # Only update the result if an unset Future is present at the
+              # given location in the hash.
+              future = @results[result[:key]]
+              if future && !future.set?
+                future.set(result[:result])
+              end
+            end
+          end
+        end
+
+        # Resets all the class instance variables for ResultWorker
+        # @api private
+        def self.reset
+          @listener_t = nil
+          @results = nil
+          @task_list = nil
+          @executor = nil
+        end
+
+        # Stops the ResultWorker, i.e., terminates the listener thread and
+        # shutdowns the executor.
+        # @api private
+        def self.stop
+          @listener_t.terminate if @listener_t
+          @executor.shutdown(0) if @executor
+          self.reset
+        end
+
+        # Registers the signal handlers
+        # @api private
+        def self.handle_signals
+          at_exit {
+            self.stop
+          }
+          %w{ TERM INT }.each do |s|
+            Signal.trap(s) do
+              self.stop
+              Kernel.exit
+            end
+          end
+        end
+
+        # Gets the result of the background job. The job is identified by the
+        # unique key which was assigned to it during scheduling.
+        # The method returns a future which the users can wait on to get the
+        # result.
+        # @api private
+        def self.get_result_future(key)
+
+          # Get the future from the results hash
+          future = self.results[key]
+
+          # Self delete the future from the results hash when it is set
+          future.on_set { |x| self.results.delete(key) }
+
+          return future
+        end
+
+      end
+
+    end
+
+  end
+end

data/lib/aws/templates/starter.rb

@@ -4,250 +4,176 @@ module AWS
     # @api private
     module Templates
 
-      # Starts an Activity or a Workflow Template execution using the default
-      # workflow class FlowDefaultWorkflowRuby
-      #
-      # @param [String or AWS::Flow::Templates::TemplateBase] name_or_klass
-      #   The Activity or the Workflow Template that needs to be scheduled via
-      #   the default workflow. This argument can either be a string that
-      #   represents a fully qualified activity name - <ActivityClass>.<method_name>
-      #   or it can be an instance of AWS::Flow::Templates::TemplateBase
-      #
-      # @param [Hash] input
-      #   Input hash for the workflow execution
-      #
-      # @param [Hash] opts
-      #   Additional options to configure the workflow or activity execution.
-      #
-      # @option opts [true, false] :wait
-      #   *Optional* This boolean flag can be set to true if the result of the
-      #   task is required. Default value is false.
-      #
-      # @option opts [Integer] :wait_timeout
-      #   *Optional* This sets the timeout value for :wait. Default value is
-      #   nil.
-      #
-      # @option opts [Hash] :exponential_retry
-      #   A hash of {AWS::Flow::ExponentialRetryOptions}. Default value is -
-      #   { maximum_attempts: 3 }
-      #
-      # @option opts [String] *Optional* :domain
-      #   Default value is FlowDefault
-      #
-      # @option opts [Integer] *Optional* :execution_start_to_close_timeout
-      #   Default value is 3600 seconds (1 hour)
-      #
-      # @option opts [Integer] *Optional* :retention_in_days
-      #   Default value is 7 days
-      #
-      # @option opts [String] *Optional* :workflow_id
-      #
-      # @option opts [Integer] *Optional* :task_priority
-      #   Default value is 0
-      #
-      # @option opts [String] *Optional* :tag_list
-      #   By default, the name of the activity task gets added to the workflow's
-      #   tag_list
-      #
-      # @option opts *Optional* :data_converter
-      #   Default value is {AWS::Flow::YAMLDataConverter}. To use the
-      #   {AWS::Flow::S3DataConverter}, set the AWS_SWF_BUCKET_NAME environment
-      #   variable name with a valid AWS S3 bucket name.
-      #
-      # @option opts *Optional* A hash of {AWS::Flow::ActivityOptions}
-      #
-      # Usage -
-      #
-      #    AWS::Flow::start("<ActivityClassName>.<method_name>", <input_hash>,
-      #    <options_hash> )
-      #
-      # Example -
-      #
-      # 1) Start an activity execution -
-      #    AWS::Flow::start("HelloWorldActivity.say_hello", { name: "World" })
-      #
-      # 2) Start an activity execution with overriden options -
-      #    AWS::Flow::start("HelloWorldActivity.say_hello", { name: "World" }, {
-      #      exponential_retry: { maximum_attempts: 10 } }
-      #    )
-      #
-      def self.start(name_or_klass, input, opts = {})
+      class Starter
+
+        # Starts an Activity or a Workflow Template execution using the default
+        # workflow class FlowDefaultWorkflowRuby
+        #
+        # @param [String or AWS::Flow::Templates::TemplateBase] name_or_klass
+        #   The Activity or the Workflow Template that needs to be scheduled via
+        #   the default workflow. This argument can either be a string that
+        #   represents a fully qualified activity name - <ActivityClass>.<method_name>
+        #   or it can be an instance of AWS::Flow::Templates::TemplateBase
+        #
+        # @param [Hash] input
+        #   Input hash for the workflow execution
+        #
+        # @param [Hash] opts
+        #   Additional options to configure the workflow or activity execution.
+        #
+        # @option opts [true, false] :get_result
+        #   *Optional* This boolean flag can be set to true if the result future
+        #   if required. The future can be waited on by using the
+        #   AWS::Flow::wait_for_all, AWS::Flow::wait_for_any methods or by
+        #   calling the ExternalFuture#get method. Default value is false.
+        #
+        # @option opts [Hash] :exponential_retry
+        #   A hash of {AWS::Flow::ExponentialRetryOptions}. Default value is -
+        #   { maximum_attempts: 3 }
+        #
+        # @option opts [String] *Optional* :domain
+        #   Default value is FlowDefault
+        #
+        # @option opts [Integer] *Optional* :execution_start_to_close_timeout
+        #   Default value is 3600 seconds (1 hour)
+        #
+        # @option opts [Integer] *Optional* :retention_in_days
+        #   Default value is 7 days
+        #
+        # @option opts [String] *Optional* :workflow_id
+        #
+        # @option opts [Integer] *Optional* :task_priority
+        #   Default value is 0
+        #
+        # @option opts [String] *Optional* :tag_list
+        #   By default, the name of the activity task gets added to the workflow's
+        #   tag_list
+        #
+        # @option opts *Optional* :data_converter
+        #   Default value is {AWS::Flow::YAMLDataConverter}. To use the
+        #   {AWS::Flow::S3DataConverter}, set the AWS_SWF_BUCKET_NAME environment
+        #   variable name with a valid AWS S3 bucket name.
+        #
+        # @option opts *Optional* A hash of {AWS::Flow::ActivityOptions}
+        #
+        # Usage -
+        #
+        #    AWS::Flow::start("<ActivityClassName>.<method_name>", <input_hash>,
+        #    <options_hash> )
+        #
+        # Example -
+        #
+        # 1) Start an activity execution -
+        #    AWS::Flow::start("HelloWorldActivity.say_hello", { name: "World" })
+        #
+        # 2) Start an activity execution with overriden options -
+        #    AWS::Flow::start("HelloWorldActivity.say_hello", { name: "World" }, {
+        #      exponential_retry: { maximum_attempts: 10 } }
+        #    )
+        #
+        def self.start(name_or_klass, input, opts = {})
+
+          options = opts.dup
+
+          if name_or_klass.is_a?(String)
+            # Add activity name as a tag to the workflow execution
+            (options[:tag_list] ||= []) << name_or_klass
+
+            # If name_or_klass passed in is a string, we are assuming the user is
+            # trying to start a single activity task. Wrap the activity information
+            # in the activity template
+            name_or_klass = AWS::Flow::Templates.activity(name_or_klass, options)
+
+            # Keep only the required options in the hash
+            keys = [
+              :domain,
+              :retention_in_days,
+              :execution_start_to_close_timeout,
+              :task_priority,
+              :get_result,
+              :workflow_id,
+              :data_converter,
+              :tag_list
+            ]
+            options.select! { |x| keys.include?(x) }
 
-        options = opts.dup
-
-        if name_or_klass.is_a?(String)
-          # Add activity name as a tag to the workflow execution
-          (options[:tag_list] ||= []) << name_or_klass
+          end
 
-          # If name_or_klass passed in is a string, we are assuming the user is
-          # trying to start a single activity task. Wrap the activity information
-          # in the activity template
-          name_or_klass = AWS::Flow::Templates.activity(name_or_klass, options)
+          # Wrap the template in a root template
+          root = AWS::Flow::Templates.root(name_or_klass)
+
+          # Get the default options and merge them with the options passed in. The
+          # order of the two hashes 'defaults' and 'options' is important here.
+          defaults = FlowConstants.defaults.select do |key|
+            [
+              :domain,
+              :prefix_name,
+              :execution_method,
+              :version,
+              :execution_start_to_close_timeout,
+              :data_converter,
+              :task_list
+            ].include?(key)
+          end
+          options = defaults.merge(options)
 
-          # Keep only the required options in the hash
-          keys = [
-            :domain,
-            :retention_in_days,
-            :execution_start_to_close_timeout,
-            :task_priority,
-            :wait,
-            :wait_timeout,
-            :workflow_id,
-            :data_converter,
-            :tag_list
-          ]
-          options.select! { |x| keys.include?(x) }
+          raise "input needs to be a Hash" unless input.is_a?(Hash)
 
-        end
+          # Set the input for the default workflow
+          workflow_input = {
+            definition: root,
+            args: input
+          }
 
-        # Wrap the template in a root template
-        root = AWS::Flow::Templates.root(name_or_klass)
+          # get_result specifies if we should return back a result future
+          # for this task
+          get_result = options.delete(:get_result)
 
-        # Get the default options and merge them with the options passed in. The
-        # order of the two hashes 'defaults' and 'options' is important here.
-        defaults = FlowConstants.defaults.select do |key|
-          [
-            :domain,
-            :prefix_name,
-            :execution_method,
-            :version,
-            :execution_start_to_close_timeout,
-            :data_converter,
-            :task_list
-          ].include?(key)
-        end
-        options = defaults.merge(options)
+          if get_result
+            # Start the default result activity worker
+            task_list = ResultWorker.start(options[:domain])
 
-        raise "input needs to be a Hash" unless input.is_a?(Hash)
+            # Set the result_step for the root template. We need to pass in the
+            # task_list to ensure the result activity task is sent to the right
+            # task list. This method will return back a unique key that will
+            # help us locate the result of this task in ResultWorker.results hash
+            key = set_result_activity(task_list, root)
+          end
 
-        # Set the input for the default workflow
-        workflow_input = {
-          definition: root,
-          args: input,
-        }
+          # Call #start_workflow with the correct options to start the workflow
+          # execution. If it fails with UnknownResourceFault, then regsiter the
+          # default types and retry.
+          AWS::Flow::Templates::Utils.register_on_failure(options[:domain]) do
+            AWS::Flow::start_workflow(workflow_input, options)
+          end
 
-        # Set the result_step for the root template if wait flag is
-        # set.
-        wait = options.delete(:wait)
-        wait_timeout = options.delete(:wait_timeout)
-        result_tasklist = set_result_activity(root) if wait
+          # Get the result identified by this key
+          ResultWorker.get_result_future(key) if get_result
 
-        # Call #start_workflow with the correct options to start the workflow
-        # execution
-        begin
-          AWS::Flow::start_workflow(workflow_input, options)
-        rescue AWS::SimpleWorkflow::Errors::UnknownResourceFault => e
-          register_defaults(options[:domain])
-          AWS::Flow::start_workflow(workflow_input, options)
         end
 
-        # Wait for result
-        get_result(result_tasklist, options[:domain], wait_timeout) if wait
-
-      end
-
-      # Sets the result activity with a unique tasklist name for the root template.
-      # @api private
-      def self.set_result_activity(root)
-        # We want the result to be sent to a specific tasklist so that no other
-        # worker gets the result of this workflow.
-        result_tasklist = "result_tasklist: #{SecureRandom.uuid}"
-
-        name = "#{FlowConstants.defaults[:result_activity_prefix]}."\
-          "#{FlowConstants.defaults[:result_activity_method]}"
-
-        # Set the result_step of the root template to the result activity and
-        # override the tasklist and timeouts.
-        root.result_step = activity(name, {
-            task_list: result_tasklist,
+        # Sets the result activity with a unique key. The key is used to match
+        # the task with the result of the task. It is provided as an input to
+        # the default result activity and is used to create a new ExternalFuture
+        # in the ResultWorker.results hash.
+        # @api private
+        def self.set_result_activity(task_list, root)
+
+          key = "result_key: #{SecureRandom.uuid}"
+          # Set the result_step of the root template to the result activity and
+          # override the tasklist and timeouts.
+          root.result_step = AWS::Flow::Templates.result(key, {
+            task_list: task_list,
             schedule_to_start_timeout: FlowConstants.defaults[:schedule_to_start_timeout],
             start_to_close_timeout: FlowConstants.defaults[:start_to_close_timeout]
-          }
-        )
-        result_tasklist
-      end
+          })
 
-      # Gets the result of the workflow execution by starting an ActivityWorker
-      # on the FlowDefaultResultActivityRuby class. The result activity will set
-      # the instance variable future :result with the result of the template.
-      # It will block till either the result future is set or till the timeout
-      # expires - whichever comes first.
-      # @api private
-      def self.get_result(tasklist, domain, timeout=nil)
+          # Create a new ExternalFuture in the ResultWorker.results hash.
+          ResultWorker.results[key] = ExternalFuture.new
 
-        swf = AWS::SimpleWorkflow.new
-        domain = swf.domains[domain]
-
-        # Create a new instance of the FlowDefaultResultActivityRuby class and
-        # add it to the ActivityWorker. We pass in the instance instead of the
-        # class itself, so that we can locally access the instance variable set
-        # by the activity method.
-        activity = FlowDefaultResultActivityRuby.new
-
-        # Create the activity worker to poll on the result tasklist
-        worker = AWS::Flow::ActivityWorker.new(domain.client, domain, tasklist, activity) {{ use_forking: false }}
-
-        # Keep polling till we get the result or timeout. A 0 or nil timeout
-        # will let the loop run to completion.
-        begin
-          Timeout::timeout(timeout) do
-            until activity.result.set?
-              worker.run_once(false)
-            end
-          end
-        rescue Timeout::Error => e
-          activity.result.set
-          return
+          key
         end
 
-        # Get the result from the future
-        result = activity.result.get
-        if result.is_a?(Hash) && result[:failure] && result[:failure].is_a?(Exception)
-          raise result[:failure]
-        end
-
-        result
-      end
-
-      # Registers the relevant defaults with the Simple Workflow Service
-      # @api private
-      def self.register_defaults(name=nil)
-        domain = name.nil? ? register_default_domain : AWS::SimpleWorkflow.new.domains[name]
-
-        register_default_workflow(domain)
-        register_default_result_activity(domain)
-      end
-
-      # Registers the default domain FlowDefault with the Simple Workflow
-      # Service
-      # @api private
-      def self.register_default_domain
-        AWS::Flow::Utilities.register_domain(FlowConstants.defaults[:domain])
-      end
-
-      # Registers the default workflow type FlowDefaultWorkflowRuby with the
-      # Simple Workflow Service
-      # @api private
-      def self.register_default_workflow(domain)
-        AWS::Flow::WorkflowWorker.new(
-          domain.client,
-          domain,
-          nil,
-          AWS::Flow::Templates.default_workflow
-        ).register
-      end
-
-      # Registers the default result activity type FlowDefaultResultActivityRuby
-      # with the Simple Workflow Service
-      # @api private
-      def self.register_default_result_activity(domain)
-        worker = AWS::Flow::ActivityWorker.new(
-          domain.client,
-          domain,
-          nil,
-          AWS::Flow::Templates.result_activity
-        ) {{ use_forking: false }}
-        worker.register
       end
 
     end