aws-flow 1.3.0 → 2.0.0

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    ZDY5Y2FlNWU1YWQ4ODI4OWIzNGVhZDY2Yjg5ZDI5MjExYzQyODgyNg==
+  data.tar.gz: !binary |-
+    ZjA4MTljNTcwNjNhYWE0MjZlZGVkMjY2YmI4YzE2Y2Y0MGZiM2Q2NQ==
+SHA512:
+  metadata.gz: !binary |-
+    NDU4NzdjNGE1NDRiNDA2NjEzZjc5YmM2NzQ4MWIxMjZhNDA2OWE4MDZiMzI1
+    NzQ0NTY3OWVmMjFmYzQwMzNhMDJhZDI4ODczNjhlMzEwMjMwYWQzOWU0MWNl
+    ODQyNTQ3NWM5ZDMzZjJkNjgxOGMwMmE5ZTliOGFjNTM0ZDJiYWU=
+  data.tar.gz: !binary |-
+    OGU3NzgxYTQ2YmZmMTBmMzNiMzAxNzQ5NWJiNTVkNDBhZTUzNThhMjRjZTE0
+    NDU5MDE1NGY3NGQ1MDYxNGRiN2ViMjY0OGM4M2VkMzRjOTliZGFiMDViOTYy
+    M2NiNGM4MmM5YzQ3ZWZmMDA0ZWU1MTAxMjM2NjE1NmMwZDBhNWY=

data/aws-flow.gemspec

@@ -12,5 +12,6 @@ Gem::Specification.new do |s|
   s.email       = ''
   s.files       = `git ls-files`.split("\n").reject {|file| file =~ /aws-flow-core/}
   s.require_paths << "lib/aws/"
+  s.required_ruby_version = "~> 1.9", ">= 1.9.1"
   s.add_dependency "aws-sdk", "~> 1", ">= 1.39.0"
 end

data/lib/aws/decider/activity.rb

@@ -94,14 +94,16 @@ module AWS
       #
       def method_missing(method_name, *args, &block)
         options = Utilities::interpret_block_for_options(ActivityOptions, block)
-        options = Utilities::merge_all_options(@options,
+        client_options = Utilities::client_options_from_method_name(method_name, @options)
+
+        options = Utilities::merge_all_options(client_options,
                                     @activity_option_map[method_name.to_sym],
                                     @option_map[method_name.to_sym],
                                     options
                                     )
         new_options = ActivityOptions.new(options)
 
-        activity_type = ActivityType.new("#{new_options.prefix_name}.#{method_name.to_s}", new_options.version, new_options.get_default_options)
+        activity_type = ActivityType.new("#{new_options.prefix_name}.#{method_name.to_s}", new_options.version, new_options.get_registration_options)
         if new_options._exponential_retry
           retry_function = new_options._exponential_retry.retry_function || FlowConstants.exponential_retry_function
           new_options._exponential_retry.return_on_start ||= new_options.return_on_start
@@ -392,9 +394,9 @@ module AWS
       property(:activities, [])
 
       # @api private
-      def _options; @activities.map(&:options); end
+      def _options; @activities; end
 
-      # Defines one or more activities with {ActivityOptions} provided in the
+      # Defines one or more activities with {ActivityRegistrationOptions} provided in the
       # supplied block.
       #
       # @param [Array] activity_names
@@ -406,7 +408,7 @@ module AWS
       #   comes from the list passed to this parameter.
       #
       # @param [Hash] block
-      #   {ActivityOptions} to use on the defined activities.
+      #   {ActivityRegistrationOptions} to use on the defined activities.
       #
       #   The following options are *required* when registering an activity:
       #
@@ -434,7 +436,7 @@ module AWS
       #     end
       #   end
       def activity(*activity_names, &block)
-        options = Utilities::interpret_block_for_options(ActivityOptions, block)
+        options = Utilities::interpret_block_for_options(ActivityRegistrationOptions, block)
         activity_names.each do |activity_name|
           prefix_name = options.prefix_name || self.to_s
           activity_type = ActivityType.new(prefix_name + "." + activity_name.to_s, options.version, options)

data/lib/aws/decider/async_decider.rb

@@ -404,6 +404,7 @@ module AWS
         @decision_helper[timer_id].consume(:handle_completion_event)
         if @decision_helper[timer_id].done?
           open_request = @decision_helper.scheduled_timers.delete(timer_id)
+          return if open_request.nil?
           open_request.blocking_promise.set(nil)
           open_request.completion_handle.complete
         end

data/lib/aws/decider/async_retrying_executor.rb

@@ -24,12 +24,12 @@ module AWS
         @execution_id = execution_id
       end
       def execute(command, options = nil)
-        return schedule_with_retry(command, nil, Hash.new { |hash, key| hash[key] = 1 }, @clock.current_time, 0) if @return_on_start
+        return schedule_with_retry(command, nil, Hash.new { |hash, key| hash[key] = 1 }, @clock.current_time, nil) if @return_on_start
         output = Utilities::AddressableFuture.new
         result_lock = Utilities::AddressableFuture.new
         error_handler do |t|
           t.begin do
-            output.set(schedule_with_retry(command, nil, Hash.new { |hash, key| hash[key] = 1 }, @clock.current_time, 0))
+            output.set(schedule_with_retry(command, nil, Hash.new { |hash, key| hash[key] = 1 }, @clock.current_time, nil))
           end
           t.rescue(Exception) do |error|
             @error_seen = error
@@ -77,7 +77,7 @@ module AWS
           failure = should_retry.get
           if ! failure.nil?
             attempts[failure.class] += 1
-            output.set(schedule_with_retry(command, failure, attempts, first_attempt_time, @clock.current_time - first_attempt_time))
+            output.set(schedule_with_retry(command, failure, attempts, first_attempt_time, @clock.current_time))
           else
             output.set(return_value.get)
           end

data/lib/aws/decider/decider.rb

@@ -245,7 +245,7 @@ module AWS
     module Workflows
       attr_accessor :version
       extend Utilities::UpwardLookups
-      @precursors = []
+      @precursors ||= []
       def look_upwards(variable)
         precursors = self.ancestors.dup
         precursors.delete(self)
@@ -263,10 +263,10 @@ module AWS
       def entry_point(input=nil)
         if input
           @entry_point = input
-          workflow_type = WorkflowType.new(self.to_s + "." + input.to_s, nil, WorkflowOptions.new(:execution_method => input))
+          workflow_type = WorkflowType.new(self.to_s + "." + input.to_s, nil, WorkflowRegistrationOptions.new(:execution_method => input))
           self.workflows.each { |workflow| workflow.name = self.to_s + "." + input.to_s }
           self.workflows.each do |workflow|
-            workflow.options = WorkflowOptions.new(:execution_method => input)
+            workflow.options = WorkflowRegistrationOptions.new(:execution_method => input)
           end
           self.workflows = self.workflows << workflow_type
         end
@@ -327,22 +327,24 @@ module AWS
 
 
       # @api private
-      def _options; self.workflows.map(&:options); end
+      def _options; self.workflows; end
 
       # Defines a new workflow.
       #
-      # @param entry_point
-      #   The entry point (method) that starts the workflow.
+      # @param workflow_names
+      #   The entry points (methods) that starts the workflow.
       #
       # @param block
-      #   A block of {WorkflowOptions} for the workflow.
-      #
-      def workflow(entry_point, &block)
-        options = Utilities::interpret_block_for_options(WorkflowOptions, block)
-        options.execution_method = entry_point
-        workflow_name = options.prefix_name || self.to_s
-        workflow_type = WorkflowType.new(workflow_name.to_s + "." + entry_point.to_s, options.version, options)
-        self.workflows = self.workflows << workflow_type
+      #   A block of {WorflowRegistrationOptions} for the workflow.
+      #
+      def workflow(*workflow_names, &block)
+        workflow_names.each do |workflow_name| 
+          options = Utilities::interpret_block_for_options(WorkflowRegistrationOptions, block)
+          options.execution_method = workflow_name
+          prefix_name = options.prefix_name || self.to_s
+          workflow_type = WorkflowType.new(prefix_name.to_s + "." + workflow_name.to_s, options.version, options)
+          self.workflows = self.workflows << workflow_type
+        end
       end
 
       # @return [MethodPair]

data/lib/aws/decider/executor.rb

@@ -134,36 +134,49 @@ module AWS
       # @api private
       def remove_completed_pids(block=false)
         @log.debug "Removing completed child processes"
+
+        # waitpid2 throws an Errno::ECHILD if there are no child processes,
+        # so we don't even call it if there aren't any pids to wait on.
+        if @pids.empty?
+          @log.debug "No child processes. Returning."
+          return
+        end
+
+        @log.debug "Current child processes: #{@pids}"
+
+        # Non-blocking wait only returns a non-null pid if the child process has exited.
+        # This is the only part where we block if block=true.
+        pid, status = Process.waitpid2(-1, block ? 0 : Process::WNOHANG)
+
         loop do
+
+          # If nothing to reap, then exit
+          break unless pid
+
+          # We have something to reap
+          @log.debug "Reaping child process=#{pid}"
+          if status.success?
+            @log.debug "Child process #{pid} exited successfully"
+          else
+            @log.error "Child process #{pid} exited with non-zero status code"
+          end
+
+          # Reap
+          @pids.delete(pid)
+
           # waitpid2 throws an Errno::ECHILD if there are no child processes,
           # so we don't even call it if there aren't any pids to wait on.
           break if @pids.empty?
+
           @log.debug "Current child processes: #{@pids}"
-          # Non-blocking wait only returns a non-null pid
-          # if the child process has exited.
-          pid, status = Process.waitpid2(-1, block ? 0 : Process::WNOHANG)
           
-          if pid
-            # We have something to reap
-            @log.debug "Reaping child process=#{pid}"
-            if status.success?
-              @log.debug "Child process #{pid} exited successfully"
-            else
-              @log.error "Child process #{pid} exited with non-zero status code"
-            end
-            # Delete the pid from the list
-            @pids.delete(pid)
-            # Contract is to block only once if block=true. If we are in this code branch and if block=true, it
-            # means we have already blocked once above, hence it is safe to exit now
-            break if block
-          else
-            # Nothing to reap, exit
-            break
-          end
-
+          # Contract is to block only once if block=true. Since we have potentially already
+          # blocked once above, we only need to do a non blocking call to waitpid2 to see if
+          # any other process is available to reap.
+          pid, status = Process.waitpid2(-1, Process::WNOHANG)
         end
       end
-
     end
+
   end
 end

data/lib/aws/decider/flow_defaults.rb

@@ -67,10 +67,12 @@ module AWS
     class FlowConstants
 
       class << self
-        attr_reader :exponential_retry_maximum_retry_interval_seconds, :exponential_retry_retry_expiration_seconds, :exponential_retry_backoff_coefficient, :exponential_retry_maximum_attempts, :exponential_retry_function, :default_data_converter, :exponential_retry_exceptions_to_include, :exponential_retry_exceptions_to_exclude, :jitter_function, :should_jitter, :exponential_retry_initial_retry_interval
+        attr_reader :exponential_retry_maximum_retry_interval_seconds, :exponential_retry_retry_expiration_seconds, :exponential_retry_backoff_coefficient, :exponential_retry_maximum_attempts, :exponential_retry_function, :default_data_converter, :exponential_retry_exceptions_to_include, :exponential_retry_exceptions_to_exclude, :jitter_function, :should_jitter, :exponential_retry_initial_retry_interval, :use_worker_task_list
       end
 
       INFINITY = -1
+      RETENTION_DEFAULT = 7
+      NUM_OF_WORKERS_DEFAULT = 1
       @exponential_retry_maximum_attempts = Float::INFINITY
       @exponential_retry_maximum_retry_interval_seconds = -1
       @exponential_retry_retry_expiration_seconds = -1
@@ -81,22 +83,33 @@ module AWS
       @exponential_retry_exceptions_to_include = [Exception]
       @exponential_retry_function = lambda do |first, time_of_failure, attempts, options|
 
-        raise ArgumentError.new("first is not an instance of Time") unless first.instance_of?(Time)
-        raise ArgumentError.new("time_of_failure can't be negative") if time_of_failure < 0
-        raise ArgumentError.new("number of attempts can't be negative") if (attempts.values.find {|x| x < 0})
+        raise ArgumentError.new("first should be an instance of Time") unless first.instance_of?(Time)
+        raise ArgumentError.new("time_of_failure should be nil or an instance of Time") unless time_of_failure.nil? || time_of_failure.instance_of?(Time)
+        raise ArgumentError.new("number of attempts should be positive") if (attempts.values.find {|x| x < 0})
         raise ArgumentError.new("number of attempts should be more than 2") if (attempts.values.reduce(0,:+) < 2)
         raise ArgumentError.new("user options must be of type ExponentialRetryOptions") unless options.is_a? ExponentialRetryOptions
 
-        initial_retry_interval = options.initial_retry_interval
-        backoff_coefficient = options.backoff_coefficient
-        maximum_retry_interval_seconds = options.maximum_retry_interval_seconds
-        retry_expiration_interval_seconds = options.retry_expiration_interval_seconds
-        result = initial_retry_interval * (backoff_coefficient ** (attempts.values.reduce(0, :+) - 2))
-        result = maximum_retry_interval_seconds if (! maximum_retry_interval_seconds.nil? && maximum_retry_interval_seconds != INFINITY && result > maximum_retry_interval_seconds)
-        seconds_since_first_attempt = time_of_failure.zero? ? 0 : -(first - time_of_failure).to_i
-        result = -1 if (! retry_expiration_interval_seconds.nil? &&
-                        retry_expiration_interval_seconds != INFINITY &&
-                        (result + seconds_since_first_attempt) >= retry_expiration_interval_seconds)
+        # get values from options
+        initial_interval = options.initial_retry_interval
+        backoff = options.backoff_coefficient
+        max_interval = options.maximum_retry_interval_seconds
+        retry_expiration = options.retry_expiration_interval_seconds
+
+        # calculate the initial retry seconds
+        result = initial_interval * (backoff ** (attempts.values.reduce(0, :+) - 2))
+
+        # check if the calculated retry seconds is greater than the maximum
+        # retry interval allowed. If it is, then replace it with maximum_retry_interval_seconds
+        result = max_interval if ( !max_interval.nil? && max_interval != INFINITY && result > max_interval)
+
+        # how much time has elapsed since the first time this task was scheduled
+        time_elapsed = time_of_failure.nil? ? 0 : (time_of_failure - first).to_i
+
+        # return -1 if retry will expire
+        result = -1 if (! retry_expiration.nil? &&
+                        retry_expiration != INFINITY &&
+                        (result + time_elapsed) >= retry_expiration)
+
         return result.to_i
       end
 
@@ -107,6 +120,7 @@ module AWS
       end
 
       @default_data_converter = YAMLDataConverter.new
+      @use_worker_task_list = "USE_WORKER_TASK_LIST"
     end
   end
 end

data/lib/aws/decider/generic_client.rb

@@ -34,11 +34,10 @@ module AWS
       #   The options to set.
       #
       def with_opts(opts = {})
-        klass = self.class.default_option_class
-        options = klass.new(opts)
         modified_instance = self.dup
-        options = klass.new(Utilities::merge_all_options(modified_instance.options, options))
-        modified_instance.options = options
+        opts.each_pair do |key, value|
+          modified_instance.options.send("#{key}=", value) if modified_instance.options.methods.map(&:to_sym).include? "#{key}=".to_sym
+        end
         modified_instance
       end
 

data/lib/aws/decider/options.rb

@@ -25,69 +25,6 @@ module AWS
       end
     end
 
-    # This module refactors out some of the common methods for the Options
-    # classes. Any class including this module should implement
-    # make_runtime_key method and default_keys method. default_keys method
-    # provides an array of keys that are considered to be the default options
-    # for that class. make_runtime_key method converts a passed in default key
-    # to it's corresponding runtime key.
-    module OptionsMethods
-
-      # Retrieves the runtime values for the default options
-      #
-      # @return [Hash]
-      #   The runtime option names and their current values.
-      #
-      def get_runtime_options
-        runtime_options = {}
-        # For the default values that are present, convert the default keys into
-        # runtime keys. i.e. remove 'default_' and 'default_task_' from the key
-        # name and merge their values with the default values
-        get_default_options.each do |key, val|
-          new_key = make_runtime_key(key)
-          new_val = get_options([new_key])
-          runtime_options[new_key] = new_val.empty? ? val : new_val.values.first
-        end
-        runtime_options
-      end
-
-      # Retrieves the default options.
-      #
-      # @return [Hash]
-      #   A hash containing the default option names and their current values.
-      #
-      def get_default_options
-        # Get the default options
-        get_options(default_keys)
-      end
-
-      # Retrieves full options. It merges the runtime options with the remaining
-      # options
-      #
-      # @return [Hash]
-      #   A hash containing the full option names and their current values.
-      #
-      def get_full_options
-        # Initialize an empty hash
-        options_hash = {}
-
-        # Get all the properties held by this class
-        options_keys = self.class.held_properties
-
-        # Remove the unnecessary options (i.e. options not recognized by swf but
-        # only by flow) from the options_keys array.
-        default_keys.concat([:from_class]).each { |x| options_keys.delete(x) }
-
-        # If the value for an option is held by the class, get it and store it
-        # in a hash
-        options_keys.each do |option|
-          options_hash[option] = self.send(option) if self.send(option)
-        end
-        # Merge the options_hash with the runtime options
-        options_hash.merge(self.get_runtime_options)
-      end
-    end
-
     # The base class for all options classes in the AWS Flow Framework for Ruby.
     class Options
       extend Utilities::UpwardLookups
@@ -247,7 +184,7 @@ module AWS
       def get_full_options
         result = {}
         SignalWorkflowOptions.held_properties.each do |option|
-          result[option] = self.send(option) if self.send(option) && self.send(option) != ""
+          result[option] = self.send(option) if self.send(option) != nil && self.send(option) != ""
         end
         result
       end
@@ -378,6 +315,13 @@ module AWS
     # Defaults for `WorkflowOptions`.
     class WorkflowDefaults < Defaults
 
+      # The default data converter. By default, this is {YAMLDataConverter}.
+      def data_converter; FlowConstants.default_data_converter; end
+
+    end
+
+    class WorkflowRegistrationDefaults < WorkflowDefaults
+
       # The default task start-to-close timeout duration. The default value is
       # `30`.
       def default_task_start_to_close_timeout; 30; end
@@ -389,8 +333,7 @@ module AWS
       # array (no tags).
       def tag_list; []; end
 
-      # The default data converter. By default, this is {YAMLDataConverter}.
-      def data_converter; FlowConstants.default_data_converter; end
+      def default_task_list; FlowConstants.use_worker_task_list; end
     end
 
     # Options for workflows.
@@ -485,7 +428,6 @@ module AWS
     #   * It must not contain the literal string "arn".
     #
     class WorkflowOptions < Options
-      include OptionsMethods
 
       properties(
         :version,
@@ -497,37 +439,54 @@ module AWS
         :execution_method
       )
 
+      property(:tag_list, [])
+      property(:child_policy, [lambda(&:to_s), lambda(&:upcase)])
+      property(:data_converter, [])
+
+      default_classes << WorkflowDefaults.new
+
+      def get_full_options
+        result = {}
+        usable_properties = self.class.held_properties
+        usable_properties.delete(:from_class)
+        usable_properties.each do |option|
+          result[option] = self.send(option) if self.send(option) != nil && self.send(option) != ""
+        end
+        result
+      end
+
+    end
+
+    class WorkflowRegistrationOptions < WorkflowOptions
+      class << self
+        def registration_options
+          [ 
+            :default_task_start_to_close_timeout,
+            :default_execution_start_to_close_timeout,
+            :default_task_list,
+            :default_child_policy
+          ]
+        end
+      end
+
       # Adding default properties
       properties(
         :default_task_start_to_close_timeout,
         :default_execution_start_to_close_timeout,
         :default_task_list
       )
-      property(:default_child_policy, [lambda(&:to_s), lambda(&:upcase)])
-
-
-      property(:tag_list, [])
-      property(:child_policy, [lambda(&:to_s), lambda(&:upcase)])
-      property(:data_converter, [])
 
-      default_classes << WorkflowDefaults.new
+      property(:default_child_policy, [lambda(&:to_s), lambda(&:upcase)])
 
-      # This method provides the default option keys for workflows
-      def default_keys
-        [:default_task_start_to_close_timeout,
-         :default_execution_start_to_close_timeout,
-         :default_task_list,
-         :default_child_policy]
-      end
+      default_classes << WorkflowRegistrationDefaults.new
 
-      # This method converts default option keys to runtime keys by replacing
-      # "default_" in the key name
-      def make_runtime_key(key)
-        key.to_s.gsub(/default_/, "").to_sym
+      def get_registration_options
+        get_options(self.class.registration_options)
       end
 
     end
 
+
     # Options for {WorkflowClient#start_execution}.
     #
     # @!attribute workflow_name
@@ -549,6 +508,10 @@ module AWS
 
     # Defaults for the {ActivityOptions} class.
     class ActivityDefaults < Defaults
+      def data_converter; FlowConstants.default_data_converter; end
+    end
+
+    class ActivityRegistrationDefaults < ActivityDefaults
 
       # The default schedule-to-start timeout for activity tasks. This timeout
       # represents the time, in seconds, between when the activity task is first
@@ -585,10 +548,9 @@ module AWS
       # set this value to "NONE" to imply no timeout value.
       def default_task_heartbeat_timeout; Float::INFINITY; end
 
-      def data_converter; FlowConstants.default_data_converter; end
+      def default_task_list; FlowConstants.use_worker_task_list; end
     end
 
-
     # Options to use on an activity or decider. The following options are
     # defined:
     #
@@ -640,7 +602,6 @@ module AWS
     #   decision.
     #
     class ActivityOptions < Options
-      include OptionsMethods
 
       properties(
         :heartbeat_timeout,
@@ -652,40 +613,11 @@ module AWS
         :input
       )
 
-      # Adding default properties
-      properties(
-        :default_task_heartbeat_timeout,
-        :default_task_list,
-        :default_task_schedule_to_close_timeout,
-        :default_task_schedule_to_start_timeout,
-        :default_task_start_to_close_timeout,
-      )
-
       property(:manual_completion, [lambda {|x| x == true}])
       property(:data_converter, [])
 
       default_classes << ActivityDefaults.new
 
-      # This method provides the default option keys for activities
-      def default_keys
-        [:default_task_heartbeat_timeout,
-         :default_task_schedule_to_close_timeout,
-         :default_task_schedule_to_start_timeout,
-         :default_task_start_to_close_timeout,
-         :default_task_list]
-      end
-
-      # This method converts default option keys to runtime keys by replacing
-      # "default_task_" in the key name. It handles the exception of task_list
-      # where only "default_" needs to be replaced.
-      def make_runtime_key(key)
-        if key =~ /task_list/
-          key.to_s.gsub(/default_/, "").to_sym
-        else
-          key.to_s.gsub(/default_task_/, "").to_sym
-        end
-      end
-
       # Gets the activity prefix name.
       #
       # @return [String]
@@ -777,6 +709,48 @@ module AWS
         retry_options = Utilities::interpret_block_for_options(ExponentialRetryOptions, block)
         @_exponential_retry = retry_options
       end
+
+      def get_full_options
+        result = {}
+        usable_properties = self.class.held_properties
+        usable_properties.delete(:from_class)
+        usable_properties.each do |option|
+          result[option] = self.send(option) if self.send(option) != nil && self.send(option) != ""
+        end
+        result
+      end
+
+    end
+
+    # This class is used to capture the options passed during activity declaration.
+    class ActivityRegistrationOptions < ActivityOptions
+      class << self
+        def registration_options
+          [
+            :default_task_heartbeat_timeout,
+            :default_task_schedule_to_close_timeout,
+            :default_task_schedule_to_start_timeout,
+            :default_task_start_to_close_timeout,
+            :default_task_list
+          ]
+        end
+      end
+
+      # Adding default properties
+      properties(
+        :default_task_heartbeat_timeout,
+        :default_task_list,
+        :default_task_schedule_to_close_timeout,
+        :default_task_schedule_to_start_timeout,
+        :default_task_start_to_close_timeout,
+      )
+
+      default_classes << ActivityRegistrationDefaults.new
+
+      def get_registration_options
+        get_options(self.class.registration_options)
+      end
+
     end
 
     # Runtime options for an activity.