aws-flow 1.1.1 → 1.2.0

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    ODhiZjUwNGRmYjRhM2M4ODJkYTVkZjViYjk5YTIzOTViZWEzZjkwMQ==
+    ODBhMDgxOTcxNDAyNGUzMzZmMDllMDdjMWRiZmE3ZjZmM2NmOTA3MQ==
   data.tar.gz: !binary |-
-    NDY4MTM4MTM4YWIyY2ExY2U1MTg2YjY3ZWYzZjA0ODc1Zjg5Njc4MQ==
+    ZDhmNDA5OTBhNThkNDhmMGQ4MjM2ZDcyMGJhM2U1YTNlMGZiYzkyNg==
 SHA512:
   metadata.gz: !binary |-
-    NDdlYjVkZDNkMzEyYWQxNmI2Y2NkNmJjMzAzYTZhNzFlMTIxNTE4NGUyY2E0
-    MDU5YzlmY2E4ZDNkZmVhYTEzODRkZDI1ZTM5ZjFjY2E3OGY5MDBmOGYzYTUy
-    ZmQzODFmZDA1NjQ0YjFlZjk1Y2NhMTMwZjkyYTMzMzU4MTQwOWU=
+    NTRlNDUxZDIyZjQyNGRhNzAyOTZhZmQ0NjkwNjI0NThmZGIwOTJmYWVlN2Y3
+    ODY2MGZiZjFmOWE4MGJjN2VhNzJkNzBkMjhiYmNmYmNlMTExNjg3NTlkZjRm
+    YzlmZGQ3MjE5MWEyM2I2YjEyYmZiN2NmOGY2YTllMDY0MzE4YTg=
   data.tar.gz: !binary |-
-    MDczZjNhNTAyNTIxM2YzODgyZWUzMTU5NTE1MzE1OWY5YjE1MTJkNGUwNjM4
-    ZGNiMWQwM2VlYTAxMTc5NzY4Zjk5MmJkMTBiMzdkNDM5ZTBlMTRjZjZmYmI0
-    ZWJkYmY0YjRhZDVjNmZlZDMyYmNmY2FiMzczZmMyOTQ0NzczMDc=
+    YTczMTcwODZmMWZkNTgxODQ2NTZjY2FjNmE4ZGRiY2I5NzA0OWVhNTcxY2Ey
+    OThhNGI3ZDY2MDM2ZmM1ZmQ3M2QyZTcyN2JkZGYzNDlkYjM4NjVlMjA1NTMw
+    N2VhMjllYTRhYTE5M2E4M2FhNGZlMGY2ZjEzYjdjMzI4NjFkZmE=

data/lib/aws/decider/decider.rb

@@ -338,7 +338,7 @@ module AWS
       #   A block of {WorkflowOptions} for the workflow.
       #
       def workflow(entry_point, &block)
-        options = Utilities::interpret_block_for_options(WorkflowOptionsWithDefaults, block)
+        options = Utilities::interpret_block_for_options(WorkflowOptions, block)
         options.execution_method = entry_point
         workflow_name = options.prefix_name || self.to_s
         workflow_type = WorkflowType.new(workflow_name.to_s + "." + entry_point.to_s, options.version, options)

data/lib/aws/decider/executor.rb

@@ -50,9 +50,7 @@ module AWS
 
       def initialize(options = {})
         unless @log = options[:logger]
-          @log = Logger.new("#{Dir.tmpdir}/forking_log")
-          @log.level = options[:log_level] || Logger::ERROR
-          @log.info("LOG INITIALIZED")
+          @log = Utilities::LogFactory.make_logger(self)
         end
         @semaphore = Mutex.new
         @max_workers = options[:max_workers] || 1
@@ -63,38 +61,38 @@ module AWS
       end
 
       def execute(&block)
-        @log.info "Here are the pids that are currently running #{@pids}"
+        @log.debug "Currently running pids: #{@pids}"
         raise RejectedExecutionException if @is_shutdown
         block_on_max_workers
-        @log.debug "PARENT BEFORE FORK #{Process.pid}"
+        @log.debug "Creating a new child process: parent=#{Process.pid}"
         child_pid = fork do
           begin
-            @log.debug "CHILD #{Process.pid}"
+            @log.debug "Inside the new child process: parent=#{Process.ppid}, child_pid=#{Process.pid}"
             # TODO: which signals to ignore?
             # ignore signals in the child
             %w{ TERM INT HUP SIGUSR2 }.each { |signal| Signal.trap(signal, 'SIG_IGN') }
+            @log.debug "Executing block from child process: parent=#{Process.ppid}, child_pid=#{Process.pid}"
             block.call
-            @log.debug "CHILD #{Process.pid} AFTER block.call"
+            @log.debug "Exiting from child process: parent=#{Process.ppid}, child_pid=#{Process.pid}"
             Process.exit!(0)
           rescue => e
-            @log.error e
-            @log.error "Definitely dying off right here"
+            @log.error "child_pid=#{Process.pid} failed while executing the task: #{e}. Exiting: parent=#{Process.ppid}, child_pid=#{Process.pid}"
             Process.exit!(1)
           end
         end
-        @log.debug "PARENT AFTER FORK #{Process.pid}, child_pid=#{child_pid}"
+        @log.debug "Created a new child process: parent=#{Process.pid}, child_pid=#{child_pid}"
         @pids << child_pid
       end
 
       def shutdown(timeout_seconds)
+        @log.debug "Shutdown requested. Currently running pids: #{@pids}"
         @is_shutdown = true
         remove_completed_pids
 
         unless @pids.empty?
-          @log.info "Exit requested, waiting up to #{timeout_seconds} seconds for child processes to finish"
-
-          # If the timeout_seconds value is set to Float::INFINITY, it will wait indefinitely till all workers finish
-          # their work. This allows us to handle graceful shutdown of workers.
+          # If the timeout_seconds value is set to Float::INFINITY, it will wait
+          # indefinitely till all workers finish their work. This allows us to
+          # handle graceful shutdown of workers.
           if timeout_seconds == Float::INFINITY
             @log.info "Exit requested, waiting indefinitely till all child processes finish"
             remove_completed_pids true while !@pids.empty?
@@ -110,7 +108,7 @@ module AWS
 
           # forcibly kill all remaining children
           unless @pids.empty?
-            @log.warn "Child processes still running, sending KILL signal: #{@pids.join(',')}"
+            @log.warn "Child processes #{@pids} still running, sending KILL signal: #{@pids.join(',')}"
             @pids.each { |pid| Process.kill('KILL', pid) }
           end
         end
@@ -120,44 +118,52 @@ module AWS
       def block_on_max_workers
         @log.debug "block_on_max_workers workers=#{@pids.size}, max_workers=#{@max_workers}"
         if @pids.size >= @max_workers
-          @log.info "Reached maximum number of workers (#{@max_workers}), \
-                     waiting for some to finish"
+          @log.info "Reached maximum number of workers (#{@max_workers}), waiting for some to finish"
           begin
             remove_completed_pids(true)
           end while @pids.size >= @max_workers
         end
+        @log.debug "Available workers: #{@max_workers - @pids.size} out of #{@max_workers}"
       end
 
       private
 
       # Removes all child processes from @pids list that have finished.
-      # Block for at least one child to finish if block argument is set to `true`.
+      # Block for at least one child to finish if block argument is set to
+      # `true`.
       # @api private
       def remove_completed_pids(block=false)
+        @log.debug "Removing completed child processes"
         loop do
           # waitpid2 throws an Errno::ECHILD if there are no child processes,
           # so we don't even call it if there aren't any pids to wait on.
           break if @pids.empty?
+          @log.debug "Current child processes: #{@pids}"
           # Non-blocking wait only returns a non-null pid
           # if the child process has exited.
           pid, status = Process.waitpid2(-1, block ? 0 : Process::WNOHANG)
-          @log.debug "#{pid}"
-          # No more children have finished.
-          break unless pid
-
-          if status.success?
-            @log.debug "Worker #{pid} exited successfully"
+          
+          if pid
+            # We have something to reap
+            @log.debug "Reaping child process=#{pid}"
+            if status.success?
+              @log.debug "Child process #{pid} exited successfully"
+            else
+              @log.error "Child process #{pid} exited with non-zero status code"
+            end
+            # Delete the pid from the list
+            @pids.delete(pid)
+            # Contract is to block only once if block=true. If we are in this code branch and if block=true, it
+            # means we have already blocked once above, hence it is safe to exit now
+            break if block
           else
-            @log.error "Worker #{pid} exited with non-zero status code"
+            # Nothing to reap, exit
+            break
           end
-          @pids.delete(pid)
-          break if pid
+
         end
       end
 
-
-
     end
-
   end
 end

data/lib/aws/decider/options.rb

@@ -25,6 +25,69 @@ module AWS
       end
     end
 
+    # This module refactors out some of the common methods for the Options
+    # classes. Any class including this module should implement
+    # make_runtime_key method and default_keys method. default_keys method
+    # provides an array of keys that are considered to be the default options
+    # for that class. make_runtime_key method converts a passed in default key
+    # to it's corresponding runtime key.
+    module OptionsMethods
+
+      # Retrieves the runtime values for the default options
+      #
+      # @return [Hash]
+      #   The runtime option names and their current values.
+      #
+      def get_runtime_options
+        runtime_options = {}
+        # For the default values that are present, convert the default keys into
+        # runtime keys. i.e. remove 'default_' and 'default_task_' from the key
+        # name and merge their values with the default values
+        get_default_options.each do |key, val|
+          new_key = make_runtime_key(key)
+          new_val = get_options([new_key])
+          runtime_options[new_key] = new_val.empty? ? val : new_val.values.first
+        end
+        runtime_options
+      end
+
+      # Retrieves the default options.
+      #
+      # @return [Hash]
+      #   A hash containing the default option names and their current values.
+      #
+      def get_default_options
+        # Get the default options
+        get_options(default_keys)
+      end
+
+      # Retrieves full options. It merges the runtime options with the remaining
+      # options
+      #
+      # @return [Hash]
+      #   A hash containing the full option names and their current values.
+      #
+      def get_full_options
+        # Initialize an empty hash
+        options_hash = {}
+
+        # Get all the properties held by this class
+        options_keys = self.class.held_properties
+
+        # Remove the unnecessary options (i.e. options not recognized by swf but
+        # only by flow) from the options_keys array.
+        default_keys.concat([:from_class]).each { |x| options_keys.delete(x) }
+
+        # If the value for an option is held by the class, get it and store it
+        # in a hash
+        options_keys.each do |option|
+          options_hash[option] = self.send(option) if self.send(option)
+        end
+        # Merge the options_hash with the runtime options
+        options_hash.merge(self.get_runtime_options)
+      end
+    end
+
     # The base class for all options classes in the AWS Flow Framework for Ruby.
     class Options
       extend Utilities::UpwardLookups
@@ -40,7 +103,9 @@ module AWS
       def self.inherited(child)
         child.precursors ||= []
         default_classes = child.ancestors.map do |precursor|
-          precursor.default_classes if precursor.methods.map(&:to_sym).include? :default_classes
+          if precursor.methods.map(&:to_sym).include? :default_classes
+            precursor.default_classes
+          end
         end.compact.flatten
         child.instance_variable_set("@default_classes", default_classes)
       end
@@ -69,8 +134,15 @@ module AWS
       #
       def get_options(options, extra_to_add = {})
         options = self.class.held_properties.compact if options.empty?
-        set_options = options.select {|option| self.send(option) != nil && self.send(option) != "" }
-        option_values = set_options.map {|option| self.send(option) == Float::INFINITY ? "NONE" : self.send(option) }
+
+        set_options = options.select do |option|
+          self.send(option) != nil && self.send(option) != ""
+        end
+
+        option_values = set_options.map do |option|
+          self.send(option) == Float::INFINITY ? "NONE" : self.send(option)
+        end
+
         result = Hash[set_options.zip(option_values)]
         result.merge(extra_to_add)
       end
@@ -280,17 +352,19 @@ module AWS
       end
     end
 
-    # Exponential retry options for the {ActivityClient#exponential_retry} method.
+    # Exponential retry options for the {ActivityClient#exponential_retry}
+    # method.
     class ExponentialRetryOptions < RetryOptions
       default_classes << RetryDefaults.new
 
-      # The backoff coefficient to use. This is a floating point value that is multiplied with the current retry
-      # interval after every retry attempt.  The default value is `2.0`, which means that each retry will take twice as
-      # long as the previous one.
+      # The backoff coefficient to use. This is a floating point value that is
+      # multiplied with the current retry interval after every retry attempt.
+      # The default value is `2.0`, which means that each retry will take twice
+      # as long as the previous one.
       property(:backoff_coefficient, [lambda(&:to_i)])
 
-      # The retry expiration interval, in seconds. This will be increased after every retry attempt by the factor
-      # provided in `backoff_coefficient`.
+      # The retry expiration interval, in seconds. This will be increased after
+      # every retry attempt by the factor provided in `backoff_coefficient`.
       property(:retry_expiration_interval_seconds, [lambda(&:to_i)])
 
       # @api private
@@ -306,10 +380,10 @@ module AWS
 
       # The default task start-to-close timeout duration. The default value is
       # `30`.
-      def task_start_to_close_timeout; 30; end
+      def default_task_start_to_close_timeout; 30; end
 
       # The default child workflow policy. The default value is `TERMINATE`.
-      def child_policy; :TERMINATE; end
+      def default_child_policy; "TERMINATE"; end
 
       # Returns a list of tags for the workflow. The default value is an empty
       # array (no tags).
@@ -411,44 +485,47 @@ module AWS
     #   * It must not contain the literal string "arn".
     #
     class WorkflowOptions < Options
-      properties(:version, :input, :workflow_id, :execution_start_to_close_timeout, :task_start_to_close_timeout, :task_list, :execution_method)
+      include OptionsMethods
+
+      properties(
+        :version,
+        :input,
+        :workflow_id,
+        :execution_start_to_close_timeout,
+        :task_start_to_close_timeout,
+        :task_list,
+        :execution_method
+      )
+
+      # Adding default properties
+      properties(
+        :default_task_start_to_close_timeout,
+        :default_execution_start_to_close_timeout,
+        :default_task_list
+      )
+      property(:default_child_policy, [lambda(&:to_s), lambda(&:upcase)])
+
+
       property(:tag_list, [])
       property(:child_policy, [lambda(&:to_s), lambda(&:upcase)])
       property(:data_converter, [])
+
       default_classes << WorkflowDefaults.new
 
-      # Returns a hash containing the runtime workflow options.
-      #
-      # @return [Hash] A hash of options with corresponding values.
-      #
-      def get_full_options
-        result = {}
-        usable_properties = self.class.held_properties
-        usable_properties.delete(:from_class)
-        usable_properties.each do |option|
-          result[option] = self.send(option) if self.send(option) && self.send(option) != ""
-        end
-        result
+      # This method provides the default option keys for workflows
+      def default_keys
+        [:default_task_start_to_close_timeout,
+         :default_execution_start_to_close_timeout,
+         :default_task_list,
+         :default_child_policy]
+      end
+
+      # This method converts default option keys to runtime keys by replacing
+      # "default_" in the key name
+      def make_runtime_key(key)
+        key.to_s.gsub(/default_/, "").to_sym
       end
-    end
 
-    # Provides the set of workflow options along with defaults.
-    #
-    # @!attribute default_task_start_to_close_timeout
-    #   (see WorkflowDefaults#task_start_to_close_timeout)
-    #
-    # @!attribute default_execution_start_to_close_timeout
-    #   (see WorkflowDefaults#execution_start_to_close_timeout)
-    #
-    # @!attribute default_task_list
-    #   (see WorkflowOptions#task_list)
-    #
-    # @!attribute default_child_policy
-    #   (see WorkflowDefaults#child_policy)
-    #
-    class WorkflowOptionsWithDefaults < WorkflowOptions
-      properties(:default_task_start_to_close_timeout, :default_execution_start_to_close_timeout, :default_task_list)
-      property(:default_child_policy, [lambda(&:to_s), lambda(&:upcase)])
     end
 
     # Options for {WorkflowClient#start_execution}.
@@ -473,6 +550,14 @@ module AWS
     # Defaults for the {ActivityOptions} class.
     class ActivityDefaults < Defaults
 
+      # The default schedule-to-start timeout for activity tasks. This timeout
+      # represents the time, in seconds, between when the activity task is first
+      # scheduled to when it is started.
+      #
+      # This default can be overridden when scheduling an activity task. You can
+      # set this value to "NONE" to imply no timeout value.
+      def default_task_schedule_to_start_timeout; Float::INFINITY; end
+
       # The default schedule-to-close timeout for activity tasks. This timeout
       # represents the time, in seconds, between when the activity task is first
       # scheduled to when it is closed (whether due to success, failure, or a
@@ -480,9 +565,17 @@ module AWS
       #
       # This default can be overridden when scheduling an activity task. You can
       # set this value to "NONE" to imply no timeout value.
-      #
       def default_task_schedule_to_close_timeout;  Float::INFINITY; end
 
+      # The default start-to-close timeout for activity tasks. This timeout
+      # represents the time, in seconds, between when the activity task is first
+      # started to when it is closed (whether due to success, failure, or a
+      # timeout).
+      #
+      # This default can be overridden when scheduling an activity task. You can
+      # set this value to "NONE" to imply no timeout value.
+      def default_task_start_to_close_timeout; Float::INFINITY; end
+
       # The default maximum time, in seconds, before which a worker processing a
       # task of this type must report progress.  If the timeout is exceeded, the
       # activity task is automatically timed out. If the worker subsequently
@@ -490,34 +583,14 @@ module AWS
       #
       # This default can be overridden when scheduling an activity task. You can
       # set this value to "NONE" to imply no timeout value.
-      #
       def default_task_heartbeat_timeout; Float::INFINITY; end
 
-      # The default schedule-to-close timeout. This timeout represents the time
-      # between when the activity task is first scheduled to when it is closed
-      # (whether due to success, failure, or a timeout).
-      #
-      # This default can be overridden when scheduling an activity task. You can
-      # set this value to "NONE" to imply no timeout value.
-      #
-      def schedule_to_close_timeout; Float::INFINITY; end
-
-      # The default maximum time before which a worker processing a task of this
-      # type must report progress. If the timeout is exceeded, the activity task
-      # is automatically timed out. If the worker subsequently attempts to
-      # record a heartbeat or returns a result, it will be ignored. This default
-      # can be overridden when scheduling an activity task.
-      #
-      # This default can be overridden when scheduling an activity task. You can
-      # set this value to "NONE" to imply no timeout value.
-      #
-      def heartbeat_timeout; Float::INFINITY; end
-
       def data_converter; FlowConstants.default_data_converter; end
     end
 
 
-    # Options to use on an activity or decider. The following options are defined:
+    # Options to use on an activity or decider. The following options are
+    # defined:
     #
     # @!attribute default_task_heartbeat_timeout
     #   The optional default maximum time, specified when registering the
@@ -567,16 +640,52 @@ module AWS
     #   decision.
     #
     class ActivityOptions < Options
+      include OptionsMethods
+
+      properties(
+        :heartbeat_timeout,
+        :task_list,
+        :schedule_to_close_timeout,
+        :schedule_to_start_timeout,
+        :start_to_close_timeout,
+        :version,
+        :input
+      )
+
+      # Adding default properties
+      properties(
+        :default_task_heartbeat_timeout,
+        :default_task_list,
+        :default_task_schedule_to_close_timeout,
+        :default_task_schedule_to_start_timeout,
+        :default_task_start_to_close_timeout,
+      )
 
-      class << self
-        attr_reader :default_options,  :runtime_options
-      end
-      properties(:default_task_heartbeat_timeout, :default_task_list, :default_task_schedule_to_close_timeout, :default_task_schedule_to_start_timeout, :default_task_start_to_close_timeout, :heartbeat_timeout, :task_list, :schedule_to_close_timeout, :schedule_to_start_timeout, :start_to_close_timeout, :version, :input)
       property(:manual_completion, [lambda {|x| x == true}])
       property(:data_converter, [])
 
       default_classes << ActivityDefaults.new
 
+      # This method provides the default option keys for activities
+      def default_keys
+        [:default_task_heartbeat_timeout,
+         :default_task_schedule_to_close_timeout,
+         :default_task_schedule_to_start_timeout,
+         :default_task_start_to_close_timeout,
+         :default_task_list]
+      end
+
+      # This method converts default option keys to runtime keys by replacing
+      # "default_task_" in the key name. It handles the exception of task_list
+      # where only "default_" needs to be replaced.
+      def make_runtime_key(key)
+        if key =~ /task_list/
+          key.to_s.gsub(/default_/, "").to_sym
+        else
+          key.to_s.gsub(/default_task_/, "").to_sym
+        end
+      end
+
       # Gets the activity prefix name.
       #
       # @return [String]
@@ -657,28 +766,6 @@ module AWS
         super(default_options, use_defaults)
       end
 
-      # Retrieves the runtime options for this activity. The runtime options returned are:
-      #
-      # * :heartbeat_timeout
-      # * :task_list
-      # * :schedule_to_close_timeout
-      # * :schedule_to_start_timeout
-      # * :start_to_close_timeout
-      #
-      # For a description of each of these options, see {#initialize}.
-      #
-      # @return [Hash]
-      #   The runtime option names and their current values.
-      #
-      def get_runtime_options
-        result = get_options([:heartbeat_timeout, :task_list, :schedule_to_close_timeout, :schedule_to_start_timeout, :start_to_close_timeout])
-        default_options = get_options([:default_task_heartbeat_timeout, :default_task_schedule_to_close_timeout, :default_task_schedule_to_start_timeout, :default_task_start_to_close_timeout])
-        default_option_keys, default_option_values = default_options.keys, default_options.values
-        default_option_keys.map! { |option| option.to_s.gsub(/default_task_/, "").to_sym }
-        default_hash = Hash[default_option_keys.zip(default_option_values)]
-        default_hash.merge(result)
-      end
-
       property(:_exponential_retry, [])
 
       # Retries the supplied block with exponential retry logic.
@@ -690,35 +777,6 @@ module AWS
         retry_options = Utilities::interpret_block_for_options(ExponentialRetryOptions, block)
         @_exponential_retry = retry_options
       end
-
-      # Retrieves the runtime options for this activity.
-      #
-      # @return [Hash]
-      #   A hash containing the runtime option names and their current values.
-      #
-      def get_full_options
-        options_hash = self.get_runtime_options
-        [:task_list, :version, :_exponential_retry, :prefix_name, :return_on_start, :manual_completion, :data_converter].each do |attribute|
-          options_hash.merge!(attribute => self.send(attribute)) if self.send(attribute)
-        end
-        options_hash
-      end
-
-      # Retrieves the default options for this activity.
-      #
-      # @return [Hash]
-      #   A hash containing the default option names and their current values.
-      #
-      #   The options retrieved are:
-      #
-      #   * :default_task_heartbeat_timeout
-      #   * :default_task_schedule_to_close_timeout
-      #   * :default_task_schedule_to_start_timeout
-      #   * :default_task_start_to_close_timeout
-      #
-      def get_default_options
-        get_options([:default_task_heartbeat_timeout, :default_task_schedule_to_close_timeout, :default_task_schedule_to_start_timeout, :default_task_start_to_close_timeout])
-      end
     end
 
     # Runtime options for an activity.