aws-flow 2.3.1 → 2.4.0

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    OWUxYzEyYWVmYzRiZmM3MGIyYzNjN2YwMGZmNmVlMDQ2Nzg4ZjY5NQ==
+    MDQyOWZlMGM0YjZiZDcwMmQyNjhjNGFhODY3NzE1ZjZhZDhiNmE4YQ==
   data.tar.gz: !binary |-
-    MzhhZTE1ODE3ZmQ4YTYxN2I3Zjc1ZjgxM2ExOTM2MWM5YTg0MTBmMQ==
+    ZTUxM2MwZTJlNzMzMmMxOWY5NTM0MGM2Y2UyMjA2MjE5NWQxMmJkOA==
 SHA512:
   metadata.gz: !binary |-
-    MWRlY2NmMGU2ZDYzMjA3ZmRiNDg1M2I1MDNmYTE2M2MwYzc4OTIxYmI4OWZi
-    Y2JiNDAwYzk5MmUyZGIyY2Y2Yjc0YjVkZjJhMjNjZWM5YjhmMzc4ZjExYTM3
-    MzBkYmU3N2EzZGJhNTU2YTc5ZTM3YTE4YzE0MmJjYjFmZDlmZjk=
+    MTJiNjFiODFlY2IyN2Q5ODc2YTViNjY4MTM0NGIwZTc0MzE1Y2FmYzM5NjUw
+    MDM5YzkyZTFkNDYzOWFmMDA4YjVmNmY5ODM3MGVkMzQwYjMyMmNiNTlmNmU0
+    NzYyYWNjNmUwYTJkNDg4OTU4ZDVkMjE5MWUxZjA0MDFkZmZiZTM=
   data.tar.gz: !binary |-
-    MjIxZjgxZjRmZTNmMjExYjBhMWJmMDA3NjQxMDdjZjFlMTMwYjJjZjI4Mjc5
-    YjllMzJlYjBkZjQ0MmMzOTc5NWI0NjllM2FkZDc4MmJhMWVjMTRhYjA1NjM1
-    NWI2NzRhOTZlMGRlMWE4MGMyZmNmYWI2OGVkNDBjNTYwNzdiY2Y=
+    ZWY5NmZjZTE2OTg3NjdiMzg5MjBmNjk3OGUyODY5ODQ5NGJlMjc1OGZkMjI5
+    YTA2NjM3MTA4MjFkMWNmZTVmODVjZTQ2ODZmZDFlMGM5Njk2MGNiMzAxMDc3
+    M2EwOGNmODg2OWQzZjAwM2EyNjhjYTAxYjQyMjJmYjI0OGI1YzE=

data/aws-flow.gemspec

@@ -7,11 +7,12 @@ Gem::Specification.new do |s|
   s.summary     = "AWS Flow Framework for Ruby"
   s.description = "Library to provide the AWS Flow Framework for Ruby"
   s.authors     = "Michael Steger, Paritosh Mohan, Jacques Thomas"
-  s.executables = ["aws-flow-ruby"]
+  s.executables = ["aws-flow-ruby", "aws-flow-utils"]
   s.homepage 	= "https://aws.amazon.com/swf/details/flow/"
   s.email       = ''
   s.files       = `git ls-files`.split("\n").reject {|file| file =~ /aws-flow-core/}
   s.require_paths << "lib/aws/"
   s.required_ruby_version = ">= 1.9.1"
-  s.add_dependency "aws-sdk-v1", "~> 1", ">= 1.60.1"
+  s.add_dependency "aws-sdk-v1", "~> 1", ">= 1.60.2"
+  s.add_dependency "lru_redux", "~> 0", ">= 0.8.1"
 end

data/bin/aws-flow-ruby

@@ -2,4 +2,4 @@
 
 require 'runner'
 
-AWS::Flow::Runner.main()
+AWS::Flow::Runner.main

data/bin/aws-flow-utils

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require 'utils'
+
+AWS::Flow::Utils.main

data/lib/aws/decider.rb

@@ -47,8 +47,15 @@ require "aws/decider/exceptions"
 require "aws/decider/task_poller"
 require "aws/decider/flow_defaults"
 require "aws/decider/implementation"
+require "aws/decider/starter"
 require "aws/decider/version"
 
+# Add the runner files
+require 'aws/runner'
+
+# Add the Workflow Template files
+require 'aws/templates'
+
 # @api private
 def get_const(name)
   name = name.split('::').reverse

data/lib/aws/decider/async_retrying_executor.rb

@@ -125,7 +125,7 @@ module AWS
       #   otherwise.
       #
       def isRetryable(failure)
-        if failure.respond_to? :cause
+        if failure.respond_to?(:cause) && !failure.cause.nil?
           failure_class = failure.cause.class
         else
           failure_class = failure.class

data/lib/aws/decider/data_converter.rb

@@ -19,12 +19,23 @@ module AWS
     # properly, and they are very handy for debugging.
     class YAMLDataConverter
 
+      # Serializes a ruby object into a YAML string.
+      #
+      # @param object
+      #   The object that needs to be serialized into a string.
+      #
       def dump(object)
         if object.is_a? Exception
           return YAML.dump_stream(object, object.backtrace)
         end
         object.to_yaml
       end
+
+      # Deserializes a YAML string into a ruby object.
+      #
+      # @param source
+      #   The source YAML string that needs to be deserialized into a ruby object.
+      #
       def load(source)
         return nil if source.nil?
         output = YAML.load source
@@ -40,5 +51,155 @@ module AWS
       end
     end
 
+    # S3DataConverter uses YAMLDataConverter internally to serialize and
+    # deserialize ruby objects. Additionally it stores objects larger than
+    # 32k characeters in AWS S3 and returns a serialized s3 link to be
+    # deserialized remotely. It caches objects locally to minimize calls to S3.
+    #
+    # AWS Flow Framework for Ruby doesn't delete files from S3 to prevent loss
+    # of data. It is recommended that users use Object Lifecycle Management in
+    # AWS S3 to auto delete files.
+    #
+    # More information about object expiration can be found at:
+    # http://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectExpiration.html
+    class S3DataConverter
+
+      require 'lru_redux'
+
+      # S3Cache is a wrapper around the LruRedux cache.
+      class S3Cache
+        attr_reader :cache
+
+        MAX_SIZE = 1000
+
+        def initialize
+          @cache = LruRedux::ThreadSafeCache.new(MAX_SIZE)
+        end
+
+        # Cache lookup
+        def [](key)
+          @cache[key]
+        end
+
+        # Cache entry
+        def []=(key, value)
+          @cache[key] = value
+        end
+
+      end
+
+      attr_reader :converter, :bucket, :cache
+
+      class << self
+        attr_accessor :conv
+
+        # Ensures singleton
+        def converter
+          return self.conv if self.conv
+          name = ENV['AWS_SWF_BUCKET_NAME']
+          if name.nil?
+            raise "Need a valid S3 bucket name to initialize S3DataConverter."\
+              " Please set the AWS_SWF_BUCKET_NAME environment variable with the"\
+              " bucket name."
+          end
+          self.conv ||= self.new(name)
+          return self.conv
+        end
+
+      end
+
+      def initialize(bucket)
+        @bucket = bucket
+        @cache = S3Cache.new
+        s3 = AWS::S3.new
+        s3.buckets.create(bucket) unless s3.buckets[bucket].exists?
+        @converter = FlowConstants.default_data_converter
+      end
+
+      # Serializes a ruby object into a string. If the size of the converted
+      # string is greater than 32k characters, the string is uploaded to an
+      # AWS S3 file and a serialized hash containing the filename is returned
+      # instead. The filename is generated at random in the following format -
+      # rubyflow_data_<UUID>.
+      #
+      # The format of the returned serialized hash is - { s3_filename: <filename> }
+      #
+      # @param object
+      #   The object that needs to be serialized into a string. By default it
+      #   serializes the object into a YAML string.
+      #
+      def dump(object)
+        string = @converter.dump(object)
+        ret = string
+        if string.size > 32768
+          filename = put_to_s3(string)
+          ret = @converter.dump({ s3_filename: filename })
+        end
+        ret
+      end
+
+      # Deserializes a string into a ruby object. If the deserialized
+      # string is a ruby hash of the format { s3_filename: <filename> }, then
+      # it will first look for the file in a local cache. In case of a cache miss, 
+      # it will try to download the file from AWS S3, deserialize the contents
+      # of the file and return the new object.
+      #
+      # @param source
+      #   The source that needs to be deserialized into a ruby object. By
+      #   default it expects the source to be a YAML string.       #
+      def load(source)
+        object = @converter.load(source)
+        ret = object
+        if object.is_a?(Hash) && object[:s3_filename]
+          ret = @converter.load(get_from_s3(object[:s3_filename]))
+        end
+        ret
+      end
+
+      # Helper method to write a string to an s3 file. A random filename is
+      # generated of the format - rubyflow_data_<UUID>
+      #
+      # @param string
+      #   The string to be uploaded to S3
+      #
+      # @api private
+      def put_to_s3(string)
+        filename = "rubyflow_data_#{SecureRandom.uuid}"
+        s3 = AWS::S3.new
+        s3.buckets[@bucket].objects.create(filename, string)
+        @cache[filename] = string
+        return filename
+      end
+
+      # Helper method to read an s3 file
+      # @param s3_filename
+      #   File name to be deleted
+      #
+      # @api private
+      def get_from_s3(s3_filename)
+        return @cache[s3_filename] if @cache[s3_filename]
+        s3 = AWS::S3.new
+        s3_object = s3.buckets[@bucket].objects[s3_filename]
+        begin
+          ret = s3_object.read
+          @cache[s3_filename] = ret
+        rescue AWS::S3::Errors::NoSuchKey => e
+          raise "Could not find key #{s3_filename} in bucket #{@bucket} on S3. #{e}"
+        end
+        return ret
+      end
+
+      # Helper method to delete an s3 file
+      # @param s3_filename
+      #   File name to be deleted
+      #
+      # @api private
+      def delete_from_s3(s3_filename)
+        s3 = AWS::S3.new
+        s3.buckets[@bucket].objects.delete(s3_filename)
+      end
+
+    end
+
   end
 end

data/lib/aws/decider/decider.rb

@@ -243,16 +243,18 @@ module AWS
     #   Sets or returns the {WorkflowOptions} for this decider.
     #
     module Workflows
-      attr_accessor :version
+      attr_accessor :version, :workflows
       extend Utilities::UpwardLookups
       @precursors ||= []
       def look_upwards(variable)
-        precursors = self.ancestors.dup
-        precursors.delete(self)
-        results = precursors.map { |x| x.send(variable) if x.methods.map(&:to_sym).include? variable }.compact.flatten.uniq
+        unless self.ancestors.nil?
+          precursors = self.ancestors.dup
+          precursors.delete(self)
+          results = precursors.map { |x| x.send(variable) if x.methods.map(&:to_sym).include? variable }.compact.flatten.uniq
+        end
       end
       property(:workflows, [])
-      @workflows = []
+      @workflows ||= []
       def self.extended(base)
         base.send :include, InstanceMethods
       end
@@ -264,11 +266,12 @@ module AWS
         if input
           @entry_point = input
           workflow_type = WorkflowType.new(self.to_s + "." + input.to_s, nil, WorkflowRegistrationOptions.new(:execution_method => input))
-          self.workflows.each { |workflow| workflow.name = self.to_s + "." + input.to_s }
-          self.workflows.each do |workflow|
+          @workflows ||= []
+          @workflows.each { |workflow| workflow.name = self.to_s + "." + input.to_s }
+          @workflows.each do |workflow|
             workflow.options = WorkflowRegistrationOptions.new(:execution_method => input)
           end
-          self.workflows = self.workflows << workflow_type
+          @workflows << workflow_type
         end
         return @entry_point if @entry_point
         raise "You must set an entry point on the workflow definition"
@@ -278,8 +281,9 @@ module AWS
       # @api private
       def version(arg = nil)
         if arg
-          self.workflows.each { |workflow| workflow.version = arg }
-          self.workflows = self.workflows << WorkflowType.new(nil, arg, WorkflowOptions.new)
+          @workflows ||= []
+          @workflows.each { |workflow| workflow.version = arg }
+          @workflows << WorkflowType.new(nil, arg, WorkflowOptions.new)
         end
         return @version
       end
@@ -325,9 +329,18 @@ module AWS
         instance_variable_get(client_name)
       end
 
+      # Convenience method to set the child workflow client
+      def child_workflow_client(name, &block)
+        client_name = "@child_client_#{name}"
+        define_method(name) do
+          return instance_variable_get(client_name) if instance_variable_get(client_name)
+          client = AWS::Flow.send(:workflow_client, nil, nil, &block)
+        end
+        instance_variable_get(client_name)
+      end
 
       # @api private
-      def _options; self.workflows; end
+      def _options; @workflows; end
 
       # Defines a new workflow.
       #
@@ -343,7 +356,8 @@ module AWS
           options.execution_method = workflow_name
           prefix_name = options.prefix_name || self.to_s
           workflow_type = WorkflowType.new(prefix_name.to_s + "." + workflow_name.to_s, options.version, options)
-          self.workflows = self.workflows << workflow_type
+          @workflows ||= []
+          @workflows << workflow_type
         end
       end
 
@@ -368,7 +382,7 @@ module AWS
         data_converter = options[:data_converter]
         signal_name = options[:signal_name]
         signal_name ||= method_name.to_s
-        data_converter ||= FlowConstants.default_data_converter
+        data_converter ||= FlowConstants.data_converter
         @signals ||= {}
         @signals[signal_name] = MethodPair.new(method_name, data_converter)
         @signals
@@ -438,7 +452,6 @@ module AWS
           client
         end
 
-
         # Creates a timer on the workflow that executes the supplied block after a specified delay.
         #
         # @param delay_seconds

data/lib/aws/decider/flow_defaults.rb

@@ -113,6 +113,26 @@ module AWS
       INFINITY = -1
       RETENTION_DEFAULT = 7
       NUM_OF_WORKERS_DEFAULT = 1
+
+      def self.defaults
+        {
+          domain: "FlowDefault",
+          prefix_name: "FlowDefaultWorkflowRuby",
+          execution_method: "start",
+          version: "1.0",
+          # execution timeout (1 hour)
+          execution_start_to_close_timeout: "3600",
+          data_converter: self.data_converter,
+          schedule_to_start_timeout: 60,
+          start_to_close_timeout: 60,
+          retry_policy: { maximum_attempts: 3 },
+          task_list: "flow_default_ruby",
+          result_activity_prefix: "FlowDefaultResultActivityRuby",
+          result_activity_version: "1.0",
+          result_activity_method: "run"
+        }
+      end
+
       @exponential_retry_maximum_attempts = Float::INFINITY
       @exponential_retry_maximum_retry_interval_seconds = -1
       @exponential_retry_retry_expiration_seconds = -1
@@ -159,6 +179,14 @@ module AWS
         random.rand(max_value)
       end
 
+      # Selects the data converter to use. By default, YAMLDataConverter is
+      # used. S3DataConverter is used when AWS_SWF_BUCKET_NAME environment
+      # variable is set.
+      def self.data_converter
+        return self.default_data_converter unless ENV['AWS_SWF_BUCKET_NAME']
+        S3DataConverter.converter
+      end
+
       @default_data_converter = YAMLDataConverter.new
       @use_worker_task_list = "USE_WORKER_TASK_LIST"
     end

data/lib/aws/decider/implementation.rb

@@ -41,7 +41,6 @@ module AWS
       AWS::Flow.send(:workflow_client, service, domain, &block)
     end
 
-
     # Execute a block with retries within a workflow context.
     #
     # @param options

data/lib/aws/decider/options.rb

@@ -340,7 +340,7 @@ module AWS
     class WorkflowDefaults < Defaults
 
       # The default data converter. By default, this is {YAMLDataConverter}.
-      def data_converter; FlowConstants.default_data_converter; end
+      def data_converter; FlowConstants.data_converter; end
 
     end
 
@@ -554,7 +554,7 @@ module AWS
 
     # Defaults for the {ActivityOptions} class.
     class ActivityDefaults < Defaults
-      def data_converter; FlowConstants.default_data_converter; end
+      def data_converter; FlowConstants.data_converter; end
     end
 
     # Default values for a registered activity type. These values are set by

data/lib/aws/decider/starter.rb

@@ -0,0 +1,207 @@
+module AWS
+  module Flow
+
+    # Utility method used to start a workflow execution with the service.
+    #
+    # @param [String or Class (that extends AWS::Flow::Workflows)] workflow
+    #   Represents an AWS Flow Framework workflow class. If not provided,
+    #   details of the workflow must be passed via the opts Hash.
+    #
+    # @param [Hash] input
+    #   Input hash for the workflow execution
+    #
+    # @param [Hash] opts
+    #   Hash of options to configure the workflow execution
+    #
+    # @option opts [String] *Required* :domain
+    #
+    # @option opts [String] *Required* :version
+    #
+    # @option opts [String] *Optional* :prefix_name
+    #   Must be specified if workflow is not passed in as an argument
+    #
+    # @option opts [String] *Optional* :execution_method
+    #   Must be specified if workflow is not passed in as an argument
+    #
+    # @option opts [String] *Optional* :from_class
+    #
+    # @option opts [String] *Optional* :workflow_id
+    #
+    # @option opts [Integer] *Optional* :execution_start_to_close_timeout
+    #
+    # @option opts [Integer] *Optional* :task_start_to_close_timeout
+    #
+    # @option opts [Integer] *Optional* :task_priority
+    #
+    # @option opts [String] *Optional* :task_list
+    #
+    # @option opts [String] *Optional* :child_policy
+    #
+    # @option opts [Array] *Optional* :tag_list
+    #
+    # @option opts *Optional* :data_converter
+    #
+    # Usage -
+    #
+    # 1) Passing a fully qualified workflow <prefix_name>.<execution_method> name -
+    #
+    #    AWS::Flow::start_workflow("HelloWorkflow.say_hello", "world", {
+    #      domain: "FooDomain",
+    #      version: "1.0"
+    #      ...
+    #    })
+    #
+    # 2) Passing workflow class name with other details in the options hash -
+    #
+    #    AWS::Flow::start_workflow("HelloWorkflow", "world", {
+    #      domain: "FooDomain",
+    #      execution_method: "say_hello",
+    #      version: "1.0"
+    #      ...
+    #    })
+    #
+    # 3) Acquiring options using the :from_class option -
+    #
+    #    AWS::Flow::start_workflow(nil, "hello", {
+    #      domain: "FooDomain",
+    #      from_class: "HelloWorkflow"
+    #    })
+    #
+    #    # This will take all the required options from the HelloWorkflow class.
+    #    # If execution_method options is not passed in, it will use the first
+    #    # workflow method in the class.
+    #
+    # 4) All workflow options are present in the options hash. This is the case
+    #    when this method is called by AWS::Flow#start
+    #
+    #    AWS::Flow::start_workflow(nil, "hello", {
+    #      domain: "FooDomain",
+    #      prefix_name: "HelloWorkflow",
+    #      execution_method: "say_hello",
+    #      version: "1.0",
+    #      ...
+    #    })
+    def self.start_workflow(workflow = nil, input, opts)
+
+      raise ArgumentError, "Please provide an options hash" if opts.nil? || !opts.is_a?(Hash)
+
+      options = opts.dup
+
+      # Get the domain out of the options hash.
+      domain = options.delete(:domain)
+
+      raise ArgumentError, "You must provide a :domain in the options hash" if domain.nil?
+
+      if options[:from_class]
+        # Do nothing. Use options as they are. They will be taken care of in the
+        # workflow client
+      elsif workflow.nil?
+        # This block is usually executed when #start_workflow is called from
+        # #start. All options required to start the workflow must be present
+        # in the options hash.
+        prefix_name = options[:prefix_name] || options[:workflow_name]
+        # Check if required options are present
+        raise ArgumentError, "You must provide a :prefix_name in the options hash" unless prefix_name
+        raise ArgumentError, "You must provide an :execution_method in the options hash" unless options[:execution_method]
+        raise ArgumentError, "You must provide a :version in the options hash" unless options[:version]
+      else
+        # When a workflow class name is given along with some options
+
+        # If a fully qualified workflow name is given, split it into prefix_name
+        # and execution_method
+        prefix_name, execution_method = workflow.to_s.split(".")
+        # If a fully qualified name is not given, then look for it in the options
+        # hash
+        execution_method ||= options[:execution_method]
+
+        # Make sure all required options are present
+        raise ArgumentError, "You must provide an :execution_method in the options hash" unless execution_method
+        raise ArgumentError, "You must provide a :version in the options hash" unless options[:version]
+
+        # Set the :prefix_name and :execution_method options correctly
+        options.merge!(
+          prefix_name: prefix_name,
+          execution_method: execution_method,
+        )
+      end
+
+      swf = AWS::SimpleWorkflow.new
+      domain = swf.domains[domain]
+
+      # Get a workflow client for the domain
+      client = workflow_client(domain.client, domain) { options }
+
+      # Start the workflow execution
+      client.start_execution(input)
+    end
+
+    # Starts an Activity or a Workflow Template execution using the
+    # default workflow class FlowDefaultWorkflowRuby
+    #
+    # @param [String or AWS::Flow::Templates::TemplateBase] name_or_klass
+    #   The Activity or the Workflow Template that needs to be scheduled via
+    #   the default workflow. This argument can either be a string that
+    #   represents a fully qualified activity name - <ActivityClass>.<method_name>
+    #   or it can be an instance of AWS::Flow::Templates::TemplateBase
+    #
+    # @param [Hash] input
+    #   Input hash for the workflow execution
+    #
+    # @param [Hash] opts
+    #   Additional options to configure the workflow or activity execution.
+    #
+    # @option opts [true, false] :wait
+    #   *Optional* This boolean flag can be set to true if the result of the
+    #   task is required. Default value is false.
+    #
+    # @option opts [Integer] :wait_timeout
+    #   *Optional* This sets the timeout value for :wait. Default value is
+    #   nil.
+    #
+    # @option opts [Hash] :exponential_retry
+    #   A hash of {AWS::Flow::ExponentialRetryOptions}. Default value is -
+    #   { maximum_attempts: 3 }
+    #
+    # @option opts [String] *Optional* :domain
+    #   Default value is FlowDefault
+    #
+    # @option opts [Integer] *Optional* :execution_start_to_close_timeout
+    #   Default value is 3600 seconds (1 hour)
+    #
+    # @option opts [String] *Optional* :workflow_id
+    #
+    # @option opts [Integer] *Optional* :task_priority
+    #   Default value is 0
+    #
+    # @option opts [String] *Optional* :tag_list
+    #   By default, the name of the activity task gets added to the workflow's
+    #   tag_list
+    #
+    # @option opts *Optional* :data_converter
+    #   Default value is {AWS::Flow::YAMLDataConverter}. To use the
+    #   {AWS::Flow::S3DataConverter}, set the AWS_SWF_BUCKET_NAME environment
+    #   variable name with a valid AWS S3 bucket name.
+    #
+    # @option opts *Optional* A hash of {AWS::Flow::ActivityOptions}
+    #
+    # Usage -
+    #
+    #    AWS::Flow::start("<ActivityClassName>.<method_name>", <input_hash>,
+    #    <options_hash> )
+    #
+    # Example -
+    #
+    # 1) Start an activity execution -
+    #    AWS::Flow::start("HelloWorldActivity.say_hello", { name: "World" })
+    #
+    # 2) Start an activity execution with overriden options -
+    #    AWS::Flow::start("HelloWorldActivity.say_hello", { name: "World" }, {
+    #      exponential_retry: { maximum_attempts: 10 } }
+    #    )
+    #
+    def self.start(name_or_klass, input, options = {})
+      AWS::Flow::Templates.start(name_or_klass, input, options)
+    end
+
+  end
+end