aws-flow 2.3.1 → 2.4.0

data/lib/aws/utils.rb

@@ -0,0 +1,270 @@
+module AWS
+  module Flow
+    module Utils
+
+      require 'optparse'
+      require 'fileutils'
+      require 'json'
+
+      # Interprets the command-line paramters pased in from the shell.
+      #
+      # @api private
+      def self.parse_command_line(argv = ARGV)
+        options = {}
+        optparse = OptionParser.new do |opts|
+          opts.banner = "Usage: aws-flow-utils -c <command> [options]"
+
+          opts.separator ""
+          options[:deploy] = {enabled: false}
+          opts.on('-c', '--command COMMAND', [:eb,:local], "Specify the command to run. (eb, local)") do |f|
+            options[:deploy][:enabled] = true
+            options[:deploy][:eb] = true if f == :eb
+            options[:deploy][:local] = true if f == :local
+          end
+
+          opts.separator ""
+          opts.separator "Commands"
+          opts.separator ""
+          opts.separator "\tlocal: Generates an AWS Flow Framework for Ruby "\
+            "application skeleton."
+          opts.separator "\t   eb: Generates an AWS Flow Framework for Ruby "\
+            "application skeleton compatible with AWS Elastic Beanstalk."
+          opts.separator ""
+          opts.separator "Specific options"
+          opts.separator ""
+
+          opts.on('-n', '--name NAME', "Set the name of the application") do |f|
+            options[:deploy][:name] = f
+          end
+
+          opts.on('-r', '--region [REGION]', "Set the AWS Region. Default "\
+                  "value is taken from environment variable AWS_REGION if "\
+                  "it is set.") do |f|
+            options[:deploy][:region] = f.downcase
+          end
+
+          opts.on('-p', '--path [PATH]', "Set the location where the AWS Flow "\
+                  "for Ruby application will be created. (Default is '.')") do |f|
+            options[:deploy][:path] = f
+          end
+
+          opts.on('-a', '--act_path [PATH]', "Set the location where activity "\
+                  "classes reside.") do |f|
+            options[:deploy][:act] = f
+          end
+
+          opts.on('-w', '--wf_path [PATH]', "Set the location where workflow "\
+                  "classes reside.") do |f|
+            options[:deploy][:wf] = f
+          end
+
+          opts.on('-A', "--activities x,y,z", Array, "Set the names of Activity "\
+                  "classes") do |f|
+            options[:deploy][:activities] = f
+          end
+
+          opts.on('-W', "--workflows x,y,z", Array, "Set the names of Workflow "\
+                  "classes") do |f|
+            options[:deploy][:workflows] = f
+          end
+
+        end
+
+        optparse.parse!(argv)
+
+        return options
+      end
+
+      class FlowUtils
+        class << self
+          attr_accessor :utils
+        end
+        self.utils ||= []
+      end
+
+      # Initializes an AWS Flow Framework for Ruby application
+      class InitConfig
+        # Self register with the FlowUtils class
+        FlowUtils.utils << self
+
+        # Generates the appropriate 1-Click URL for beanstalk deployments based on
+        # the options passed by the user
+        def self.generate_url(options)
+          url = "https://console.aws.amazon.com/elasticbeanstalk/"
+          url = "#{url}?region=#{options[:region]}#/newApplication?"
+          url = "#{url}applicationName=#{options[:name]}"
+          url = "#{url}&solutionStackName=Ruby"
+          url = "#{url}&instanceType=m1.large"
+          puts "AWS Elastic Beanstalk 1-Click URL: #{url}"
+        end
+
+        # Generates the necessary file structure for a valid AWS Flow Ruby
+        # application that can be deployed to Elastic Beanstalk
+        def self.generate_files(options)
+          dirname = "#{options[:path]}/#{options[:name]}"
+          puts "Your AWS Flow Framework for Ruby application will be located at: "\
+            "#{File.absolute_path(dirname)}/"
+          create_app_dir(dirname)
+          create_flow_dir(options)
+          write_rack_config(dirname) if options[:eb]
+          write_worker_config(options, dirname)
+          write_gemfile(dirname)
+        end
+
+        # Creates the toplevel application directory
+        def self.create_app_dir(dirname)
+          FileUtils.mkdir_p(dirname)
+        end
+
+        # Creates the flow/ directory structure for the user applicaiton and
+        # copies the activity and workflow files if a location is provided.
+        def self.create_flow_dir(options)
+          dirname = "#{options[:path]}/#{options[:name]}/flow"
+          FileUtils.mkdir_p(dirname)
+
+          str = require_files(options[:act], dirname)
+          # Default to a helpful string if no activity files are given
+          str = str.empty? ? "# You can require your activity files here." : str
+          # Write the generated string to the activities.rb file
+          write_to_file("#{dirname}/activities.rb", str)
+
+          str = require_files(options[:wf], dirname)
+          # Default to a helpful string if no workflow files are given
+          str = str.empty? ? "# You can require your workflow files here." : str
+          # Write the generated string to the workflows.rb file
+          write_to_file("#{dirname}/workflows.rb", str)
+
+        end
+
+        # Helper method to copy files recursively into a directory and generate
+        # a require string
+        # @api private
+        def self.require_files(src, dest)
+          str = ""
+          return str if src.nil?
+          # If a valid location is passed in, copy the file(s) into the flow/
+          # directory.
+          FileUtils.cp_r(src, dest)
+          if File.directory?(src)
+            # If the given path is a directory, recursively get the relative
+            # paths of all the files in the directory and generated the
+            # require_relative string.
+            Dir.glob("#{dest}/#{File.basename src}/**/*.rb").each do |x|
+              a = Pathname.new(x)
+              rel_path = a.relative_path_from(Pathname.new("#{dest}"))
+              rel_path = rel_path.to_s.split(".rb").first
+              str = "require_relative '#{rel_path}'\n#{str}"
+            end
+          else
+            # If the given path is a file, just require the basename of the file
+            str = "require_relative '#{File.basename src}'"
+          end
+          str
+        end
+
+        # Creates a rack config for the beanstalk deployment
+        # @api private
+        def self.write_rack_config(dirname)
+          write_to_file(
+            "#{dirname}/config.ru",
+            "# Rack config file.\n\n"\
+            "system(\"pkill -9 ruby\")\n\nsystem(\"aws-flow-ruby -f worker.json&\")\n\n"\
+            "run Proc.new { |env| [200, {'Content-Type' => 'text/html'}, ['Done!']] }"
+          )
+        end
+
+        # Creates the json worker config for the user application
+        def self.write_worker_config(options, dirname)
+          spec = {}
+          spec.merge!(generate_worker_spec("activity", :activities, options))
+          spec.merge!(generate_worker_spec("workflow", :workflows, options))
+
+          spec = spec.empty? ? "" : JSON.pretty_generate(spec)
+          write_to_file("#{dirname}/worker.json", spec)
+        end
+
+        # Helper method to generate a worker config
+        # @api private
+        def self.generate_worker_spec(name, key, options)
+          spec = {}
+          if options[key]
+            spec = {
+              "#{name}_workers" => [
+                {
+                  "#{name}_classes" => options[key],
+                  "number_of_workers" => 10
+                }
+              ]
+            }
+          end
+          return spec
+        end
+
+        # Creates the Gemfile for the user application
+        def self.write_gemfile(dirname)
+          write_to_file(
+            "#{dirname}/Gemfile",
+            "source 'http://www.rubygems.org'\n\ngem 'aws-flow', '~> 2.4.0'"
+          )
+        end
+
+        # Helper method to write a string to a file
+        def self.write_to_file(name, string)
+          File.open(name, 'w') { |f| f.write(string) }
+        end
+
+        # Validates various options
+        def self.check_options(options)
+          raise ArgumentError, "You must specify a name for your application" unless options[:name]
+          options[:region] ||= ENV['AWS_REGION'].downcase if ENV['AWS_REGION']
+          options[:path] ||= "."
+
+          # We need the region for the 1-Click URL
+          if options[:eb]
+            raise ArgumentError, "You must specify an AWS region argument or export a "\
+              "variable AWS_REGION in your shell" unless options[:region]
+          end
+
+          # If a location for activity classes is provided, then ensure that it
+          # exists
+          if options[:act] && !File.exist?(options[:act])
+            raise ArgumentError, "Please provide a valid location where your activity "\
+              "classes are located."
+          end
+
+          # If a location for workflow classes is provided, then ensure that it
+          # exists
+          if options[:wf] && !File.exist?(options[:wf])
+            raise ArgumentError, "Please provide a valid location where your workflow "\
+              "classes are located."
+          end
+
+        end
+
+        # Main method that will be called by the FlowUtils utility.
+        def self.generate(options)
+          return unless options[:deploy][:enabled]
+          opts = options[:deploy]
+          check_options(opts)
+          puts "---"
+          generate_files(opts)
+          generate_url(opts) if opts[:eb]
+          puts "---"
+        end
+
+      end
+
+      # Invoked from the shell.
+      #
+      # @api private
+      def self.main
+        FlowUtils.utils.each { |x| x.generate(parse_command_line) }
+      end
+
+    end
+  end
+end
+
+if __FILE__ == $0
+  AWS::Flow::Utils.main
+end

data/spec/aws/decider/integration/activity_spec.rb

@@ -2,9 +2,16 @@ require_relative 'setup'
 
 describe Activities do
   before(:all) do
+    @bucket = ENV['AWS_SWF_BUCKET_NAME']
+    ENV['AWS_SWF_BUCKET_NAME'] = nil
     @swf, @domain = setup_swf
   end
 
+  after(:all) do
+    ENV['AWS_SWF_BUCKET_NAME'] = @bucket
+  end
+
+
   it "ensures that a real activity will get scheduled" do
     task_list = "activity_task_list"
     class Blah
@@ -341,4 +348,3 @@ describe Activities do
     end
   end
 end
-

data/spec/aws/decider/integration/data_converter_spec.rb

@@ -0,0 +1,39 @@
+require_relative 'setup'
+
+describe S3DataConverter do
+
+  it "tests regular sized input" do
+    converter = FlowConstants.defaults[:data_converter]
+    list = {
+      input: "asdf",
+      output: "ddd",
+      test: 123,
+    }
+    s3_link = converter.dump(list)
+    converter.load(s3_link).should == list
+    expect_any_instance_of(AWS::S3).not_to receive(:buckets)
+  end
+
+  it "tests greater than 32k input" do
+    converter = FlowConstants.defaults[:data_converter]
+    list = {
+      input: "asdf",
+      test: "a"*33000,
+    }
+    s3_link = converter.dump(list)
+    converter.load(s3_link).should == list
+  end
+
+  it "tests cache" do
+    converter = FlowConstants.defaults[:data_converter]
+    list = {
+      input: "asdf",
+      test: "a"*33000
+    }
+    s3_link = converter.dump(list)
+    key = YAMLDataConverter.new.load(s3_link)
+
+    converter.cache[key[:s3_filename]].should_not be_nil
+  end
+
+end

data/spec/aws/decider/integration/integration_spec.rb

@@ -31,7 +31,11 @@ class TestHistoryAttributes
 end
 
 describe "RubyFlowDecider" do
+    
   before(:all) do
+    @bucket = ENV['AWS_SWF_BUCKET_NAME']
+    ENV['AWS_SWF_BUCKET_NAME'] = nil
+
     class MyWorkflow
       extend AWS::Flow::Workflows
       version "1"
@@ -51,6 +55,9 @@ describe "RubyFlowDecider" do
     kill_executors
     kill_executors
   end
+  after(:all) do
+    ENV['AWS_SWF_BUCKET_NAME'] = @bucket
+  end
 
   it "runs an empty workflow, making sure base configuration stuff is correct" do
     target_workflow = @domain.workflow_types.page(:per_page => 1000).select { |x| x.name == "blank_workflow_test"}
@@ -240,7 +247,7 @@ describe "RubyFlowDecider" do
       history_events.last.should == "WorkflowExecutionFailed"
       workflow_execution.events.to_a.last.attributes.reason.should include("[TRUNCATED]")
       details = workflow_execution.events.to_a.last.attributes.details
-      exception = FlowConstants.default_data_converter.load(details)
+      exception = FlowConstants.data_converter.load(details)
       exception.class.should == AWS::Flow::ActivityTaskFailedException
     end
 
@@ -295,7 +302,7 @@ describe "RubyFlowDecider" do
       last_event = workflow_execution.events.to_a.last
       last_event.event_type.should == "WorkflowExecutionFailed"
       details = workflow_execution.events.to_a.last.attributes.details
-      exception = FlowConstants.default_data_converter.load(details)
+      exception = FlowConstants.data_converter.load(details)
       exception.class.should == RuntimeError
       exception.backtrace.first.should include ("[TRUNCATED]")
     end
@@ -314,7 +321,7 @@ describe "RubyFlowDecider" do
       last_event.event_type.should == "WorkflowExecutionFailed"
       workflow_execution.events.to_a.last.attributes.reason.should include("[TRUNCATED]")
       details = workflow_execution.events.to_a.last.attributes.details
-      exception = FlowConstants.default_data_converter.load(details)
+      exception = FlowConstants.data_converter.load(details)
       exception.class.should == RuntimeError
     end
 
@@ -424,7 +431,7 @@ describe "RubyFlowDecider" do
       last_event.event_type.should == "WorkflowExecutionFailed"
       workflow_execution.events.to_a.last.attributes.reason.should include("[TRUNCATED]")
       details = workflow_execution.events.to_a.last.attributes.details
-      exception = FlowConstants.default_data_converter.load(details)
+      exception = FlowConstants.data_converter.load(details)
       exception.class.should == AWS::Flow::ChildWorkflowFailedException
       exception.cause.class.should == RuntimeError
     end
@@ -469,7 +476,7 @@ describe "RubyFlowDecider" do
       last_event.event_type.should == "WorkflowExecutionFailed"
       workflow_execution.events.to_a.last.attributes.reason.should include("[TRUNCATED]")
       details = workflow_execution.events.to_a.last.attributes.details
-      exception = FlowConstants.default_data_converter.load(details)
+      exception = FlowConstants.data_converter.load(details)
       exception.class.should == AWS::Flow::ChildWorkflowFailedException
       exception.cause.class.should == AWS::Flow::ChildWorkflowFailedException
     end

data/spec/aws/decider/integration/options_spec.rb

@@ -52,6 +52,15 @@ describe "task_priority" do
       worker.register
       activity_worker.register
 
+      type = @domain.workflow_types.to_a.find { |x| x.name == "WorkflowForTaskPriority.start" }
+      type.default_task_priority.should == 10
+
+      type = @domain.activity_types.to_a.find { |x| x.name == "ActivityForTaskPriority.run_1" }
+      type.default_task_priority.should == 20
+
+      type = @domain.activity_types.to_a.find { |x| x.name == "ActivityForTaskPriority.run_2" }
+      type.default_task_priority.should == 0
+
       client = AWS::Flow::workflow_client(@domain.client, @domain) { {from_class: "WorkflowForTaskPriority"} }
       execution = client.start_execution
 
@@ -63,10 +72,10 @@ describe "task_priority" do
       wait_for_execution(execution)
 
       event = execution.events.select { |x| x.event_type =~ /ActivityTaskScheduled/ }
-      event.first.attributes[:task_priority].should == "20"
-      event.last.attributes[:task_priority].should == "0"
+      event.first.attributes[:task_priority].should == 20
+      event.last.attributes[:task_priority].should == 0
       events = execution.events.select { |x| x.event_type =~ /DecisionTaskScheduled/ }
-      events.first.attributes[:taskPriority].should == "10"
+      events.first.attributes[:taskPriority].should == 10
     end
 
     it "ensures that overriden values of task priority are assigned when workflow is started and when activity is scheduled" do
@@ -80,6 +89,8 @@ describe "task_priority" do
       activity_worker = ActivityWorker.new(@domain.client, @domain, task_list, ActivityForTaskPriority)
       worker.register
       activity_worker.register
+      type = @domain.workflow_types.to_a.find { |x| x.name == "WorkflowForTaskPriority.start" }
+      type.default_task_priority.should == 10
 
       client = AWS::Flow::workflow_client(@domain.client, @domain) { {from_class: "WorkflowForTaskPriority"} }
       execution = client.start_execution { { task_priority: "100" } }
@@ -91,9 +102,9 @@ describe "task_priority" do
       wait_for_execution(execution)
 
       event = execution.events.select { |x| x.event_type =~ /ActivityTaskScheduled/ }
-      event.first.attributes[:task_priority].should == "200"
+      event.first.attributes[:task_priority].should == 200
       events = execution.events.select { |x| x.event_type =~ /DecisionTaskScheduled/ }
-      events.first.attributes[:taskPriority].should == "100"
+      events.first.attributes[:taskPriority].should == 100
     end
 
   end
@@ -110,7 +121,7 @@ describe "task_priority" do
           }
         end
         def entry_point
-          continue_as_new
+          continue_as_new { { task_list: "continue_as_new_foo" } }
         end
       end
 
@@ -128,7 +139,7 @@ describe "task_priority" do
       execution.events.map(&:event_type).last.should == "WorkflowExecutionContinuedAsNew"
       execution.status.should == :continued_as_new
       events = execution.events.select { |x| x.event_type == "WorkflowExecutionContinuedAsNew" }
-      events.first.attributes.task_priority.should == "50"
+      events.first.attributes.task_priority.should == 50
     end
 
 
@@ -150,7 +161,7 @@ describe "task_priority" do
       execution.events.map(&:event_type).last.should == "WorkflowExecutionContinuedAsNew"
       execution.status.should == :continued_as_new
       events = execution.events.select { |x| x.event_type == "WorkflowExecutionContinuedAsNew" }
-      events.first.attributes.task_priority.should == "100"
+      events.first.attributes.task_priority.should == 100
     end
   end
 
@@ -190,18 +201,21 @@ describe "task_priority" do
       parent_client = AWS::Flow::workflow_client(@domain.client, @domain) { { from_class: "ChildWorkflowsTestParentWorkflow" } }
       @child_worker = WorkflowWorker.new(@domain.client, @domain, "test2", ChildWorkflowsTestChildWorkflow)
       @parent_worker = WorkflowWorker.new(@domain.client, @domain, "test", ChildWorkflowsTestParentWorkflow)
+      @child_worker.register
+      @parent_worker.register
 
       @forking_executor = ForkingExecutor.new(:max_workers => 3)
       @forking_executor.execute { @parent_worker.start }
       @forking_executor.execute { @child_worker.start }
       @forking_executor.execute { @child_worker.start }
-      sleep 2
 
       workflow_execution = parent_client.start_execution
       wait_for_execution(workflow_execution)
 
       events = workflow_execution.events.map(&:event_type)
       events.should include("ChildWorkflowExecutionStarted", "ChildWorkflowExecutionCompleted", "WorkflowExecutionCompleted")
+      events = workflow_execution.events.select { |x| x.event_type =~ /ChildWorkflowExecutionStarted/ }
+      events[0].attributes.workflow_execution.task_priority.should == 100
       @forking_executor.shutdown 0
     end