aws-flow 1.0.6 → 1.0.7

data/Gemfile

@@ -5,4 +5,5 @@ gemspec
 group :test do
   gem "rspec", "1.3.0"
   gem "rake"
+  gem "factory_girl"
 end

data/lib/aws/decider/activity_definition.rb

@@ -77,7 +77,12 @@ module AWS
         ensure
           @instance._activity_execution_context = nil
         end
-        return @converter.dump result
+        converted_result = @converter.dump(result)
+        # We are going to have to convert this object into a string to submit it, and that's where the 32k limit will be enforced, so it's valid to turn the object to a string and check the size of the result
+        if converted_result.to_s.size > 32768
+          return @converter.dump("The result was too large, so we could not serialize it correctly. You can find the full result in the ActivityTaskPoller logs."), result, true
+        end
+        return converted_result, result, false
       end
 
     end

data/lib/aws/decider/generic_client.rb

@@ -85,10 +85,13 @@ module AWS
             result[:return_on_start] = true
             result
           end
-          # Otherwise, it will expect an options object passed in, and will do things on that object. So make our new Proc do that, and add an option
+          # Otherwise, it will expect an options object passed in, and will do
+          # things on that object. So make our new Proc do that, and add an
+          # option
         else modified_options = Proc.new do |x|
             result = block.call(x)
-            # We need to copy the hash to make sure that we don't mutate it
+            # Same as the above dup, we'll copy to avoid any possible mutation
+            # of inputted objects
             result = result.dup
             result.return_on_start = true
             result

data/lib/aws/decider/task_poller.rb

@@ -72,9 +72,10 @@ module AWS
           end
           @service.respond_decision_task_completed(task_completed_request)
         rescue AWS::SimpleWorkflow::Errors::UnknownResourceFault => e
-          # Log stuff
           @logger.debug "Error in the poller, #{e}"
           @logger.debug "The error class in #{e.class}"
+        rescue Exception => e
+          @logger.debug "Error in the poller, #{e}"
         end
       end
     end
@@ -97,16 +98,18 @@ module AWS
           context = ActivityExecutionContext.new(@service, @domain, task)
           activity_implementation = @activity_definition_map[activity_type]
           raise "This activity worker was told to work on activity type #{activity_type.name}, but this activity worker only knows how to work on #{@activity_definition_map.keys.map(&:name).join' '}" unless activity_implementation
-          output = activity_implementation.execute(task.input, context)
+          output, original_result, too_large = activity_implementation.execute(task.input, context)
           @logger.debug "Responding on task_token #{task.task_token} for task #{task}"
-          if output.length > 32768
-            output = output.slice(0..32767)
-            respond_activity_task_failed_with_retry(task.task_token, "You cannot send a response with a result greater thank 32768. Please reduce the response size. The first part of the output is included in the details field.", output )
+          if too_large
+            @logger.warn "The output of this activity was too large (greater than 2^15), and therefore aws-flow could not return it to SWF. aws-flow is now attempting to mark this activity as failed. For reference, the result was #{original_result}"
+            respond_activity_task_failed_with_retry(task.task_token, "An activity cannot send a response with a result larger than 32768 characters. Please reduce the response size. A truncated prefix output is included in the details field.", output)
           elsif ! activity_implementation.execution_options.manual_completion
             @service.respond_activity_task_completed(:task_token => task.task_token, :result => output)
           end
         rescue ActivityFailureException => e
+          @logger.debug "The activity failed, with original output of #{original_result} and dataconverted result of #{output}. aws-flow will now attempt to fail it."
           respond_activity_task_failed_with_retry(task.task_token, e.message, e.details)
+
         end
         #TODO all the completion stuffs
       end
@@ -136,7 +139,10 @@ module AWS
       end
 
       def process_single_task(task)
-        @service = AWS::SimpleWorkflow.new.client.with_http_handler(AWS::Core::Http::NetHttpHandler.new(AWS.config.to_h))
+        previous_config = @service.config.to_h
+        previous_config.delete(:http_handler)
+        @service = AWS::SimpleWorkflow.new(previous_config).client
+        @service = @service.with_http_handler(AWS::Core::Http::NetHttpHandler.new(previous_config))
         begin
           begin
             execute(task)

data/lib/aws/decider/utilities.rb

@@ -39,7 +39,6 @@ module AWS
         future
       end
 
-
       # @!visibility private
       def self.merge_all_options(*args)
         args.compact!

data/lib/aws/decider/version.rb

@@ -16,7 +16,7 @@
 module AWS
   module Flow
     def self.version
-      "1.0.6"
+      "1.0.7"
     end
   end
 end

data/lib/aws/decider/worker.rb

@@ -42,6 +42,18 @@ module AWS
         @task_list = task_list_to_poll
         @options = Utilities::interpret_block_for_options(WorkerOptions, block)
         args.each { |klass_or_instance| add_implementation(klass_or_instance) } if args
+        @shutting_down = false
+        %w{ TERM INT }.each do |signal|
+          Signal.trap(signal) do
+            if @shutting_down
+              @executor.shutdown(0) if @executor
+              Kernel.exit! 1
+            else
+              @shutting_down = true
+              @shutdown_first_time_function.call if @shutdown_first_time_function
+            end
+          end
+        end
       end
 
       # @!visibility private
@@ -113,6 +125,8 @@ module AWS
         @workflow_definition_map = {}
         @workflow_type_options = []
         super(service, domain, task_list, *args)
+
+
       end
 
       def set_workflow_implementation_types(workflow_implementation_types)
@@ -196,6 +210,7 @@ module AWS
       def run_once(should_register = false, poller = nil)
         register if should_register
         poller = WorkflowTaskPoller.new(@service, @domain, DecisionTaskHandler.new(@workflow_definition_map, @options), @task_list, @options) if poller.nil?
+        Kernel.exit if @shutting_down
         poller.poll_and_process_single_task
       end
     end
@@ -235,22 +250,11 @@ module AWS
         max_workers = @options.execution_workers if @options
         max_workers = 20 if (max_workers.nil? || max_workers.zero?)
         @executor = ForkingExecutor.new(:max_workers => max_workers, :logger => @logger)
-        super(service, domain, task_list, *args)
-
-        @shutting_down = false
-        %w{ TERM INT }.each do |signal|
-          Signal.trap(signal) do
-            if @shutting_down
-              @executor.shutdown 0
-              Kernel.exit! 1
-            else
-              @shutting_down = true
-              @executor.shutdown Float::INFINITY
-              Kernel.exit
-            end
-          end
+        @shutdown_first_time_function = lambda do
+          @executor.shutdown Float::INFINITY
+          Kernel.exit
         end
-
+        super(service, domain, task_list, *args)
       end
 
       # Adds an Activity implementation to this ActivityWorker.

data/test/aws/decider_spec.rb

@@ -60,6 +60,7 @@ class FakeDomain
   def page; FakePage.new(@workflow_type_object); end
   def workflow_executions; FakeWorkflowExecutionCollecton.new; end
   def name; "fake_domain"; end
+
 end
 
 
@@ -193,11 +194,11 @@ describe ActivityDefinition do
   end
   it "ensures that an activity definition can handle one argument" do
     activity_definition = ActivityDefinition.new(MyActivity.new, :test_one_argument, nil , nil, TrivialConverter.new)
-    activity_definition.execute(5, nil).should == 5
+    activity_definition.execute(5, nil).first.should == 5
   end
   it "ensures that you can get the activity context " do
     activity_definition = ActivityDefinition.new(MyActivity.new, :test_getting_context, nil , nil, TrivialConverter.new)
-    (activity_definition.execute(nil, ActivityExecutionContext.new(nil, nil, nil)).is_a? ActivityExecutionContext).should == true
+    (activity_definition.execute(nil, ActivityExecutionContext.new(nil, nil, nil)).first.is_a? ActivityExecutionContext).should == true
   end
   it "ensures that the activity context gets unset after the execute" do
     activity_definition = ActivityDefinition.new(MyActivity.new, :test_getting_context, nil , nil, TrivialConverter.new)
@@ -210,11 +211,12 @@ describe ActivityDefinition do
   end
   it "ensures that an activity definition can handle multiple arguments" do
     activity_definition = ActivityDefinition.new(MyActivity.new, :test_three_arguments, nil , nil, TrivialConverter.new)
-    activity_definition.execute([1,2,3], nil).should == 6
+
+    activity_definition.execute([1,2,3], nil).first.should == 6
   end
   it "ensures that an activity definition can handle no arguments" do
     activity_definition = ActivityDefinition.new(MyActivity.new, :test_no_arguments, nil , nil, TrivialConverter.new)
-    activity_definition.execute(nil, nil).should == :no_arguments
+    activity_definition.execute(nil, nil).first.should == :no_arguments
   end
 end
 
@@ -232,6 +234,7 @@ describe WorkflowDefinitionFactory do
       def multiple_arguments(arg1, arg2, arg3)
         arg3
       end
+
     end
     class WorkflowDefinition
       attr_accessor :decision_helper, :workflow_method, :converter
@@ -272,7 +275,7 @@ describe WorkflowDefinitionFactory do
     end
   end
 end
-p
+
 describe ForkingExecutor do
   it "makes sure that forking executors basic execute works" do
     test_file_name = "ForkingExecutorTestFile"
@@ -1310,7 +1313,6 @@ describe "Misc tests" do
     File.unlink(test_file_name)
   end
 
-
   it "ensures that using send_async doesn't mutate the original hash" do
     class GenericClientTest < GenericClient
       def call_options(*args, &options)
@@ -1370,7 +1372,18 @@ describe FlowConstants do
   end
 
 end
+class TestWorkflow
+  extend Workflows
+  workflow :entry_point do
+    {
+      :execution_start_to_close_timeout => 30, :version => "1"
+    }
+  end
+  def entry_point
 
+  end
+
+end
 class TestActivity
   extend Activity
 
@@ -1399,41 +1412,106 @@ class TestActivityWorker < ActivityWorker
   end
 end
 
+class FakeTaskPoller < WorkflowTaskPoller
+  def get_decision_tasks
+    nil
+  end
+end
+def dumb_fib(n)
+  n < 1 ? 1 : dumb_fib(n - 1) + dumb_fib(n - 2)
+end
+describe WorkflowWorker do
+  it "will test whether WorkflowWorker shuts down cleanly when an interrupt is received" do
+    task_list = "TestWorkflow_tasklist"
+    service = FakeServiceClient.new
+    workflow_type_object = double("workflow_type", :name => "TestWorkflow.entry_point", :start_execution => "" )
+    domain = FakeDomain.new(workflow_type_object)
+    workflow_worker = WorkflowWorker.new(service, domain, task_list)
+    workflow_worker.add_workflow_implementation(TestWorkflow)
+    pid = fork do
+      loop do
+        workflow_worker.run_once(true, FakeTaskPoller.new(service, domain, nil, task_list, nil))
+      end
+    end
+    # Send an interrupt to the child process
+    Process.kill("INT", pid)
+    # Adding a sleep to let things get setup correctly (not ideal but going with
+    # this for now)
+    sleep 5
+    return_pid, status = Process.wait2(pid, Process::WNOHANG)
+    Process.kill("KILL", pid) if return_pid.nil?
+    return_pid.should_not be nil
+    status.success?.should be_true
+  end
+
+  it "will test whether WorkflowWorker dies cleanly when two interrupts are received" do
+    class FakeTaskPoller
+      def poll_and_process_single_task
+        dumb_fib(5000)
+      end
+    end
+    task_list = "TestWorkflow_tasklist"
+    service = FakeServiceClient.new
+    workflow_type_object = double("workflow_type", :name => "TestWorkflow.entry_point", :start_execution => "" )
+    domain = FakeDomain.new(workflow_type_object)
+    workflow_worker = WorkflowWorker.new(service, domain, task_list)
+    workflow_worker.add_workflow_implementation(TestWorkflow)
+    pid = fork do
+      loop do
+        workflow_worker.run_once(true, FakeTaskPoller.new(service, domain, nil, task_list, nil))
+      end
+    end
+    # Send an interrupt to the child process
+    sleep 3
+    2.times { Process.kill("INT", pid); sleep 2 }
+    return_pid, status = Process.wait2(pid, Process::WNOHANG)
+
+    Process.kill("KILL", pid) if return_pid.nil?
+    return_pid.should_not be nil
+    status.success?.should be_false
+  end
+
+end
 describe ActivityWorker do
 
-  # it "will test whether the ActivityWorker shuts down cleanly when an interrupt is received" do
-
-  #   task_list = "TestWorkflow_tasklist"
-  #   service = FakeServiceClient.new
-  #   workflow_type_object = double("workflow_type", :name => "TestWorkflow.entry_point", :start_execution => "" )
-  #   domain = FakeDomain.new(workflow_type_object)
-  #   forking_executor = ForkingExecutor.new
-  #   activity_worker = TestActivityWorker.new(service, domain, task_list, forking_executor) { {:logger => FakeLogger.new} }
-
-  #   activity_worker.add_activities_implementation(TestActivity)
-  #   # Starts the activity worker in a forked process. Also, attaches an at_exit
-  #   # handler to the process. When the process exits, the handler checks whether
-  #   # the executor's internal is_shutdown variable is set correctly or not.
-  #   pid = fork do
-  #     at_exit {
-  #       activity_worker.executor.is_shutdown.should == true
-  #     }
-  #     activity_worker.start true
-  #   end
-  #   # Adding a sleep to let things get setup correctly (not ideal but going with
-  #   # this for now)
-  #   #sleep 1
-  #   # Send an interrupt to the child process
-  #   Process.kill("INT", pid)
-  #   status = Process.waitall
-  #   status[0][1].success?.should be_true
-  # end
+  class FakeDomain
+    def activity_tasks
+      sleep 30
+    end
+  end
+  it "will test whether the ActivityWorker shuts down cleanly when an interrupt is received" do
+
+    task_list = "TestWorkflow_tasklist"
+    service = FakeServiceClient.new
+    workflow_type_object = double("workflow_type", :name => "TestWorkflow.entry_point", :start_execution => "" )
+    domain = FakeDomain.new(workflow_type_object)
+    forking_executor = ForkingExecutor.new
+    activity_worker = TestActivityWorker.new(service, domain, task_list, forking_executor) { {:logger => FakeLogger.new} }
+    activity_worker.add_activities_implementation(TestActivity)
+    # Starts the activity worker in a forked process. Also, attaches an at_exit
+    # handler to the process. When the process exits, the handler checks whether
+    # the executor's internal is_shutdown variable is set correctly or not.
+    pid = fork do
+      at_exit {
+        activity_worker.executor.is_shutdown.should == true
+      }
+      activity_worker.start true
+    end
+    # Send an interrupt to the child process
+    Process.kill("INT", pid)
+    # Adding a sleep to let things get setup correctly (not ideal but going with
+    # this for now)
+    sleep 5
+    return_pid, status = Process.wait2(pid, Process::WNOHANG)
+    Process.kill("KILL", pid) if return_pid.nil?
+    return_pid.should_not be nil
+
+    status.success?.should be_true
+  end
 
   # This method will take a long time to run, allowing us to test our shutdown
   # scenarios
-  def dumb_fib(n)
-    n < 1 ? 1 : dumb_fib(n - 1) + dumb_fib(n - 2)
-  end
+
 
   it "will test whether the ActivityWorker shuts down immediately if two or more interrupts are received" do
     task_list = "TestWorkflow_tasklist"

data/test/aws/factories.rb

@@ -13,10 +13,15 @@
 # permissions and limitations under the License.
 ##
 
+
 require 'aws/flow'
 include AWS::Flow::Core
 
 
+require 'factory_girl'
+require 'aws/flow'
+include AWS::Flow::Core
+
 class FlowFactory
 
   attr_accessor :async_scope
@@ -42,33 +47,9 @@ class FlowFactory
     return @async_scope
   end
 
-end
-
-class WorkflowGenerator
-  class << self
-    def generate_workflow(domain, options = {})
-
-      name = options[:name] || "default_name"
-      version = options[:version] || "1"
-      task_list = options[:task_list] || "default_task_list"
-      child_policy = options[:child_policy] || :request_cancel
-      task_start_to_close = options[:task_start_to_close] || 3600
-      default_execution_timeout = options[:default_execution_timeout] || 24 * 3600
-
-
-      target_workflow = domain.workflow_types.page.select { |x| x.name == name}
-      if target_workflow.length == 0
-        workflow_type = domain.workflow_types.create(name, version,
-                                                     :default_task_list => task_list,
-                                                     :default_child_policy => child_policy,
-                                                     :default_task_start_to_close_timeout => task_start_to_close,
-                                                     :default_execution_start_to_close_timeout => default_execution_timeout)
-      else
-        workflow_type = target_workflow.first
-      end
+  def generate_daemon_task(options = {})
+    scope = generate_scope
+    task = DaemonTask.new
 
-      return workflow_type
-    end
   end
-
 end

data/test/aws/integration_spec.rb

@@ -43,17 +43,23 @@ def setup_swf
   File.open(file_name, 'w+') {|f| f.write(last_run)}
   current_date = Time.now.strftime("%d-%m-%Y")
   config_file = File.open('credentials.cfg') { |f| f.read }
-  config = YAML.load(config_file).first
-  AWS.config(config)
-  swf = AWS::SimpleWorkflow.new
+  if config_file.include? ":Primary"
+    yaml_config = YAML.load(config_file)
+    swf = AWS::SimpleWorkflow.new(yaml_config[:Primary])
+    secondary_swf = AWS::SimpleWorkflow.new(yaml_config[:Secondary])
+  else
+    config = YAML.load(config_file).first
+    AWS.config(config)
+    swf = AWS::SimpleWorkflow.new
+    secondary_swf = nil
+  end
   $RUBYFLOW_DECIDER_DOMAIN = "rubyflow_decider_domain_#{current_date}-#{last_run}"
   begin
     domain = swf.domains.create($RUBYFLOW_DECIDER_DOMAIN, "10")
   rescue AWS::SimpleWorkflow::Errors::DomainAlreadyExistsFault => e
     domain = swf.domains[$RUBYFLOW_DECIDER_DOMAIN]
   end
-
-  return swf, domain, $RUBYFLOW_DECIDER_DOMAIN
+  return swf, domain, $RUBYFLOW_DECIDER_DOMAIN, secondary_swf
 end
 
 
@@ -76,8 +82,8 @@ describe "RubyFlowDecider" do
       version "1"
       # TODO more of the stuff from the proposal
     end
-    @swf, @domain, $RUBYFLOW_DECIDER_DOMAIN = setup_swf
-    $swf, $domain = @swf, @domain
+    @swf, @domain, $RUBYFLOW_DECIDER_DOMAIN, @swf_secondary = setup_swf
+    $swf, $domain, $swf_secondary = @swf, @domain, @swf_secondary
     # If there are any outstanding decision tasks before we start the test, that
     # could really mess things up, and make the tests non-idempotent. So lets
     # clear those out
@@ -530,7 +536,7 @@ describe "RubyFlowDecider" do
         # Make sure we return something that's over 32k. Note this won't
         # necessarily work with all converters, as it's pretty trivially
         # compressible
-        return "a" * 33000
+        return ":" + "a" * 33000
       end
     end
     workflow_execution = @my_workflow_client.start_execution
@@ -539,11 +545,61 @@ describe "RubyFlowDecider" do
     @worker.run_once
     history_events = workflow_execution.events.map(&:event_type)
     # Previously, it would time out, as the failure would include the original
-    # large output that killed the completion call. Thus, we need to check that
-    # we fail the ActivityTask correctly.
+    # large output that killed the completion and failure call. Thus, we need to
+    # check that we fail the ActivityTask.
     history_events.should include "ActivityTaskFailed"
+
+    workflow_execution.events.to_a.last.attributes.details.should_not =~ /Psych/
     history_events.last.should == "WorkflowExecutionFailed"
+  end
+
+  it "ensures that activities can be processed with different configurations" do
+    class TwoConfigActivity
+      extend Activities
+      activity :run_activity1 do
+        {
+          :default_task_heartbeat_timeout => "3600",
+          :default_task_list => "TwoConfigTaskList",
+          :default_task_schedule_to_start_timeout => 120,
+          :default_task_start_to_close_timeout => 120,
+          :version => "1",
+        }
+      end
+      def run_activity1
+      end
+    end
 
+    class TwoConfigWorkflow
+      extend Workflows
+      activity_client(:activity) { { :from_class => TwoConfigActivity }}
+      workflow :entry_point do
+        {
+          :version => 1,
+          :default_execution_start_to_close_timeout => 30,
+          :default_child_policy => "request_cancel",
+          :default_task_list => "TwoConfigTaskList"
+        }
+      end
+      def entry_point
+        activity.run_activity1
+        activity.run_activity1 { {:task_list => "other_config_task_list"} }
+      end
+    end
+    worker = WorkflowWorker.new(@swf.client, @domain, "TwoConfigTaskList", TwoConfigWorkflow)
+    activity_worker = ActivityWorker.new(@swf.client, @domain, "TwoConfigTaskList", TwoConfigActivity) {{ :use_forking => false }}
+    swf = AWS::SimpleWorkflow.new(access_key_id: "AWS_SWF_KEY", secret_access_key: "AWS_SWF_SECRET")
+    activity_worker_different_config = ActivityWorker.new($swf_secondary.client, @domain, "other_config_task_list", TwoConfigActivity) {{ :use_forking => false }}
+    my_workflow_client = workflow_client(@swf.client, @domain) {{:from_class => TwoConfigWorkflow}}
+
+    worker.register
+    activity_worker.register
+    workflow_execution = my_workflow_client.start_execution
+    worker.run_once
+    activity_worker.run_once
+    worker.run_once
+    activity_worker_different_config.run_once
+    worker.run_once
+    workflow_execution.events.map(&:event_type).last == "WorkflowExecutionCompleted"
   end
 
   it "ensures that not filling in details/reason for activity_task_failed is handled correctly" do
@@ -2288,7 +2344,7 @@ describe "RubyFlowDecider" do
         options.task_list = "timeout_test"
       end
       my_workflow_client = my_workflow_factory.get_client
-      num_tests = 30
+      num_tests = 15
       workflow_executions = []
       1.upto(num_tests)  { |i| workflow_executions << my_workflow_client.entry_point }
       forking_executor  = ForkingExecutor.new(:max_workers => 3)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: aws-flow
 version: !ruby/object:Gem::Version
-  version: 1.0.6
+  version: 1.0.7
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-12-13 00:00:00.000000000 Z
+date: 2013-12-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk