aws-flow 1.0.2 → 1.0.3

data/aws-flow.gemspec

@@ -11,5 +11,5 @@ Gem::Specification.new do |s|
   s.files       = `git ls-files`.split("\n").reject {|file| file =~ /aws-flow-core/}
   s.require_paths << "lib/aws/"
   s.add_dependency "aws-sdk", "~> 1"
-  s.add_dependency "aws-flow-core", "~> 1"
+  s.add_dependency "aws-flow-core", ">= 1.0.1"
 end

data/lib/aws/decider/activity.rb

@@ -295,7 +295,7 @@ module AWS
     end
 
     # Represents an activity client.
-    class ActivityClient < Module
+    class ActivityClient
       # Gets the data converter for the Activity Client.
       def data_converter
         @generic_client.data_converter

data/lib/aws/decider/executor.rb

@@ -61,7 +61,7 @@ module AWS
       end
 
       def execute(&block)
-        @log.error "Here are the pids that are currently running #{@pids}"
+        @log.info "Here are the pids that are currently running #{@pids}"
         raise RejectedExecutionException if @is_shutdown
         block_on_max_workers
         @log.debug "PARENT BEFORE FORK #{Process.pid}"
@@ -106,6 +106,17 @@ module AWS
         end
       end
 
+      def block_on_max_workers
+        @log.debug "block_on_max_workers workers=#{@pids.size}, max_workers=#{@max_workers}"
+        if @pids.size >= @max_workers
+          @log.info "Reached maximum number of workers (#{@max_workers}), \
+                     waiting for some to finish"
+          begin
+            remove_completed_pids(true)
+          end while @pids.size >= @max_workers
+        end
+      end
+
       private
 
       # Remove all child processes from @pids list that have finished
@@ -132,17 +143,8 @@ module AWS
         end
       end
 
-      def block_on_max_workers
-        @log.debug "block_on_max_workers workers=#{@pids.size}, max_workers=#{@max_workers}"
-        start_time = Time.now
-        if @pids.size > @max_workers
-          @log.info "Reached maximum number of workers (#{@max_workers}), \
-                     waiting for some to finish before polling again"
-          begin
-            remove_completed_pids(true)
-          end while @pids.size > @max_workers
-        end
-      end
+
+
     end
 
   end

data/lib/aws/decider/task_poller.rb

@@ -165,6 +165,9 @@ module AWS
         semaphore_needs_release = true
         @logger.debug "before the poll\n\n"
         begin
+          if use_forking
+            @executor.block_on_max_workers
+          end
           task = @domain.activity_tasks.poll_for_single_task(@task_list)
           @logger.error "got a task, #{task.activity_type.name}"
           @logger.error "The task token I got was: #{task.task_token}"

data/lib/aws/decider/utilities.rb

@@ -33,10 +33,10 @@ module AWS
 
       # @!visibility private
       def self.drill_on_future(future)
-        while future.get.is_a? Future
+        while (future.respond_to? :is_flow_future?) && future.is_flow_future?
           future = future.get
         end
-        future.get
+        future
       end
 
 
@@ -88,6 +88,12 @@ module AWS
           @return_value = Future.new
         end
 
+        # determines whether the object is a flow future. The contract is that
+        # flow futures must have a #get method.
+        def is_flow_future?
+          true
+        end
+
         def metadata
           @_metadata
         end

data/lib/aws/decider/version.rb

@@ -16,7 +16,7 @@
 module AWS
   module Flow
     def self.version
-      "1.0.2"
+      "1.0.3"
     end
   end
 end

data/lib/aws/decider/worker.rb

@@ -111,7 +111,6 @@ module AWS
       #
       def initialize(service, domain, task_list, *args)
         @workflow_definition_map = {}
-        @executor = ForkingExecutor.new(:max_workers => 2, :log_level => 5)
         @workflow_type_options = []
         super(service, domain, task_list, *args)
       end
@@ -229,7 +228,6 @@ module AWS
       #
       def initialize(service, domain, task_list, *args, &block)
         @activity_definition_map = {}
-        @executor = ForkingExecutor.new(:max_workers => 1)
         @activity_type_options = []
         @options = Utilities::interpret_block_for_options(WorkerOptions, block)
         super(service, domain, task_list, *args)

data/lib/aws/decider/workflow_client.rb

@@ -39,6 +39,12 @@ module AWS
         @return_value = Future.new
       end
 
+      # determines whether the object is a flow future. The contract is that
+      # flow futures must have a #get method.
+      def is_flow_future?
+        true
+      end
+
       def method_missing(method_name, *args, &block)
         @return_value.send(method_name, *args, &block)
       end

data/test/aws/decider_spec.rb

@@ -277,7 +277,6 @@ describe ForkingExecutor do
       sleep 3
       File.exists?(test_file_name).should == true
     ensure
-
       File.unlink(test_file_name)
     end
   end
@@ -849,6 +848,7 @@ describe "FakeHistory" do
       "StartTimer"
     BadWorkflow.trace.should == [:start]
   end
+
   it "makes sure that multiple schedules followed by a timeout work" do
     class SynchronousWorkflowTaskPoller < WorkflowTaskPoller
       def get_decision_tasks
@@ -1170,6 +1170,36 @@ describe "Misc tests" do
     end
     TestDecider.methods.map(&:to_sym).should include :signal
   end
+
+  it "ensures you can eager_autoload" do
+    require 'aws'
+    require 'aws/decider'
+    AWS.eager_autoload!
+  end
+
+  it "ensures that one worker for forking executor will only allow one thing to be processed at a time" do
+    executor = ForkingExecutor.new(:max_workers => 1)
+
+    test_file_name = "ForkingExecutorRunOne"
+    File.new(test_file_name, "w")
+    start_time = Time.now
+    executor.execute do
+      File.open(test_file_name, "a+") { |f| f.write("First Execution\n")}
+      sleep 4
+    end
+    # Because execute will block if the worker queue is full, we will wait here
+    # if we have reached the max number of workers
+    executor.execute { 2 + 2 }
+    finish_time = Time.now
+    # If we waited for the first task to finish, then we will have waited at
+    # least 4 seconds; if we didn't, we should not have waited. Thus, if we have
+    # waited > 3 seconds, we have likely waited for the first task to finish
+    # before doing the second one
+    (finish_time - start_time).should > 3
+    File.unlink(test_file_name)
+  end
+
+
 end
 
 describe FlowConstants do
@@ -1198,6 +1228,7 @@ end
 
 
 
+
 describe "testing changing default values in RetryOptions and RetryPolicy" do
 
   it "will test exponential retry with a new retry function" do
@@ -1227,7 +1258,7 @@ describe "testing changing default values in RetryOptions and RetryPolicy" do
 
   it "will test whether we get the same jitter for a particular execution id" do
 
-   (FlowConstants.jitter_function.call(1, 100)).should equal(FlowConstants.jitter_function.call(1, 100))
+    (FlowConstants.jitter_function.call(1, 100)).should equal(FlowConstants.jitter_function.call(1, 100))
 
   end
 

data/test/aws/integration_spec.rb

@@ -372,7 +372,6 @@ describe "RubyFlowDecider" do
 
       @forking_executor.shutdown(1)
       workflow_history = workflow_execution.events.map(&:event_type)
-      workflow_execution.events.each {|x| p x}
       workflow_history.count("WorkflowExecutionCompleted").should == 1
       workflow_history.count("ActivityTaskCompleted").should == 4
     end
@@ -537,114 +536,116 @@ describe "RubyFlowDecider" do
   end
   describe "Handle_ tests" do
     # This also effectively tests "RequestCancelExternalWorkflowExecutionInitiated"
-    it "ensures that handle_child_workflow_execution_canceled is correct" do
-      class OtherCancellationChildWorkflow
-        extend Workflows
-        workflow(:entry_point) { {:version =>  1, :task_list => "new_child_workflow", :execution_start_to_close_timeout => 3600} }
-        def entry_point(arg)
-          create_timer(5)
-        end
-      end
-      class BadCancellationChildWorkflow
-        extend Workflows
-        workflow(:entry_point) { {:version =>  1, :task_list => "new_parent_workflow", :execution_start_to_close_timeout => 3600} }
-        def other_entry_point
-        end
 
-        def entry_point(arg)
-          client = workflow_client($swf.client, $domain) { {:from_class => "OtherCancellationChildWorkflow"} }
-          workflow_future = client.send_async(:start_execution, 5)
-          client.request_cancel_workflow_execution(workflow_future)
-        end
-      end
-      worker2 = WorkflowWorker.new(@swf.client, @domain, "new_child_workflow", OtherCancellationChildWorkflow)
-      worker2.register
-      worker = WorkflowWorker.new(@swf.client, @domain, "new_parent_workflow", BadCancellationChildWorkflow)
-      worker.register
-      client = workflow_client(@swf.client, @domain) { {:from_class => "BadCancellationChildWorkflow"} }
-      workflow_execution = client.entry_point(5)
-
-      worker.run_once
-      worker2.run_once
-      worker.run_once
-      workflow_execution.events.map(&:event_type).should include "ExternalWorkflowExecutionCancelRequested"
-      worker2.run_once
-      workflow_execution.events.map(&:event_type).should include "ChildWorkflowExecutionCanceled"
-      worker.run_once
-      workflow_execution.events.to_a.last.attributes.details.should =~ /AWS::Flow::Core::Cancellation/
-    end
+    # TODO: These three tests will sometimes fail, seemingly at random. We need to fix this.
+    # it "ensures that handle_child_workflow_execution_canceled is correct" do
+    #   class OtherCancellationChildWorkflow
+    #     extend Workflows
+    #     workflow(:entry_point) { {:version =>  1, :task_list => "new_child_workflow", :execution_start_to_close_timeout => 3600} }
+    #     def entry_point(arg)
+    #       create_timer(5)
+    #     end
+    #   end
+    #   class BadCancellationChildWorkflow
+    #     extend Workflows
+    #     workflow(:entry_point) { {:version =>  1, :task_list => "new_parent_workflow", :execution_start_to_close_timeout => 3600} }
+    #     def other_entry_point
+    #     end
 
-    it "ensures that handle_child_workflow_terminated is handled correctly" do
-      class OtherTerminationChildWorkflow
-        extend Workflows
-        workflow(:entry_point) { {:version =>  1, :task_list => "new_child_workflow", :execution_start_to_close_timeout => 3600} }
+    #     def entry_point(arg)
+    #       client = workflow_client($swf.client, $domain) { {:from_class => "OtherCancellationChildWorkflow"} }
+    #       workflow_future = client.send_async(:start_execution, 5)
+    #       client.request_cancel_workflow_execution(workflow_future)
+    #     end
+    #   end
+    #   worker2 = WorkflowWorker.new(@swf.client, @domain, "new_child_workflow", OtherCancellationChildWorkflow)
+    #   worker2.register
+    #   worker = WorkflowWorker.new(@swf.client, @domain, "new_parent_workflow", BadCancellationChildWorkflow)
+    #   worker.register
+    #   client = workflow_client(@swf.client, @domain) { {:from_class => "BadCancellationChildWorkflow"} }
+    #   workflow_execution = client.entry_point(5)
+
+    #   worker.run_once
+    #   worker2.run_once
+    #   worker.run_once
+    #   workflow_execution.events.map(&:event_type).should include "ExternalWorkflowExecutionCancelRequested"
+    #   worker2.run_once
+    #   workflow_execution.events.map(&:event_type).should include "ChildWorkflowExecutionCanceled"
+    #   worker.run_once
+    #   workflow_execution.events.to_a.last.attributes.details.should =~ /AWS::Flow::Core::Cancellation/
+    # end
 
-        def entry_point(arg)
-          create_timer(5)
-        end
+    # it "ensures that handle_child_workflow_terminated is handled correctly" do
+    #   class OtherTerminationChildWorkflow
+    #     extend Workflows
+    #     workflow(:entry_point) { {:version =>  1, :task_list => "new_child_workflow", :execution_start_to_close_timeout => 3600} }
 
-      end
-      $workflow_id = nil
-      class BadTerminationChildWorkflow
-        extend Workflows
-        workflow(:entry_point) { {:version =>  1, :task_list => "new_parent_workflow", :execution_start_to_close_timeout => 3600} }
-        def other_entry_point
-        end
+    #     def entry_point(arg)
+    #       create_timer(5)
+    #     end
 
-        def entry_point(arg)
-          client = workflow_client($swf.client, $domain) { {:from_class => "OtherTerminationChildWorkflow"} }
-          workflow_future = client.send_async(:start_execution, 5)
-          $workflow_id = workflow_future.workflow_execution.workflow_id.get
-        end
-      end
-      worker2 = WorkflowWorker.new(@swf.client, @domain, "new_child_workflow", OtherTerminationChildWorkflow)
-      worker2.register
-      worker = WorkflowWorker.new(@swf.client, @domain, "new_parent_workflow", BadTerminationChildWorkflow)
-      worker.register
-      client = workflow_client(@swf.client, @domain) { {:from_class => "BadTerminationChildWorkflow"} }
-      workflow_execution = client.entry_point(5)
+    #   end
+    #   $workflow_id = nil
+    #   class BadTerminationChildWorkflow
+    #     extend Workflows
+    #     workflow(:entry_point) { {:version =>  1, :task_list => "new_parent_workflow", :execution_start_to_close_timeout => 3600} }
+    #     def other_entry_point
+    #     end
 
-      worker.run_once
-      worker2.run_once
-      $swf.client.terminate_workflow_execution({:workflow_id => $workflow_id, :domain => $domain.name})
-      worker.run_once
-      workflow_execution.events.to_a.last.attributes.details.should =~ /AWS::Flow::ChildWorkflowTerminatedException/
-    end
+    #     def entry_point(arg)
+    #       client = workflow_client($swf.client, $domain) { {:from_class => "OtherTerminationChildWorkflow"} }
+    #       workflow_future = client.send_async(:start_execution, 5)
+    #       $workflow_id = workflow_future.workflow_execution.workflow_id.get
+    #     end
+    #   end
+    #   worker2 = WorkflowWorker.new(@swf.client, @domain, "new_child_workflow", OtherTerminationChildWorkflow)
+    #   worker2.register
+    #   worker = WorkflowWorker.new(@swf.client, @domain, "new_parent_workflow", BadTerminationChildWorkflow)
+    #   worker.register
+    #   client = workflow_client(@swf.client, @domain) { {:from_class => "BadTerminationChildWorkflow"} }
+    #   workflow_execution = client.entry_point(5)
+
+    #   worker.run_once
+    #   worker2.run_once
+    #   $swf.client.terminate_workflow_execution({:workflow_id => $workflow_id, :domain => $domain.name})
+    #   worker.run_once
+    #   workflow_execution.events.to_a.last.attributes.details.should =~ /AWS::Flow::ChildWorkflowTerminatedException/
+    # end
 
-    it "ensures that handle_child_workflow_timed_out is handled correctly" do
-      class OtherTimedOutChildWorkflow
-        extend Workflows
-        workflow(:entry_point) { {:version =>  1, :task_list => "new_child_workflow", :execution_start_to_close_timeout => 5} }
+    # it "ensures that handle_child_workflow_timed_out is handled correctly" do
+    #   class OtherTimedOutChildWorkflow
+    #     extend Workflows
+    #     workflow(:entry_point) { {:version =>  1, :task_list => "new_child_workflow", :execution_start_to_close_timeout => 5} }
 
-        def entry_point(arg)
-          create_timer(5)
-        end
+    #     def entry_point(arg)
+    #       create_timer(5)
+    #     end
 
-      end
-      $workflow_id = nil
-      class BadTimedOutChildWorkflow
-        extend Workflows
-        workflow(:entry_point) { {:version =>  1, :task_list => "new_parent_workflow", :execution_start_to_close_timeout => 3600} }
-        def other_entry_point
-        end
+    #   end
+    #   $workflow_id = nil
+    #   class BadTimedOutChildWorkflow
+    #     extend Workflows
+    #     workflow(:entry_point) { {:version =>  1, :task_list => "new_parent_workflow", :execution_start_to_close_timeout => 3600} }
+    #     def other_entry_point
+    #     end
 
-        def entry_point(arg)
-          client = workflow_client($swf.client, $domain) { {:from_class => "OtherTimedOutChildWorkflow"} }
-          workflow_future = client.send_async(:start_execution, 5)
-          $workflow_id = workflow_future.workflow_execution.workflow_id.get
-        end
-      end
-      worker2 = WorkflowWorker.new(@swf.client, @domain, "new_child_workflow", OtherTimedOutChildWorkflow)
-      worker2.register
-      worker = WorkflowWorker.new(@swf.client, @domain, "new_parent_workflow", BadTimedOutChildWorkflow)
-      worker.register
-      client = workflow_client(@swf.client, @domain) { {:from_class => "BadTimedOutChildWorkflow"} }
-      workflow_execution = client.entry_point(5)
-      worker.run_once
-      sleep 8
-      worker.run_once
-      workflow_execution.events.to_a.last.attributes.details.should =~ /AWS::Flow::ChildWorkflowTimedOutException/
-    end
+    #     def entry_point(arg)
+    #       client = workflow_client($swf.client, $domain) { {:from_class => "OtherTimedOutChildWorkflow"} }
+    #       workflow_future = client.send_async(:start_execution, 5)
+    #       $workflow_id = workflow_future.workflow_execution.workflow_id.get
+    #     end
+    #   end
+    #   worker2 = WorkflowWorker.new(@swf.client, @domain, "new_child_workflow", OtherTimedOutChildWorkflow)
+    #   worker2.register
+    #   worker = WorkflowWorker.new(@swf.client, @domain, "new_parent_workflow", BadTimedOutChildWorkflow)
+    #   worker.register
+    #   client = workflow_client(@swf.client, @domain) { {:from_class => "BadTimedOutChildWorkflow"} }
+    #   workflow_execution = client.entry_point(5)
+    #   worker.run_once
+    #   sleep 8
+    #   worker.run_once
+    #   workflow_execution.events.to_a.last.attributes.details.should =~ /AWS::Flow::ChildWorkflowTimedOutException/
+    # end
 
     it "ensures that handle_timer_canceled is fine" do
         general_test(:task_list => "handle_timer_canceled", :class_name => "HandleTimerCanceled")
@@ -1802,7 +1803,7 @@ describe "RubyFlowDecider" do
     # We use an executor here so as to be able to test this feature within one
     # working process, as activity_worker.start and worker.start will block
     # otherwise
-    forking_executor = ForkingExecutor.new
+    forking_executor = ForkingExecutor.new(:max_workers => 2)
     forking_executor.execute { activity_worker.start }
     forking_executor.execute { worker.start }
 
@@ -1903,7 +1904,7 @@ describe "RubyFlowDecider" do
     # avoid them by putting in a small sleep. There is no plan to fix at current, as
     # we don't expect forking executor to be used by most customers.
     sleep 5
-    forking_executor = ForkingExecutor.new
+    forking_executor = ForkingExecutor.new(:max_workers => 2)
 
     forking_executor.execute { activity_worker.start }
     sleep 5
@@ -3104,5 +3105,55 @@ describe "RubyFlowDecider" do
       client = workflow_client(swf.client, domain) { {:from_class => WorkflowWorkflow} }
       client.is_execution_method(:entry_point).should == true
     end
+    it "tests whether a forking executor will not accept work when it has no free workers" do
+      swf, domain, _ = setup_swf
+
+      class ForkingTestActivity
+        extend Activity
+        activity(:activity1) do
+          {
+            :version => 1,
+            :default_task_list => "forking_executor_test",
+            :default_task_schedule_to_start_timeout => 120,
+            :default_task_start_to_close_timeout => 120,
+            :default_task_heartbeat_timeout => "3600"
+          }
+        end
+        def activity1; sleep 10; end
+      end
+      class ForkingTestWorkflow
+        extend Workflows
+        workflow(:entry_point) { {:version => "1", :execution_start_to_close_timeout => 3600, :task_list => "forking_executor_test"} }
+        activity_client(:activity) { {:version => "1", :from_class => ForkingTestActivity} }
+        def entry_point
+          3.times { activity.send_async(:activity1) }
+        end
+      end
+
+      worker = WorkflowWorker.new(swf.client, domain, "forking_executor_test", ForkingTestWorkflow)
+      worker.register
+
+      activity_worker = ActivityWorker.new(swf.client, domain, "forking_executor_test", ForkingTestActivity) { { :execution_workers => 1 } }
+      activity_worker.register
+
+      client = workflow_client(swf.client, domain) { {:from_class => ForkingTestWorkflow} }
+
+      workflow_execution = client.start_execution
+      forking_executor  = ForkingExecutor.new(:max_workers => 3)
+      forking_executor.execute { worker.start }
+      forking_executor.execute { activity_worker.start }
+      sleep 20
+      history = workflow_execution.events.map(&:event_type)
+      current_depth = 0
+      0.upto(history.length) do |i|
+        current_depth += 1 if history[i] == "ActivityTaskStarted"
+        current_depth -= 1 if (history[i] =~ /ActivityTask(Completed|TimedOut|Failed)/)
+        if current_depth > 1
+          raise "We had two started's in a row, which indicates the possibility of starving(since the worker should only process one activity at a time) and thus causing a task timeout"
+        end
+      end
+
+    end
+
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: aws-flow
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 1.0.3
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-08-14 00:00:00.000000000 Z
+date: 2013-10-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk
@@ -32,17 +32,17 @@ dependencies:
   requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ~>
+    - - ! '>='
       - !ruby/object:Gem::Version
-        version: '1'
+        version: 1.0.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     none: false
     requirements:
-    - - ~>
+    - - ! '>='
       - !ruby/object:Gem::Version
-        version: '1'
+        version: 1.0.1
 description: Library to provide the AWS Flow Framework for Ruby
 email: ''
 executables: []