aws-flow 1.0.0

data/aws-flow-core/lib/aws/flow/flow_utils.rb

@@ -0,0 +1,50 @@
+##
+# Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License").
+# You may not use this file except in compliance with the License.
+# A copy of the License is located at
+#
+#  http://aws.amazon.com/apache2.0
+#
+# or in the "license" file accompanying this file. This file is distributed
+# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+# express or implied. See the License for the specific language governing
+# permissions and limitations under the License.
+##
+
+# This file simply contains definitions and functions useful to the overall running of flow
+module AWS
+  module Flow
+    module Core
+      class IllegalStateException < Exception; end
+      class CancellationException < Exception
+        attr_accessor :reason, :details
+        def initialize(reason = nil, details = nil)
+          @reason = reason
+          @details = details
+        end
+      end
+
+      def make_backtrace(parent_backtrace)
+        # 1 frame for the function that actually removes the stack traces
+        # 1 frame for the function that calls into the function that removes
+        # frames in AsyncBacktrace
+        # 1 frame for the call into this function
+        # 1 frame for the initialize call of the BRE or External Task
+        # 1 frame for the new call into the BRE or ET
+        # 1 frame for the AsyncScope initialize that the BRE/ET has to be in
+
+        # "./lib/aws/rubyflow/asyncBacktrace.rb:75:in `caller'"
+        # "./lib/aws/rubyflow/asyncBacktrace.rb:21:in `create'"
+        # "./lib/aws/rubyflow/flow.rb:16:in `make_backtrace'"
+        # "./lib/aws/rubyflow/flow.rb:103:in `initialize'"
+        # "./lib/aws/rubyflow/asyncScope.rb:17:in `new'"
+        # "./lib/aws/rubyflow/asyncScope.rb:17:in `initialize'"
+
+        frames_to_skip = 7
+        backtrace = AsyncBacktrace.create(parent_backtrace, frames_to_skip)
+      end
+    end
+  end
+end

data/aws-flow-core/lib/aws/flow/future.rb

@@ -0,0 +1,109 @@
+##
+# Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License").
+# You may not use this file except in compliance with the License.
+# A copy of the License is located at
+#
+#  http://aws.amazon.com/apache2.0
+#
+# or in the "license" file accompanying this file. This file is distributed
+# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+# express or implied. See the License for the specific language governing
+# permissions and limitations under the License.
+##
+
+# This file contains our Future implementation, which allows us to have asynchronous, blocking promises
+
+module AWS
+  module Flow
+    module Core
+      class AlreadySetException < Exception; end
+
+      # A Future represents the result of an asynchronous computation. Methods are
+      # provided to check if the computation is complete(Future#set), to wait for
+      # its completion(Future#wait), and to retrieve the result of the
+      # computation(Future#get). The result can only be retrieved using method get
+      # when the computation has completed, blocking if necessary until it is
+      # ready. This is okay, however, because it will block that Fiber, and
+      # another Fiber will start executing
+      class Future
+
+        # Sets the value of the future, and notifies all of the Fibers that tried
+        # to get when this future wasn't ready.
+        def set(result=nil)
+          raise AlreadySetException if @set
+          @set = true
+          @result = result
+          @conditional.broadcast if @conditional
+          @listeners.each { |b| b.call(self) } if @listeners
+          self
+        end
+
+        # Blocks if Future is not set
+        # raises CancellationError when task is cancelled
+        def get
+          until @set
+            @conditional ||= FiberConditionVariable.new
+            @conditional.wait
+          end
+          @result
+        end
+
+        def set?
+          @set
+        end
+
+        def unset
+          @set = nil
+          @result = nil
+        end
+
+        # Add a callback, block, which will fire when the future is set
+        def on_set(&block)
+          @listeners ||= []
+          # TODO probably want to use lambda here
+          @listeners << block
+        end
+      end
+
+      # Based on the ruby core source:
+      # https://github.com/ruby/ruby/blob/trunk/lib/thread.rb
+      class FiberConditionVariable
+        #
+        # Creates a new ConditionVariable
+        #
+        def initialize
+          @waiters = []
+        end
+
+
+        # Have the current fiber wait on this condition variable, and wake up when
+        # the FiberConditionVariable is signalled/broadcaster
+        def wait
+          fiber = ::Fiber.current
+          @waiters << fiber
+          Fiber.yield
+          self
+        end
+
+        #
+        # Wakes up the first fiber in line waiting for this lock.
+        #
+        def signal
+          t = @waiters.shift
+          t.schedule if t && t.alive?
+          self
+        end
+
+        #
+        # Wakes up all fibers waiting for this lock.
+        #
+        def broadcast
+          signal until @waiters.empty?
+          self
+        end
+      end
+    end
+  end
+end

data/aws-flow-core/lib/aws/flow/implementation.rb

@@ -0,0 +1,151 @@
+#--
+# Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License").
+# You may not use this file except in compliance with the License.
+# A copy of the License is located at
+#
+#  http://aws.amazon.com/apache2.0
+#
+# or in the "license" file accompanying this file. This file is distributed
+# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+# express or implied. See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+# This file contains the externally visible parts of flow that are expected to be used by customers of flow
+module AWS
+  module Flow
+    module Core
+      class NoContextException < Exception; end
+
+      # @param block
+      #   The block of code to be executed when the task is run.
+      #
+      # @raise [NoContextException]
+      #   If the current fiber does not respond to {#__context__}.
+      #
+      # @return [Future]
+      #   The tasks result, which is a {Future}.
+      #
+      def task(future = nil, &block)
+        fiber = ::Fiber.current
+        raise NoContextException unless fiber.respond_to? :__context__
+        context = fiber.__context__
+        t = Task.new(nil, &block)
+        task_context = TaskContext.new(:parent => context.get_closest_containing_scope, :task => t)
+        context << t
+        t.result
+      end
+
+      #
+      # @param block
+      #   The block of code to be executed when the daemon task is run.
+      #
+      # @return [Future]
+      #   The tasks result, which is a {Future}.
+      #
+      # @raise [NoContextException]
+      #   If the current fiber does not respond to {#__context__}.
+      #
+      def daemon_task(&block)
+        fiber = ::Fiber.current
+        raise NoContextException unless fiber.respond_to? :__context__
+        context = fiber.__context__
+        t = DaemonTask.new(nil, &block)
+        task_context = TaskContext.new(:parent => context.get_closest_containing_scope, :task => t)
+        context << t
+        t.result
+      end
+
+      #
+      # @param block
+      #   The block of code to be executed when the external task is run.
+      #
+      # @return [nil]
+      #
+      # @raise [NoContextException]
+      #   If the current fiber does not respond to {#__context__}.
+      #
+      def external_task(&block)
+        fiber = ::Fiber.current
+        raise NoContextException unless fiber.respond_to? :__context__
+        context = fiber.__context__
+        t = ExternalTask.new(:parent => context.get_closest_containing_scope, &block)
+        task_context = TaskContext.new(:parent => context.get_closest_containing_scope, :task => t)
+        context << t
+        nil
+      end
+
+
+      #
+      #
+      # * *Args*    :
+      #   - block -> a block, which is passed in a BeginRescueEnsureWrapper, and which will define the BeginRescueEnsure#begin, BeginRescueEnsure#rescue, and BeginRescueEnsure#ensure methods
+      # * *Returns* :
+      #   - The result of the begin statement, if there is no error, otherwise the value of the return statement
+      # * *Raises* :
+      #   - +NoContextException+ -> If the current fiber does not respond to #__context__
+      #
+      def error_handler(&block)
+        fiber = ::Fiber.current
+        raise NoContextException unless fiber.respond_to? :__context__
+        context = fiber.__context__
+        begin_rescue_ensure = BeginRescueEnsure.new(:parent => context.get_closest_containing_scope)
+        bge = BeginRescueEnsureWrapper.new(block, begin_rescue_ensure)
+        context << bge
+        context << begin_rescue_ensure
+        begin_rescue_ensure
+      end
+
+      # @param block
+      #   A code block, which is passed within a {BeginRescueEnsureWrapper}, and which must define the
+      #   {BeginRescueEnsure#begin}, {BeginRescueEnsure#rescue}, and {BeginRescueEnsure#ensure} methods.
+      # @!visibility private
+      def _error_handler(&block)
+        error_handler(&block).result
+      end
+
+
+      def wait_for_function(function, *futures)
+        conditional = FiberConditionVariable.new
+        futures.flatten!
+        return nil if futures.empty?
+        result = futures.select(&:set?)
+        return futures.find(&:set?)if function.call(result, futures)
+        futures.each do |f|
+          f.on_set do |set_one|
+            result << set_one
+            conditional.broadcast if function.call(result, futures)
+          end
+        end
+        conditional.wait
+        result
+      end
+
+      # Blocks until *any* of the arguments are set.
+      #
+      # @param [Array] futures
+      #   A list of futures to wait for. The function will return when at least one of these is set.
+      #
+      # @return [Array]
+      #   A list of the set futures, in the order of being set.
+      #
+      def wait_for_any(*futures)
+        wait_for_function(lambda {|result, future_list| result.length >= 1 }, futures)
+      end
+
+      # Blocks until *all* of the arguments are set.
+      #
+      # @param [Array<Future>] futures
+      #   A list of futures to wait for. The function will return only when all of them are set.
+      #
+      # @return [Array]
+      #   A list of the set futures, in the order of being set.
+      #
+      def wait_for_all(*futures)
+        wait_for_function(lambda {|result, future_list| result.size == future_list.size}, futures)
+      end
+    end
+  end
+end

data/aws-flow-core/lib/aws/flow/simple_dfa.rb

@@ -0,0 +1,85 @@
+##
+# Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License").
+# You may not use this file except in compliance with the License.
+# A copy of the License is located at
+#
+#  http://aws.amazon.com/apache2.0
+#
+# or in the "license" file accompanying this file. This file is distributed
+# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+# express or implied. See the License for the specific language governing
+# permissions and limitations under the License.
+##
+
+module AWS
+  module Flow
+    module Core
+
+      # Contains a Data Flow Analysis (DFA)-like framework, where transition functions can perform arbitrary computation
+      # before moving to the next state
+      module SimpleDFA
+        attr_accessor :transitions, :symbols, :states, :start_state
+
+        # Creates a new SimpleDFA instance.
+        #
+        # @param start_state
+        #   The state with which to start the framework.
+        #
+        def init(start_state)
+          include InstanceMethods
+          @start_state = start_state
+          @symbols = []
+          @states = []
+          @transitions = {}
+          @states << start_state
+        end
+
+        # @return the start state
+        #   The start state that was provided when this instance was created.
+        #
+        def get_start_state
+          @start_state
+        end
+
+        # @return [Array]
+        #   The list of all transitions that were added with {#add_transition}.
+        #
+        def get_transitions
+          @transitions
+        end
+
+        def define_general(state, &block)
+          @symbols.each do |symbol|
+            if @transitions[[state, symbol]].nil?
+              @transitions[[state, symbol]] = block
+            end
+          end
+        end
+
+        def add_transition(state, symbol, &block)
+          @symbols << symbol unless @symbols.include? symbol
+          @states << state unless @states.include? state
+          @transitions[[state, symbol]] = block
+        end
+
+        def uncovered_transitions
+          @states.product(@symbols) - @transitions.keys
+        end
+
+        module InstanceMethods
+          attr_accessor :current_state
+
+          def consume(symbol)
+            @current_state ||= self.class.get_start_state
+            func_to_call = self.class.get_transitions[[@current_state, symbol]]
+            raise "This is not a legal transition" unless func_to_call
+            func_to_call.call(self)
+          end
+        end
+      end
+
+    end
+  end
+end

data/aws-flow-core/lib/aws/flow/tasks.rb

@@ -0,0 +1,405 @@
+##
+# Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License").
+# You may not use this file except in compliance with the License.
+# A copy of the License is located at
+#
+#  http://aws.amazon.com/apache2.0
+#
+# or in the "license" file accompanying this file. This file is distributed
+# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+# express or implied. See the License for the specific language governing
+# permissions and limitations under the License.
+##
+
+# Contains all the task methods, which allow arbitrary blocks of code to be run asynchronously
+
+module AWS
+  module Flow
+    module Core
+      class Task < FlowFiber
+        attr_reader :result, :block
+        attr_accessor :backtrace, :__context__, :parent
+
+        # Creates a new Task.
+        #
+        # @param __context__
+        #   A Task needs a reference to the __context__ that created it so that when the "task" macro is called it can
+        #   find the __context__ which the new Task should be added to.
+        #
+        # @param block
+        #   A block of code that will be run by the task.
+        #
+        def initialize(__context__, &block)
+          @__context__ = __context__
+          @result = Future.new
+          @block = block
+
+          # Is the task alive?
+          def alive?
+            super && !@cancelled
+            #!!@alive# && !@cancelled
+          end
+
+          # @return
+          #   The executor for this task.
+          def executor
+            @__context__.executor
+          end
+
+          super() do
+            begin
+              # Not return because 1.9 will freak about local jump problems if you
+              # try to return, as this is inside a block
+              next if @cancelled
+              @result.set(lambda(&block).call)
+              next if @cancelled
+              @__context__.remove(self)
+            rescue Exception => e
+              if @backtrace != e
+                backtrace = AsyncBacktrace.create_from_exception(@backtrace, e)
+                e.set_backtrace(backtrace.backtrace) if backtrace
+              end
+              @__context__.fail(self, e)
+            ensure
+            end
+          end
+        end
+
+        #
+        # Passes the get_heirs calls to the context, to ensure uniform handling of get_heirs
+        #
+        def get_heirs
+          @__context__.get_heirs
+        end
+
+        #
+        # Returns true/false, depending on if we are in a daemon task or not
+        #
+        def is_daemon?
+          return false
+        end
+
+        # Used by Future#signal to schedule the task for re-evaluation.
+        #
+        # This will simply add the task back to the list of things to be run in the parent's event loop.
+        #
+        def schedule
+          @__context__ << self
+        end
+
+        # Cancel will prevent the execution of this particular task, if possible
+        #
+        # @param error
+        #   The error that is the cause of the cancellation
+        #
+        def cancel(error)
+          @cancelled = true
+          @__context__.remove(self)
+        end
+
+        # Fails the given task with the specified error.
+        #
+        # @param error
+        #   The error that is the cause of the cancellation
+        #
+        # @!visibility private
+        def fail(this_task, error)
+          @__context__.fail(this_task, error)
+        end
+        # def fail(this_task, error)
+        #   raise error
+        # end
+
+        # Adds a task to this task's context
+        #
+        # @param this_task
+        #   The task to add.
+        #
+        def <<(this_task)
+          @__context__.parent << this_task
+        end
+
+        # Removes a task from this task's context
+        #
+        # @param this_task
+        #   The task to remove.
+        #
+        def remove(this_task)
+          @__context__.remove(this_task)
+        end
+
+
+      end
+
+
+      # Similar to a regular Task in all functioning except that it is not assured a chance to execute. Whereas a
+      # begin/run/execute block cannot be closed out while there are still nonDaemonHeirs, it can happily entered the
+      # closed state with daemon heirs, making them essentially unrunnable.
+      class DaemonTask < Task
+
+        def is_daemon?
+          return true
+        end
+
+      end
+
+      # External Tasks are used to bridge asynchronous execution to external asynchronous APIs or events. It is passed a block, like so
+      #
+      #     external_task do |t|
+      #       t.cancellation_handler { |h, cause| h.fail(cause) }
+      #       t.initiate_task { |h| trace << :task_started; h.complete; }
+      #     end
+      #
+      # The {ExternalTask#initiate_task} method is expected to call an external API and return without blocking.
+      # Completion or failure of the external task is reported through ExternalTaskCompletionHandle, which is passed into
+      # the initiate_task and cancellation_handler blocks. The cancellation handler, defined in the same block as the
+      # initiate_task, is used to report the cancellation of the external task.
+      #
+      class ExternalTask < FlowFiber
+        attr_reader :block
+        attr_accessor :cancelled, :inCancellationHandler, :parent, :backtrace, :__context__
+
+
+        # Will always be false, provides a common api for BRE's to ensure they are maintaining their nonDaemonHeirsCount
+        # correctly
+        # @!visibility private
+        def is_daemon?
+          false
+        end
+
+        #
+        #  Passes the get_heirs calls to the context, to ensure uniform handling of
+        #  get_heirs
+        #
+        # @!visibility private
+        def get_heirs
+          @__context__.get_heirs
+        end
+
+        # @!visibility private
+        def initialize(options = {}, &block)
+          @inCancellationHandler = false
+          @block = block
+          # TODO: What should the default value be?
+          @parent = options[:parent]
+          @handle = ExternalTaskCompletionHandle.new(self)
+          block.call(self)
+        end
+
+        # This method is here because the way we create ExternalTasks is a little
+        # tricky - if the parent isn't passed in on construction(as is the case with
+        # the external_task function), then the parent will only be set after
+        # ExternalTask#initialize is called. We'd prefer to set it in the initiailze,
+        # however, the backtrace relies on the parent's backtrace, and so we cannot do
+        # that. Instead, we use this method to lazily create it, when it is
+        # called. The method itself simply sets the backtrace to the the
+        # make_backtrace of the parent's backtrace, if the backtrace is not already
+        # set, and will otherwise simply return the backtrace
+        # @!visibility private
+        def backtrace
+          @backtrace ||= make_backtrace(@parent.backtrace)
+        end
+
+        # Add a task which removes yourself, and pass it through the parents executor
+        # @!visibility private
+        def remove_from_parent
+          @__context__.executor << FlowFiber.new { @parent.remove(self) }
+        end
+
+        # Add a task which fails yourself with the suppiled error, and pass it through
+        # the parents executor
+        # @!visibility private
+        def fail_to_parent(error)
+          @__context__.executor << FlowFiber.new { @parent.fail(self, error) }
+        end
+
+
+        # Part of the interface provided by Fiber, has to overridden to properly
+        # reflect that an ExternalTasks alive-ness relies on it's
+        # ExternalTaskCompletionHandle
+        # @!visibility private
+        def alive?
+          ! @handle.completed
+        end
+
+        # @!visibility private
+        def cancel(cause)
+          return if @cancelled
+          return if @handle.failure != nil || @handle.completed
+          @cancelled = true
+          if @cancellation_task != nil
+            begin
+              @inCancellationHandler = true
+              @cancellation_task.call(cause)
+            rescue Exception => e
+              if ! self.backtrace.nil?
+                backtrace = AsyncBacktrace.create_from_exception(@backtrace, e)
+                e.set_backtrace(backtrace.backtrace) if backtrace
+              end
+              @handle.failure = e
+            ensure
+              @inCancellationHandler = false
+              if ! @handle.failure.nil?
+                fail_to_parent(@handle.failure)
+              elsif @handle.completed
+                remove_from_parent
+              end
+            end
+          else
+            remove_from_parent
+          end
+        end
+
+        # Store the cancellation handler block passed in for later reference
+        # @!visibility private
+        def cancellation_handler(&block)
+          @cancellation_task = lambda { |cause| block.call(@handle, cause) }
+        end
+
+        # Store the block passed in for later
+        # @!visibility private
+        def initiate_task(&block)
+          @initiation_task = lambda { block.call(@handle) }
+        end
+
+        # From the interface provided by Fiber, will execute the External Task
+        # @!visibility private
+        def resume
+          return if @cancelled
+          begin
+            @cancellation_handler = @initiation_task.call
+          rescue Exception => e
+            backtrace = AsyncBacktrace.create_from_exception(self.backtrace, e)
+            e.set_backtrace(backtrace.backtrace) if backtrace
+            @parent.fail(self, e)
+          end
+        end
+      end
+
+      # Used to complete or fail an external task initiated through
+      # ExternalTask#initiate_task, and thus handles the logic of what to do when the
+      # external task is failed.
+      # @!visibility private
+      class ExternalTaskCompletionHandle
+        attr_accessor :completed, :failure, :external_task
+
+        # @!visibility private
+        def initialize(external_task)
+          @external_task = external_task
+        end
+
+        # Will merge the backtrace, set the @failure, and then fail the task from the parent
+        # @!visibility private
+        #
+        # @param error
+        #   The exception to fail on
+        #
+        # @raise IllegalStateException
+        #   Raises if failure hasn't been set, or if the task is already completed
+        #
+        # @!visibility private
+        def fail(error)
+          if ! @failure.nil?
+            raise IllegalStateException, "Invalid ExternalTaskCompletionHandle"
+          end
+          if @completed
+            raise IllegalStateException, "Already completed"
+          end
+          #TODO Might want to flip the logic to only alert if variable is set
+          if @stacktrace.nil?
+            if ! @external_task.backtrace.nil?
+              backtrace = AsyncBacktrace.create_from_exception(@external_task.backtrace, error)
+              error.set_backtrace(backtrace.backtrace) if backtrace
+            end
+          end
+          @failure = error
+          if ! @external_task.inCancellationHandler
+            @external_task.fail_to_parent(error)
+          end
+        end
+
+        # Set's the task to complete, and removes it from it's parent
+        #
+        # @raise IllegalStateException
+        #   If the failure hasn't been set, or if the task is already completed
+        #
+        # @!visibility private
+        def complete
+          if ! failure.nil?
+            raise IllegalStateException, ""
+          end
+
+          if @completed
+            raise IllegalStateException, "Already Completed"
+          end
+          @completed = true
+          @external_task.remove_from_parent if ! @external_task.inCancellationHandler
+        end
+      end
+
+      # TaskContext is the class that holds some meta-information for tasks, and which stores the parent link for tasks.
+      # It seperates some of the concerns between tasks and what they have to know to follow back up the chain.
+      #
+      #  All the methods here will simply delegate calls, either up to the parent, or down to the task
+      # @!visibility private
+      class TaskContext
+
+        attr_accessor :daemon, :parent, :backtrace, :cancelled
+
+        def initialize(options = {})
+          @parent = options[:parent]
+          @task = options[:task]
+          @task.__context__ = self
+          @non_cancelling = options[:non_cancelling]
+          @daemon = options[:daemon]
+        end
+
+        # @!visibility private
+        def get_closest_containing_scope
+          @task
+          # @ parent
+        end
+
+        # @!visibility private
+        def alive?
+          @task.alive?
+        end
+
+        # @!visibility private
+        def executor
+          @parent.executor
+        end
+
+        # @!visibility private
+        def get_heirs
+          str = "I am a #{@task.class}
+        and my block looks like #{@task.block}"
+        end
+
+        # @!visibility private
+        def fail(this_task, error)
+          @parent.fail(this_task, error)
+        end
+
+        # @!visibility private
+        def remove(thread)
+          @parent.remove(thread)
+        end
+
+        # @!visibility private
+        def cancel(error_type)
+          @task.cancelled = true
+          @parent.cancel(self)
+        end
+
+        # @!visibility private
+        def <<(task)
+          @parent << task
+        end
+
+      end
+    end
+  end
+end