oflow 0.3.0

data/lib/oflow/link.rb

@@ -0,0 +1,43 @@
+
+module OFlow
+
+  # A Link is the data needed to link one Task with another so that when the
+  # ship() method is called the data can be delivered to the destination Task.
+  class Link
+    
+    # Name of the target.
+    attr_reader :target_name
+    # Operation to provide the target.
+    attr_reader :op
+    # The actual target Task or Flow.
+    attr_reader :target
+    # Flag indicating the Link is from a Flow to a Task contained in the Flow.
+    attr_reader :ingress
+
+    # Creates a new Link. This is called from link() and route() methods on
+    # Tasks and Flows.
+    # @param target_name [Symbol] target Task base name
+    # @param op [Symbol] operation to use on the target
+    # @param ingress [Boolean] indicates the Link is internal
+    # @return [Link] new Link
+    def initialize(target_name, op, ingress=false)
+      @target_name = target_name
+      @op = op
+      @target = nil
+      @ingress = ingress
+    end
+
+    # Delivers a package (Box) to the target.
+    # @param box [Box] package to deliver
+    def ship(box)
+      @target.receive(@op, box)
+    end
+
+    # Returns a string representation of the Link.
+    def to_s()
+      "Link{ingress: #{@ingress}, target_name: #{@target_name}, op: #{op}, target: #{@target}}"
+    end
+    alias inspect to_s
+
+  end # Link
+end # OFlow

data/lib/oflow/pattern.rb

@@ -0,0 +1,8 @@
+
+module OFlow
+
+  # TBD
+  class Pattern
+
+  end # Pattern
+end # OFlow

data/lib/oflow/stamp.rb

@@ -0,0 +1,39 @@
+
+require 'time'
+
+module OFlow
+
+  # Information used to identify a location and time that a Box was
+  # received. Stamps are placed in Trackers.
+  class Stamp
+
+    # Full name of Task that created the Stamp in a Tracker.
+    attr_reader :location
+    # Operation that caused the Stamp to be created.
+    attr_reader :op
+    # The time the Stamp was created.
+    attr_reader :time
+
+    # Create a new Stamp.
+    # @param location [String] full name of Task that created the Stamp in a Tracker
+    # @param op [Symbol] operation that caused the Stamp to be created
+    # @param time [Time] time the Stamp was created
+    def initialize(location, op=nil, time=nil)
+      @location = location
+      @op = op
+      @time = (time || Time.now).utc
+    end
+
+    # Returns a string composed of the location and operation.
+    def where()
+      "#{@location}-#{@op}"
+    end
+
+    # Returns a String representation of the Stamp.
+    def to_s()
+      "#{@location}-#{@op}@#{@time.iso8601(9)}"
+    end
+    alias inspect to_s
+
+  end # Stamp
+end # OFlow

data/lib/oflow/task.rb

@@ -0,0 +1,415 @@
+
+module OFlow
+
+  # The Task class provides the asynchronous functionality for the system. Each
+  # Task has it's own thread and receives requests as an operation and Box of
+  # data by the receive() method. The request is put on a queue and popped off
+  # one at a time and handed to an Actor associatesd with the Task.
+  class Task
+    include HasName
+    include HasLinks
+    include HasErrorHandler
+    include HasLog
+
+    # value of @state that indicates the Task is being created.
+    STARTING = 0
+    # value of @state that indicates the Task is not currently processing requests
+    STOPPED  = 1
+    # value of @state that indicates the Task is currently ready to process requests
+    RUNNING  = 2
+    # value of @state that indicates the Task is shutting down
+    CLOSING  = 3
+    # value of @state that indicates the Task is not receiving new requests.
+    BLOCKED  = 4
+    # value of @state that indicates the Task is processing one request and will
+    # stop after that processing is complete
+    STEP     = 5
+
+    # The current processing state of the Task
+    attr_reader :state
+    # the Actor
+    attr_reader :actor
+
+    # A Task is initialized by specifying a class to create an instance of.
+    # @param flow [Flow] Flow containing the Task
+    # @param name [name] Task base name
+    # @param actor_class [Class] _class Class of the Actor to create
+    # @param options [Hash] additional options for the Task or Actor
+    def initialize(flow, name, actor_class, options={})
+      @state = STARTING
+      @queue = []
+      @req_mutex = Mutex.new()
+      @req_thread = nil
+      @step_thread = nil
+      @waiting_thread = nil
+      @req_timeout = 0.0
+      @max_queue_count = nil
+      @current_req = nil
+      @proc_cnt = 0
+      @loop = nil
+
+      init_name(flow, name)
+      init_links()
+      set_options(options)
+
+      @actor = actor_class.new(self, options)
+      raise Exception.new("#{actor} does not respond to the perform() method.") unless @actor.respond_to?(:perform)
+
+      @state = RUNNING
+      return unless @actor.with_own_thread()
+
+      @loop = Thread.start(self) do |me|
+        Thread.current[:name] = me.full_name()
+        while CLOSING != @state
+          begin
+            if RUNNING == @state || STEP == @state || BLOCKED == @state
+              req = nil
+              if @queue.empty?
+                @waiting_thread.wakeup() unless @waiting_thread.nil?
+                sleep(1.0)
+                next
+              end
+              @req_mutex.synchronize {
+                req = @queue.pop()
+              }
+              @req_thread.wakeup() unless @req_thread.nil?
+
+              @current_req = req
+              begin
+                @actor.perform(req.op, req.box) unless req.nil?
+              rescue Exception => e
+                handle_error(e)
+              end
+              @proc_cnt += 1
+              @current_req = nil
+              if STEP == @state
+                @step_thread.wakeup() unless @step_thread.nil?
+                @state = STOPPED
+              end
+            elsif STOPPED == @state
+              sleep(1.0)
+            end
+          rescue Exception => e
+                puts "*** #{full_name} #{e.class}: #{e.message}"
+            @current_req = nil
+            # TBD Env.rescue(e)
+          end
+        end
+      end
+    end
+
+    # Returns the state of the Task as a String.
+    # @return [String] String representation of the state
+    def state_string()
+      ss = 'UNKNOWN'
+      case @state
+      when STOPPED
+        ss = 'STOPPED'
+      when RUNNING
+        ss = 'RUNNING'
+      when CLOSING
+        ss = 'CLOSING'
+      when STEP
+        ss = 'STEP'
+      end
+      ss
+    end
+
+    # Returns a String that describes the Task.
+    # @param detail [Fixnum] higher values result in more detail in the description
+    # @param indent [Fixnum] the number of spaces to indent the description
+    def describe(detail=0, indent=0)
+      i = ' ' * indent
+      lines = ["#{i}#{name} (#{actor.class}) {"]
+      @links.each { |local,link|
+        lines << "  #{i}#{local} => #{link.target_name}:#{link.op}"
+      }
+      if 1 <= detail
+        lines << "  #{i}queued: #{queue_count()} (#{busy? ? 'busy' : 'idle'})"
+        lines << "  #{i}state: #{state_string()}"
+        @actor.options.each do |key,value|
+          lines << "  #{i}#{key} = #{value} (#{value.class})"
+        end
+        if 2 <= detail
+          lines << "  #{i}processing: #{@current_req.describe(detail)}" unless @current_req.nil?
+          # Copy queue so it doesn't change while working with it and don't
+          # block the queue. It is okay for the requests to be stale as it is
+          # for a display that will be out of date as soon as it is displayed.
+          reqs = []
+          @req_mutex.synchronize {
+            reqs = Array.new(@queue)
+          }
+          lines << "  #{i}queue:"
+          reqs.each do |req|
+            lines << "    #{i}#{req.describe(detail)}"
+          end
+        end
+      end
+      lines << i + "}"
+      lines.join("\n")
+    end
+
+    # Returns the number of requests on the queue.
+    # @return [Fixnum] number of queued requests
+    def queue_count()
+      @queue.length
+    end
+
+    # Returns a score indicating how backed up the queue is. This is used for
+    # selecting an Actor when stepping from the Inspector.
+    # @return [Fixnum] a measure of how backed up a Task is
+    def backed_up()
+      cnt = @queue.size()
+      return 0 if 0 == cnt
+      if @max_queue_count.nil? || 0 == @max_queue_count
+        cnt = 80 if 80 < cnt
+        cnt
+      else
+        cnt * 100 / @max_queue_count
+      end
+    end
+
+    # Returns the true if any requests are queued or a request is being processed.
+    # @return [true|false] true if busy, false otherwise
+    def busy?()
+      !@current_req.nil? || !@queue.empty?
+    end
+
+    # Returns the default timeout for the time to wait for the Task to be
+    # ready to accept a request using the receive() method.
+    # @return [Float] current timeout for the receive() method
+    def request_timeout()
+      @req_timeout
+    end
+
+    # Returns the maximum number of requests allowed on the normal processing
+    # queue. A value of nil indicates there is no limit.
+    # @return [NilClass|Fixnum] maximum number of request that can be queued
+    def max_queue_count()
+      @max_queue_count
+    end
+
+    # Returns the total number of requested processed.
+    # @return [Fixnum] number of request processed
+    def proc_count()
+      @proc_cnt
+    end
+
+    # Causes the Actor to stop processing any more requests after the current
+    # request has finished.
+    def stop()
+      @state = STOPPED
+    end
+
+    # Causes the Task to process one request and then stop. The max_wait is
+    # used to avoid getting stuck if the processing takes too long.
+    # @param max_wait [Float|Fixnum] _wait maximum time to wait for the step to complete
+    def step(max_wait=5)
+      return nil if @loop.nil?
+      return nil if STOPPED != @state || @queue.empty?
+      @state = STEP
+      @step_thread = Thread.current
+      @loop.wakeup()
+      sleep(max_wait)
+      @step_thread = nil
+      self
+    end
+
+    # Wakes up the Task if it has been stopped or if Env.shutdown() has been called.
+    def wakeup()
+      @loop.wakeup() unless @loop.nil?
+    end
+
+    # Restarts the Task's processing thread.
+    def start()
+      @state = RUNNING
+      @loop.wakeup() unless @loop.nil?
+    end
+
+    # Closes the Task by exiting the processing thread. If flush is true then
+    # all requests in the queue are processed first.
+    def shutdown(flush_first=false)
+      return if @loop.nil?
+      if flush_first
+        @state = BLOCKED
+        flush()
+      end
+      @state = CLOSING
+      begin
+        # if the loop has already exited this will raise an Exception that can be ignored
+        @loop.wakeup()
+      rescue
+        # ignore
+      end
+      @loop.join()
+    end
+
+    # Waits for all processing to complete before returning.
+    def flush()
+      return if @loop.nil?
+      @waiting_thread = Thread.current
+      begin
+        @loop.wakeup()
+      rescue
+        # ignore
+      end
+      while busy?
+        sleep(2.0)
+      end
+      @waiting_thread = nil
+    end
+
+    # The current state of the Task.
+    def state=(s)
+      @state = s
+    end
+
+    # The expected inputs the Task supports or nil if not implemented.
+    def inputs()
+      @actor.inputs()
+    end
+
+    # The expected outputs the Task supports or nil if not implemented.
+    def outputs()
+      @actor.outputs()
+    end
+
+    # Creates a Request and adds it to the queue.
+    # @param op [Symbol] operation to perform
+    # @param box [Box] contents or data for the request
+    def receive(op, box)
+      return if CLOSING == @state
+
+      raise BlockedError.new() if BLOCKED == @state
+
+      box = box.receive(full_name, op) unless box.nil?
+      # Special case for starting state so that an Actor can place an item on
+      # the queue before the loop is started.
+      if @loop.nil? && STARTING != @state # no thread task
+        begin
+          @actor.perform(op, box)
+        rescue Exception => e
+          handle_error(e)
+        end
+        return
+      end
+      unless @max_queue_count.nil? || 0 == @max_queue_count || @queue.size() < @max_queue_count
+        @req_thread = Thread.current
+        sleep(timeout) unless @req_timeout.nil? || 0 == @req_timeout
+        @req_thread = nil
+        raise BusyError.new() unless @queue.size() < @max_queue_count
+      end
+      @req_mutex.synchronize {
+        @queue.insert(0, Request.new(op, box))
+      }
+      @loop.wakeup() if RUNNING == @state
+    end
+
+    # Sends a message to another Task or Flow.
+    # @param dest [Symbol] identifies the link that points to the destination Task or Flow
+    # @param box [Box] contents or data for the request
+    def ship(dest, box)
+      box.freeze() unless box.nil?
+      link = resolve_link(dest)
+      raise LinkError.new(dest) if link.nil? || link.target.nil?
+      link.target.receive(link.op, box)
+      link
+    end
+
+    # Processes the initialize() options. Subclasses should call super.
+    # @param options [Hash] options to be used for initialization
+    # @option options [Fixnum] :max_queue_count maximum number of requests
+    #         that can be queued before backpressure is applied to the caller.
+    # @option options [Float] :request_timeout timeout in seconds to wait
+    #         before raising a BusyError if the request queue is too long.
+    def set_options(options)
+      @max_queue_count = options.fetch(:max_queue_count, @max_queue_count)
+      @req_timeout = options.fetch(:request_timeout, @req_timeout).to_f
+    end
+    
+    def _validation_errors()
+      errors = []
+      @links.each_value { |lnk| _check_link(lnk, errors) }
+
+      unless (outs = @actor.outputs()).nil?
+        outs.each do |spec|
+          if find_link(spec.dest).nil?
+            errors << ValidateError::Problem.new(full_name, ValidateError::Problem::MISSING_ERROR, "Missing link for '#{spec.dest}'.")
+          end
+        end
+      end
+      errors
+    end
+
+    def has_input(op)
+      ins = @actor.inputs()
+      return true if ins.nil?
+      op = op.to_sym unless op.nil?
+      ins.each { |spec| return true if spec.op.nil? || spec.op == op }
+      false
+    end
+
+    def _check_link(lnk, errors)
+      if lnk.target.nil?
+        errors << ValidateError::Problem.new(full_name, ValidateError::Problem::LINK_ERROR, "Failed to find task '#{lnk.target_name}'.")
+        return
+      end
+      unless lnk.target.has_input(lnk.op)
+        errors << ValidateError::Problem.new(full_name, ValidateError::Problem::INPUT_ERROR, "'#{lnk.op}' not allowed on '#{lnk.target.full_name}'.")
+        return
+      end
+
+      # TBD
+      # Verify target has link.op as input if input is specified.
+
+    end
+
+    # Resolves all links. Called by the system.
+    def resolve_all_links()
+      @links.each_value { |lnk|
+        set_link_target(lnk) if lnk.target.nil?
+      }
+    end
+
+    private
+
+    # Internal class used to store information about asynchronous method
+    # invocations.
+    class Request
+      attr_accessor :op
+      attr_accessor :box
+
+      def initialize(op, box)
+        @op = op
+        @box = box
+      end
+
+      def describe(detail=3)
+        if @box.nil?
+          "#{@op}()"
+        elsif 2 >= detail
+          @op
+        elsif 3 >= detail || @box.tracker.nil?
+          "#{@op}(#{@box.contents})"
+        else
+          "#{@op}(#{@box.contents}) #{@box.tracker}"
+        end
+      end
+
+    end # Request
+
+    class Link
+      attr_reader :name
+      attr_reader :task
+      attr_reader :op
+
+      def initialize(name, task, op)
+        @name = name
+        @task = task
+        @op = op
+      end
+
+    end # Link
+
+  end # Task
+end # OFlow

data/lib/oflow/test/action.rb

@@ -0,0 +1,21 @@
+
+module OFlow
+  module Test
+    
+    class Action
+      attr_reader :dest
+      attr_reader :box
+
+      def initialize(dest, box)
+        @dest = dest
+        @box = box
+      end
+
+      def to_s()
+        "#{@dest}: #{box.contents}"
+      end
+      alias inspect to_s
+
+    end # Action
+  end # Test
+end # OFlow

data/lib/oflow/test/actorwrap.rb

@@ -0,0 +1,62 @@
+
+module OFlow
+  module Test
+    class ActorWrap
+
+      attr_reader :actor
+      attr_accessor :name
+
+      # Array of Actions. Log entries appear with a destination of :log.
+      attr_reader :history
+
+      def initialize(name, actor_class, options={})
+        @name = name
+        @actor = actor_class.new(self, options)
+        @history = []
+      end
+
+      def reset()
+        @history = []
+      end
+      
+      def full_name()
+        ":test:#{@name}"
+      end
+
+      # Calls perform on the actor instance
+      def receive(op, box)
+        @actor.perform(op, box)
+      end
+
+      # Task API that adds entry to history.
+      def ship(dest, box)
+        @history << Action.new(dest, box)
+      end
+
+      def log_msg(level, msg, fn)
+        @history << Action.new(:log, Box.new([level, msg, fn]))
+      end
+
+      def debug(msg)
+        log_msg(:debug, msg, full_name())
+      end
+
+      def info(msg)
+        log_msg(:info, msg, full_name())
+      end
+
+      def error(msg)
+        log_msg(:error, msg, full_name())
+      end
+
+      def warn(msg)
+        log_msg(:warn, msg, full_name())
+      end
+
+      def fatal(msg)
+        log_msg(:fatal, msg, full_name())
+      end
+
+    end # ActorWrap
+  end # Test
+end # OFlow

data/lib/oflow/test.rb

@@ -0,0 +1,8 @@
+
+module OFlow
+  module Test
+  end
+end
+
+require 'oflow/test/action'
+require 'oflow/test/actorwrap'

data/lib/oflow/tracker.rb

@@ -0,0 +1,109 @@
+
+require 'socket'
+
+module OFlow
+
+  # A Tracker is used to track data through the system. They are attached to
+  # Boxes and are updated when they are received by Flows and Tasks.
+  class Tracker
+    # Gets the machine address. This is used for generating unique identifiers
+    # for the Tracker instances.
+    def self.get_machine()
+      machine = "unknown"
+      Socket.ip_address_list.each do |addr|
+        next unless addr.ip?
+        next if addr.ipv6_linklocal?
+        next if addr.ipv4_loopback? || addr.ipv6_loopback?
+        next if addr.ipv4_multicast? || addr.ipv6_multicast?
+        machine = addr.ip_address
+        break
+      end
+      machine
+    end
+
+    @@machine = self.get_machine()
+    @@pid = Process.pid
+    @@last_nano = 0
+    @@nano_mutex = Mutex.new()
+
+    # The identifier of the Tracker.
+    attr_reader :id
+    # The Stamps that were placed in the Tracker as it is received.
+    attr_reader :track
+
+    def self.create(location, op=nil)
+      t = Tracker.new(gen_id(), [Stamp.new(location, op).freeze()])
+      t.track.freeze()
+      t.freeze()
+    end
+
+    # Returns an updated instance with a Stamp for the location, operation, and
+    # current time.
+    # @param location [String] full name of Task where the Tracker was received
+    # @param op [Symbol] operation that caused the Stamp to be created
+    # @return [Tracker] updated Tracker
+    def receive(location, op)
+      t = Tracker.new(@id, Array.new(@track) << Stamp.new(location, op).freeze())
+      t.track.freeze()
+      t.freeze()
+    end
+
+    # Returns a String representation of the Tracker.
+    def to_s()
+      "Tracker{#{@id}, track: #{@track}}"
+    end
+    alias inspect to_s
+
+    # When a package is split and travels on more than one route the Tracker can
+    # be merged with this method. The returned Tracker contains both tracks.
+    # @param t2 [Tracker] other Tracker
+    # @return [Tracker] merged Tracker
+    def merge(t2)
+      raise Exception.new("Can not merge #{t2.id} into #{@id}. Different IDs.") if t2.id != @id
+      comb = []
+      s2 = t2.track.size
+      for i in 0..@track.size
+        break if s2 <= i
+        unless @track[i] == t2.track[i]
+          if @track[-1].location == t2.track[-1].location
+            comb << [@track[i..-2], t2.track[i..-2]]
+            comb << @track[-1]
+          else
+            comb << [@track[i..-1], t2.track[i..-1]]
+          end
+          break
+        end
+        comb << @track[i]
+      end
+      comb.freeze
+      Tracker.new(@id, comb)
+    end
+
+    private
+
+    def self.gen_id()
+      nano = (Time.now.to_f * 1000000000.0).to_i
+      @@nano_mutex.synchronize do
+        while nano <= @@last_nano
+          nano += 1
+        end
+        @@last_nano = nano
+      end
+      "#{@@machine}.#{@@pid}.#{nano}"
+    end
+
+    def initialize(id, track)
+      @id = id
+      @track = track
+    end
+
+    def id=(i)
+      @id = i
+    end
+
+    def track=(t)
+      @track = t
+    end
+
+  end # Tracker
+end # OFlow