orchestrator 1.0.1

data/lib/orchestrator/device/command_queue.rb

@@ -0,0 +1,213 @@
+require 'algorithms'
+require 'bisect'
+
+
+# Transport -> connection (make break etc)
+# * attach connected, disconnected callbacks
+# * udp, makebreak and tcp transports
+# Manager + CommandProcessor + Transport
+
+
+module Orchestrator
+    module Device
+        class CommandQueue
+
+
+            OFFLINE_MSG = Error::CommandCanceled.new 'command canceled as module went offline'
+
+
+            attr_accessor :waiting
+            attr_reader :state
+            attr_reader :pause
+
+
+            # init -> mod.load -> post_init
+            # So config can be set in on_load if desired
+            def initialize(loop, callback)
+                @loop = loop
+                @callback = callback
+
+                @named_commands = {
+                    # name: [[priority list], command]
+                    # where command may be nil
+                }
+                @comparison = method(:comparison)
+                @pending_commands = Containers::Heap.new(&@comparison)
+
+                @waiting = nil      # Last command sent that was marked as waiting
+                @pause = 0
+                @state = :online    # online / offline
+                @pause_shift = method(:pause_shift)
+                @move_forward = method(:move_forward)
+            end
+
+            def shift_next_tick
+                @pause += 1
+                @loop.next_tick @pause_shift
+            end
+
+            def shift
+                return if @pause > 0 # we are waiting for the next_tick?
+
+                @waiting = nil  # Discard the current command
+                if length > 0
+                    next_cmd = @pending_commands.pop
+
+                    if next_cmd.is_a? Symbol # (named command)
+                        result = @named_commands[next_cmd]
+                        result[0].shift
+                        cmd = result[1]
+                        if cmd.nil?
+                            shift_next_tick if length > 0
+                            return  # command already executed, this is a no-op
+                        else
+                            result[1] = nil
+                        end
+                    else
+                        cmd = next_cmd
+                    end
+
+                    @waiting = cmd if cmd[:wait]
+                    shift_promise = @callback.call cmd
+
+                    if shift_promise.is_a? ::Libuv::Q::Promise
+                        @pause += 1
+                        shift_promise.finally do # NOTE:: This schedule may not be required...
+                            @loop.schedule @move_forward
+                        end
+                    else
+                        shift_next_tick if length > 0
+                    end
+                end
+            end
+
+            def push(command, priority)
+                if @state == :offline && command[:name].nil?
+                    return
+                end
+
+                if command[:name]
+                    name = command[:name].to_sym
+
+                    current = @named_commands[name] ||= [[], nil]
+
+                    # Chain the promises if the named command is already in the queue
+                    cmd = current[1]
+                    cmd[:defer].resolve(command[:defer].promise) if cmd
+
+                    
+                    current[1] = command   # replace the old command
+                    priors = current[0]
+
+                    # Only add commands of higher priority to the queue
+                    if priors.empty? || priors[-1] < priority
+                        priors << priority
+                        queue_push(@pending_commands, name, priority)
+                    end
+                else
+                    queue_push(@pending_commands, command, priority)
+                end
+
+                if @waiting.nil? && @state == :online
+                    shift  # This will trigger the callback
+                end
+            end
+
+            def length
+                @pending_commands.size
+            end
+
+
+            # If offline we'll only maintain named command state and queue
+            def online
+                @state = :online
+
+                # next tick is important as it allows the module time to updated
+                # any named commands that it desires in the connected callback
+                shift_next_tick
+            end
+
+            def offline(clear = false)
+                @state = :offline
+
+                if clear
+                    @waiting[:defer].reject(OFFLINE_MSG) if @waiting
+                    cancel_all(OFFLINE_MSG)
+                    @waiting = nil
+                else
+                    # Keep named commands
+                    new_queue = Containers::Heap.new(&@comparison)
+
+                    while length > 0
+                        cmd = @pending_commands.pop
+                        if cmd.is_a? Symbol
+                            res = @named_commands[cmd][0]
+                            pri = res.shift
+                            res << pri
+                            queue_push(new_queue, cmd, pri)
+                        else
+                            cmd[:defer].reject(OFFLINE_MSG)
+                        end
+                    end
+                    @pending_commands = new_queue
+                    
+                    # clear waiting if it is not a named command.
+                    # The processor will re-queue it if retry on disconnect is set
+                    if @waiting && @waiting[:name].nil?
+                        @waiting = nil
+                    end
+                end
+            end
+
+            def cancel_all(msg)
+                while length > 0
+                    cmd = @pending_commands.pop
+                    if cmd.is_a? Symbol
+                        res = @named_commands[cmd]
+                        if res
+                            res[1][:defer].reject(msg)
+                            @named_commands.delete(cmd)
+                        end
+                    else
+                        cmd[:defer].reject(msg)
+                    end
+                end
+            end
+
+
+            protected
+
+
+            # If we next_tick a shift then a push may be able to
+            # sneak in before that command is shifted.
+            # If the new push is a waiting command then the next
+            # tick shift will discard it which is undesirable
+            def pause_shift
+                @pause -= 1
+                shift
+            end
+
+            def move_forward
+                @pause -= 1
+                if !@waiting && length > 0
+                    shift
+                end
+            end
+
+
+            # Queue related methods
+            def comparison(x, y)
+                if x[0] == y[0]
+                    x[1] < y[1]
+                else
+                    (x[0] <=> y[0]) == 1
+                end
+            end
+
+            def queue_push(queue, obj, pri)
+                pri = [pri, Time.now.to_f]
+                queue.push(pri, obj)
+            end
+        end
+    end
+end

data/lib/orchestrator/device/manager.rb

@@ -0,0 +1,83 @@
+module Orchestrator
+    module Device
+        class Manager < ::Orchestrator::Core::ModuleManager
+            def initialize(*args)
+                super(*args)
+
+                # Do we want to start here?
+                # Should be ok.
+                @thread.next_tick method(:start)
+            end
+
+            attr_reader :processor, :connection
+
+            def start
+                return true unless @processor.nil?
+                @processor = Processor.new(self)
+
+                super # Calls on load (allows setting of tls certs)
+
+                # Load UV-Rays abstraction here
+                @connection = if @settings.udp
+                    UdpConnection.new(self, @processor)
+                elsif @settings.makebreak
+                    ::UV.connect(@settings.ip, @settings.port, MakebreakConnection, self, @processor, @settings.tls)
+                else
+                    ::UV.connect(@settings.ip, @settings.port, TcpConnection, self, @processor, @settings.tls)
+                end
+
+                @processor.transport = @connection
+                true # for REST API
+            end
+
+            def stop
+                super
+                @processor.terminate unless @processor.nil?
+                @processor = nil
+                @connection.terminate unless @connection.nil?
+                @connection = nil
+            end
+
+            def notify_connected
+                if @instance.respond_to? :connected, true
+                    begin
+                        @instance.__send__(:connected)
+                    rescue => e
+                        @logger.print_error(e, 'error in module connected callback')
+                    end
+                end
+
+                update_connected_status(true)
+            end
+
+            def notify_disconnected
+                if @instance.respond_to? :disconnected, true
+                    begin
+                        @instance.__send__(:disconnected)
+                    rescue => e
+                        @logger.print_error(e, 'error in module disconnected callback')
+                    end
+                end
+
+                update_connected_status(false)
+            end
+
+            def notify_received(data, resolve, command = nil)
+                begin
+                    blk = command.nil? ? nil : command[:on_receive]
+                    if blk.respond_to? :call
+                        blk.call(data, resolve, command)
+                    elsif @instance.respond_to? :received, true
+                        @instance.__send__(:received, data, resolve, command)
+                    else
+                        @logger.warn('no received function provided')
+                        :abort
+                    end
+                rescue => e
+                    @logger.print_error(e, 'error in received callback')
+                    return :abort
+                end
+            end
+        end
+    end
+end

data/lib/orchestrator/device/mixin.rb

@@ -0,0 +1,35 @@
+module Orchestrator
+    module Device
+        module Mixin
+            include ::Orchestrator::Core::Mixin
+
+            def send(data, options = {}, &blk)
+                options[:data] = data
+                options[:defer] = @__config__.thread.defer
+                options[:on_receive] = blk if blk     # on command success
+                @__config__.thread.schedule do
+                    @__config__.processor.queue_command(options)
+                end
+                options[:defer].promise
+            end
+
+            def disconnect
+                @__config__.thread.schedule do
+                    @__config__.connection.disconnect
+                end
+            end
+
+            def config(options)
+                @__config__.thread.schedule do
+                    @__config__.processor.config = options
+                end
+            end
+
+            def defaults(options)
+                @__config__.thread.schedule do
+                    @__config__.processor.send_options(options)
+                end
+            end
+        end
+    end
+end

data/lib/orchestrator/device/processor.rb

@@ -0,0 +1,441 @@
+require 'set'
+
+
+# Transport -> connection (make break etc)
+# * attach connected, disconnected callbacks
+# * udp, makebreak and tcp transports
+# Manager + CommandProcessor + Transport
+
+
+module Orchestrator
+    module Device
+        class Processor
+            include ::Orchestrator::Transcoder
+
+
+            # Any command that waits:
+            # send('power on?').then( execute after command complete )
+            # Named commands mean the promise may never resolve
+            # Totally replaces emit as we don't care and makes cross module request super easy
+            # -- non-wait commands resolve after they have been written to the socket!!
+
+
+            SEND_DEFAULTS = {
+                wait: true,                 # wait for a response before continuing with sends
+                delay: 0,                   # make sure sends are separated by at least this (in milliseconds)
+                delay_on_receive: 0,        # delay the next send by this (milliseconds) after a receive
+                max_waits: 3,               # number of times we will ignore valid tokens before retry
+                retries: 2,                 # Retry attempts before we give up on the command
+                hex_string: false,          # Does the input need conversion
+                timeout: 5000,              # Time we will wait for a response
+                priority: 50,               # Priority of a send
+                force_disconnect: false     # Mainly for use with make and break
+
+                # Other options include:
+                # * emit callback to occur once command complete (may be discarded if a named command)
+                # * on_receive (alternative to received function)
+            }
+
+            CONFIG_DEFAULTS = {
+                tokenize: false,    # If replaced with a callback can define custom tokenizers
+                size_limit: 524288, # 512kb buffer max
+                clear_queue_on_disconnect: false,
+                flush_buffer_on_disconnect: false,
+                priority_bonus: 20,  # give commands bonus priority under certain conditions
+                update_status: true, # auto update connected status?
+                thrashing_threshold: 1500  # min milliseconds between connection retries
+
+                # Other options include:
+                # * inactivity_timeout (used with make and break)
+                # * delimiter (string or regex to match message end)
+                # * indicator (string or regex to match message start)
+                # * verbose (throw errors or silently recover)
+                # * wait_ready (wait for some signal before signaling connected)
+                # * encoding (BINARY) (force encoding on incoming data)
+            }
+
+
+            SUCCESS = Set.new([true, :success, :abort, nil, :ignore])
+            FAILURE = Set.new([false, :retry, :failed, :fail])
+            DUMMY_RESOLVER = proc {}
+            TERMINATE_MSG = Error::CommandCanceled.new 'command canceled due to module shutdown'
+            UNNAMED = 'unnamed'
+
+
+            attr_reader :config, :queue, :thread
+            attr_accessor :transport
+
+            # For statistics only
+            attr_reader :last_sent_at, :last_receive_at, :timeout
+
+
+            # init -> mod.load -> post_init
+            # So config can be set in on_load if desired
+            def initialize(man)
+                @man = man
+
+                @thread = @man.thread
+                @logger = @man.logger
+                @defaults = SEND_DEFAULTS.dup
+                @config = CONFIG_DEFAULTS.dup
+
+                @queue = CommandQueue.new(@thread, method(:send_next))
+                @responses = []
+                @wait = false
+                @connected = false
+                @checking = Mutex.new
+                @bonus = 0
+
+                @last_sent_at = 0
+                @last_receive_at = 0
+
+
+                # Used to indicate when we can start the next response processing
+                @head = ::Libuv::Q::ResolvedPromise.new(@thread, true)
+                @tail = ::Libuv::Q::ResolvedPromise.new(@thread, true)
+
+                # Method variables
+                @resolver = proc { |resp| @thread.schedule { resolve_callback(resp) } }
+
+                @resp_success = proc { |result| @thread.next_tick { resp_success(result) } }
+                @resp_failure = proc { |reason| @thread.next_tick { resp_failure(reason) } }
+            end
+
+            ##
+            # Helper functions ------------------
+            def send_options(options)
+                @defaults.merge!(options)
+            end
+
+            def config=(options)
+                @config.merge!(options)
+                # use tokenize to signal a buffer update
+                new_buffer if options.include?(:tokenize)
+            end
+
+            #
+            # Public interface
+            def queue_command(options)
+                # Make sure we are sending appropriately formatted data
+                raw = options[:data]
+
+                if raw.is_a?(Array)
+                    options[:data] = array_to_str(raw)
+                elsif options[:hex_string] == true
+                    options[:data] = hex_to_byte(raw)
+                end
+
+                data = options[:data]
+                options[:retries] = 0 if options[:wait] == false
+
+                if options[:name].is_a? String
+                    options[:name] = options[:name].to_sym
+                end
+
+                # merge in the defaults
+                options = @defaults.merge(options)
+
+                @queue.push(options, options[:priority] + @bonus)
+
+            rescue => e
+                options[:defer].reject(e)
+                @logger.print_error(e, 'error queuing command')
+            end
+
+            ##
+            # Callbacks -------------------------
+            def connected
+                @connected = true
+                new_buffer
+                @man.notify_connected
+                if @config[:update_status]
+                    @man.trak(:connected, true)
+                end
+            end
+
+            def disconnected
+                @connected = false
+                @man.notify_disconnected
+                if @config[:update_status]
+                    @man.trak(:connected, false)
+                end
+                if @buffer && @config[:flush_buffer_on_disconnect]
+                    check_data(@buffer.flush)
+                end
+                @buffer = nil
+
+                if @queue.waiting
+                    resp_failure(:disconnected)
+                end
+            end
+
+            def buffer(data)
+                @last_receive_at = @thread.now
+
+                if @buffer
+                    @responses.concat @buffer.extract(data)
+                else
+                    # tokenizing buffer above will enforce encoding
+                    if @config[:encoding]
+                        data.force_encoding(@config[:encoding])
+                    end
+                    @responses << data
+                end
+
+                # if we are waiting we don't want to process this data just yet
+                if !@wait
+                    check_next
+                end
+            end
+
+            def terminate
+                @thread.schedule method(:do_terminate)
+            end
+
+            def check_next
+                return if @checking.locked? || @responses.length <= 0
+                @checking.synchronize {
+                    loop do
+                        check_data(@responses.shift)
+                        break if @wait || @responses.length == 0
+                    end
+                }
+            end
+
+
+            protected
+
+
+            def new_buffer
+                tokenize = @config[:tokenize]
+                if tokenize
+                    if tokenize.respond_to? :call
+                        @buffer = tokenize.call
+                    else
+                        @buffer = ::UV::BufferedTokenizer.new(@config)
+                    end
+                elsif @buffer
+                    # remove the buffer if none
+                    @buffer = nil
+                end
+            end
+
+            def do_terminate
+                if @queue.waiting
+                    @queue.waiting[:defer].reject(TERMINATE_MSG)
+                end
+                @queue.cancel_all(TERMINATE_MSG)
+            end
+
+            # Check transport response data
+            def check_data(data)
+                resp = nil
+
+                # Provide commands with a bonus in this section
+                @bonus = @config[:priority_bonus]
+
+                begin
+                    cmd = @queue.waiting
+                    if cmd
+                        @wait = true
+                        @defer = @thread.defer
+                        @defer.promise.then @resp_success, @resp_failure
+
+                        # Disconnect before processing the response
+                        transport.disconnect if cmd[:force_disconnect]
+
+                        # Send response, early resolver and command
+                        resp = @man.notify_received(data, @resolver, cmd)
+                    else
+                        resp = @man.notify_received(data, DUMMY_RESOLVER, nil)
+                        # Don't need to trigger Queue next here as we are not waiting on anything
+                    end
+                rescue => e
+                    # NOTE:: This error should never be called
+                    @logger.print_error(e, 'error processing response data')
+                    @defer.reject :abort if @defer
+                ensure
+                    @bonus = 0
+                end
+
+                # Check if response is a success or failure
+                resolve_callback(resp) unless resp == :async
+            end
+
+            def resolve_callback(resp)
+                if @defer
+                    if FAILURE.include? resp
+                        @defer.reject resp
+                    else
+                        @defer.resolve resp
+                    end
+                    @defer = nil
+                end
+            end
+
+            def resp_failure(result_raw)
+                if @queue.waiting
+                    begin
+                        result = result_raw.is_a?(Fixnum) ? :timeout : result_raw
+                        cmd = @queue.waiting
+                        debug = "with #{result}: <#{cmd[:name] || UNNAMED}> "
+                        if cmd[:data]
+                            debug << "#{cmd[:data].inspect}" 
+                        else
+                            debug << cmd[:path]
+                        end
+                        @logger.debug "command failed #{debug}"
+
+                        if cmd[:retries] == 0
+                            err = Error::CommandFailure.new "command aborted #{debug}"
+                            cmd[:defer].reject(err)
+                            @logger.warn err.message
+                        else
+                            cmd[:retries] -= 1
+                            cmd[:wait_count] = 0      # reset our ignore count
+                            @queue.push(cmd, cmd[:priority] + @config[:priority_bonus])
+                        end
+                    rescue => e
+                        # Prevent the queue from ever pausing - this should never be called
+                        @logger.print_error(e, 'error handling request failure')
+                    end
+                end
+
+                clear_timers
+
+                @wait = false
+                @queue.waiting = nil
+                check_next                    # Process already received
+                @queue.shift if @connected    # Then send a new command
+            end
+
+            # We only care about queued commands here
+            # Promises resolve on the next tick so processing
+            #  is guaranteed to have completed
+            # Check for queue wait as we may have gone offline
+            def resp_success(result)
+                if @queue.waiting && result && result != :ignore
+                    if result == :abort
+                        cmd = @queue.waiting
+                        err = Error::CommandFailure.new "module aborted command with #{result}: <#{cmd[:name] || UNNAMED}> #{(cmd[:data] || cmd[:path]).inspect}"
+                        @queue.waiting[:defer].reject(err)
+                    else
+                        @queue.waiting[:defer].resolve(result)
+                        call_emit @queue.waiting
+                    end
+
+                    clear_timers
+
+                    @wait = false
+                    @queue.waiting = nil
+                    check_next      # Process pending
+                    @queue.shift    # Send the next command
+
+                    # Else it must have been a nil or :ignore
+                elsif @queue.waiting
+                    cmd = @queue.waiting
+                    cmd[:wait_count] ||= 0
+                    cmd[:wait_count] += 1
+                    if cmd[:wait_count] > cmd[:max_waits]
+                        resp_failure(:max_waits_exceeded)
+                    else
+                        check_next
+                    end
+
+                else  # ensure consistent state (offline event may have occurred)
+
+                    clear_timers
+
+                    @wait = false
+                    check_next
+                end
+            end
+
+            # If a callback was in place for the current
+            def call_emit(cmd)
+                callback = cmd[:emit]
+                if callback
+                    @thread.next_tick do
+                        begin
+                            callback.call
+                        rescue => e
+                            @logger.print_error(e, 'error in emit callback')
+                        end
+                    end
+                end
+            end
+
+
+            # Callback for queued commands
+            def send_next(command)
+                # Check for any required delays between sends
+                if command[:delay] > 0
+                    gap = @last_sent_at + command[:delay] - @thread.now
+                    if gap > 0
+                        defer = @thread.defer
+                        sched = schedule.in(gap) do
+                            defer.resolve(process_send(command))
+                        end
+                        # in case of shutdown we need to resolve this promise
+                        sched.catch do
+                            defer.reject(:shutdown)
+                        end
+                        defer.promise
+                    else
+                        process_send(command)
+                    end
+                else
+                    process_send(command)
+                end
+            end
+
+            def process_send(command)
+                # delay on receive
+                if command[:delay_on_receive] > 0
+                    gap = @last_receive_at + command[:delay_on_receive] - @thread.now
+
+                    if gap > 0
+                        defer = @thread.defer
+                        
+                        sched = schedule.in(gap) do
+                            defer.resolve(process_send(command))
+                        end
+                        # in case of shutdown we need to resolve this promise
+                        sched.catch do
+                            defer.reject(:shutdown)
+                        end
+                        defer.promise
+                    else
+                        transport_send(command)
+                    end
+                else
+                    transport_send(command)
+                end
+            end
+
+            def transport_send(command)
+                @transport.transmit(command)
+                @last_sent_at = @thread.now
+
+                if @queue.waiting
+                    # Set up timers for command timeout
+                    @timeout = schedule.in(command[:timeout], @resp_failure)
+                else
+                    # resole the send promise early as we are not waiting for the response
+                    command[:defer].resolve(:no_wait)
+                    call_emit command   # the command has been sent
+                end
+                nil # ensure promise chain is not propagated
+            end
+
+            def clear_timers
+                @timeout.cancel if @timeout
+                @timeout = nil
+            end
+
+            def schedule
+                @schedule ||= @man.get_scheduler
+            end
+        end
+    end
+end