orchestrator 1.0.1

data/lib/orchestrator/engine.rb

@@ -0,0 +1,70 @@
+require 'set'
+
+
+module Orchestrator
+    class Engine < ::Rails::Engine
+        isolate_namespace Orchestrator
+        
+        
+        # NOTE:: if we ever have any tasks
+        #rake_tasks do
+        #    load "tasks/orchestrator_tasks.rake"
+        #end
+        
+        #
+        # Define the application configuration
+        #
+        config.before_initialize do |app|                        # Rails.configuration
+            app.config.orchestrator = ActiveSupport::OrderedOptions.new
+            app.config.orchestrator.module_paths = []
+
+            # Clearance levels defined in code
+            #app.config.orchestrator.clearance_levels = Set.new([:Admin, :Support, :User, :Public])
+
+            # Access checking callback - used at the system level
+            # Will always be passed a system id and the user attempting to access
+            app.config.orchestrator.check_access = proc { |system, user|
+                true
+            }
+
+            # if not zero all UDP sockets must be transmitted from a single thread
+            app.config.orchestrator.datagram_port = 0    # ephemeral port (random selection)
+            app.config.orchestrator.broadcast_port = 0   # ephemeral port (random selection)
+        end
+        
+        #
+        # Discover the possible module location paths after initialization is complete
+        #
+        config.after_initialize do |app|
+            
+            ActiveSupport::Dependencies.autoload_paths.each do |path|
+                Pathname.new(path).ascend do |v|
+                    if ['app', 'vendor'].include?(v.basename.to_s)
+                        app.config.orchestrator.module_paths << "#{v.to_s}/modules"
+                        break
+                    end
+                end
+            end
+            
+            app.config.orchestrator.module_paths.uniq!
+
+            # Force design documents
+            temp = ::Couchbase::Model::Configuration.design_documents_paths
+            ::Couchbase::Model::Configuration.design_documents_paths = [File.expand_path(File.join(File.expand_path("../", __FILE__), '../../app/models/orchestrator'))]
+            ::Orchestrator::ControlSystem.ensure_design_document!
+            ::Orchestrator::Module.ensure_design_document!
+            ::Orchestrator::Zone.ensure_design_document!
+            ::Couchbase::Model::Configuration.design_documents_paths = temp
+
+            # Start the control system by initializing it
+            ctrl = ::Orchestrator::Control.instance
+
+            # Don't auto-load if running in the console or as a rake task
+            unless ENV['ORC_NO_BOOT'] || defined?(Rails::Console) || Rails.env.test? || defined?(Rake)
+                ctrl.loop.next_tick do
+                    ctrl.mount.then ctrl.method(:boot)
+                end
+            end
+        end
+    end
+end

data/lib/orchestrator/errors.rb

@@ -0,0 +1,23 @@
+module Orchestrator
+    class Error < StandardError
+
+        # Called from:
+        # * request_proxy
+        # * requests_proxy
+        class ProtectedMethod < Error; end
+        class ModuleUnavailable < Error; end
+
+        # Called from:
+        # * dependency_manager
+        class FileNotFound < Error; end
+
+        # Called from:
+        # * control
+        class ModuleNotFound < Error; end
+
+        # Called from:
+        # * Device -> Processor
+        class CommandFailure < Error; end
+        class CommandCanceled < Error; end
+    end
+end

data/lib/orchestrator/logger.rb

@@ -0,0 +1,115 @@
+require 'set'
+
+
+# Note:: We should be logging the User id in the log
+# see: http://pastebin.com/Wrjp8b8e (rails log_tags)
+module Orchestrator
+    class Logger
+        LEVEL = {
+            debug: 0,
+            info: 1,
+            warn: 2,
+            error: 3,
+            fatal: 4
+        }.freeze
+
+        # TODO:: Make this a config item
+        DEFAULT_LEVEL = 1
+
+        def initialize(loop, mod)
+            @loop = loop
+            @mod_id = mod.id
+            if mod.respond_to? :dependency
+                @klass = mod.dependency.class_name
+            else
+                @klass = 'User' # Filter by user driven events and behavior
+            end
+            @level = DEFAULT_LEVEL
+            @listeners = Set.new
+            @logger = ::Orchestrator::Control.instance.logger
+        end
+
+        def level=(level)
+            @level = LEVEL[level] || level
+        end
+        attr_reader :level
+
+        # Add listener
+        def add(listener)
+            @loop.schedule do
+                @listeners.add listener
+            end
+            listener.promise.finally do
+                @loop.schedule do
+                    @listeners.delete listener
+                end
+            end
+            listener
+        end
+
+        def delete(listener)
+            @loop.schedule do
+                @listeners.delete listener
+                if @listeners.size == 0
+                    level = DEFAULT_LEVEL   # back to efficient logging
+                end
+            end
+        end
+
+       
+        def debug(msg)
+            if @level <= 0
+                log(:debug, msg)
+            end
+        end
+
+        def info(msg)
+            if @level <= 1
+                log(:info, msg)
+            end
+        end
+
+        def warn(msg)
+            if @level <= 2
+                log(:warn, msg)
+            end
+        end
+
+        def error(msg)
+            if @level <= 3
+                log(:error, msg)
+            end
+        end
+
+        def fatal(msg)
+            if @level <= 4
+                log(:fatal, msg)
+            end
+        end
+
+        def print_error(e, msg = '')
+            msg << "\n#{e.message}"
+            msg << "\n#{e.backtrace.join("\n")}" if e.respond_to?(:backtrace) && e.backtrace
+            error(msg)
+        end
+
+
+        protected
+
+
+        def log(level, msg)
+            @loop.schedule do
+                if LEVEL[level] >= DEFAULT_LEVEL
+                    @loop.work do
+                        @logger.tagged(@klass, @mod_id) {
+                            @logger.send(level, msg)
+                        }
+                    end
+                end
+                @listeners.each do |listener|
+                    listener.notify(@klass, @mod_id, level, msg)
+                end
+            end
+        end
+    end
+end

data/lib/orchestrator/logic/manager.rb

@@ -0,0 +1,18 @@
+module Orchestrator
+    module Logic
+        class Manager < ::Orchestrator::Core::ModuleManager
+            def initialize(*args)
+                super(*args)
+
+                # Do we want to start here?
+                # Should be ok.
+                @thread.next_tick method(:start)
+            end
+
+            # Access to other modules in the same control system
+            def system
+                @system ||= ::Orchestrator::Core::SystemProxy.new(@thread, @settings.control_system_id)
+            end
+        end
+    end
+end

data/lib/orchestrator/logic/mixin.rb

@@ -0,0 +1,11 @@
+module Orchestrator
+    module Logic
+        module Mixin
+            include ::Orchestrator::Core::Mixin
+
+            def system
+                @__config__.system
+            end
+        end
+    end
+end

data/lib/orchestrator/service/manager.rb

@@ -0,0 +1,63 @@
+module Orchestrator
+    module Service
+        class Manager < ::Orchestrator::Core::ModuleManager
+            def initialize(*args)
+                super(*args)
+
+                # Do we want to start here?
+                # Should be ok.
+                @thread.next_tick method(:start)
+            end
+
+            attr_reader :processor, :connection
+
+            def start
+                @processor = Orchestrator::Device::Processor.new(self)
+
+                super # Calls on load (allows setting of tls certs)
+
+                @connection = TransportHttp.new(self, @processor)
+                @processor.transport = @connection
+            end
+
+            def stop
+                super
+                @processor.terminate
+                @processor = nil
+                @connection.terminate
+                @connection = nil
+            end
+
+            # NOTE:: Same as Device::Manager:-------
+            # TODO:: Need to have a guess about when a device may be off line
+
+            def notify_connected
+                if @instance.respond_to? :connected, true
+                    begin
+                        @instance.__send__(:connected)
+                    rescue => e
+                        @logger.print_error(e, 'error in module connected callback')
+                    end
+                end
+
+                update_connected_status(true)
+            end
+
+            def notify_received(data, resolve, command = nil)
+                begin
+                    blk = command.nil? ? nil : command[:on_receive]
+                    if blk.respond_to? :call
+                        blk.call(data, resolve, command)
+                    elsif @instance.respond_to? :received, true
+                        @instance.__send__(:received, data, resolve, command)
+                    else
+                        @logger.warn('no received function provided')
+                        :abort
+                    end
+                rescue => e
+                    @logger.print_error(e, 'error in received callback')
+                end
+            end
+        end
+    end
+end

data/lib/orchestrator/service/mixin.rb

@@ -0,0 +1,56 @@
+module Orchestrator
+    module Service
+        module Mixin
+            include ::Orchestrator::Core::Mixin
+
+            def request(method, path, options = {}, &blk)
+                defer = @__config__.thread.defer
+                options[:method] = method
+                options[:path] = path
+                options[:defer] = defer
+                options[:max_waits] = 0  # HTTP will only ever respond to a request
+                options[:on_receive] = blk if blk     # on command success
+                @__config__.thread.schedule do
+                    @__config__.processor.queue_command(options)
+                end
+                defer.promise
+            end
+
+            def get(path, options = {}, &blk)
+                request(:get, path, options, &blk)
+            end
+
+            def post(path, options = {}, &blk)
+                request(:post, path, options, &blk)
+            end
+
+            def put(path, options = {}, &blk)
+                request(:put, path, options, &blk)
+            end
+
+            def delete(path, options = {}, &blk)
+                request(:delete, path, options, &blk)
+            end
+
+            def config(options)
+                @__config__.thread.schedule do
+                    @__config__.processor.config = options
+                end
+            end
+
+            def defaults(options)
+                @__config__.thread.schedule do
+                    @__config__.processor.send_options(options)
+                end
+            end
+
+            def clear_cookies
+                # TODO::
+            end
+
+            def use_middleware(klass)
+                # TODO::
+            end
+        end
+    end
+end

data/lib/orchestrator/service/transport_http.rb

@@ -0,0 +1,55 @@
+
+module Orchestrator
+    module Service
+        class TransportHttp
+            def initialize(manager, processor)
+                @manager = manager
+                @settings = @manager.settings
+                @processor = processor
+
+                # Load http endpoint after module has had a chance to update the config
+                config = @processor.config
+                config[:tls] ||= @settings.tls
+                config[:tokenize] = false
+                @server = UV::HttpEndpoint.new @settings.uri, config
+
+                @manager.thread.next_tick do
+                    # Call connected (we only need to do this once)
+                    # We may never be connected, this is just to signal that we are ready
+                    @processor.connected
+                end
+            end
+
+            def transmit(cmd)
+                return if @terminated
+
+                # TODO:: Support multiple simultaneous requests (multiple servers)
+
+                @server.request(cmd[:method], cmd).then(
+                    proc { |result|
+                        # Make sure the request information is always available
+                        result[:request] = cmd
+                        @processor.buffer(result)
+                        nil
+                    },
+                    proc { |failure|
+                        # Fail fast (no point waiting for the timeout)
+                        if @processor.queue.waiting #== cmd
+                            @processor.__send__(:resp_failure, :failed)
+                        end
+
+                        # TODO:: Log failure with more detail
+                        nil
+                    }
+                )
+
+                nil
+            end
+
+            def terminate
+                @terminated = true
+                @server.close_connection(:after_writing)
+            end
+        end
+    end
+end

data/lib/orchestrator/status.rb

@@ -0,0 +1,229 @@
+require 'set'
+
+
+module Orchestrator
+    Subscription = Struct.new(:sys_name, :sys_id, :mod_name, :mod_id, :index, :status, :callback, :on_thread) do
+        def initialize(*args)
+            super(*args)
+
+            @do_callback = method(:do_callback)
+        end
+
+        def notify(update)
+            if update != @last_update
+                @last_update = update
+                on_thread.schedule @do_callback
+            end
+        end
+
+        def value
+            @last_update
+        end
+
+
+        protected
+
+
+        # This is always called on the subscribing thread
+        def do_callback
+            callback.call(self)
+        end
+    end
+
+    class Status
+        def initialize(thread)
+            @thread = thread
+            @controller = ::Orchestrator::Control.instance
+
+            @find_subscription = method(:find_subscription)
+
+            # {:mod_id => {status => Subscriptions}}
+            @subscriptions = {}
+            # {:system_id => Subscriptions}
+            @systems = {}
+        end
+
+
+        attr_reader :thread
+
+
+        # Subscribes to updates from a system module
+        # Modules do not have to exist and updates will be triggered as soon as they are
+        def subscribe(opt)     # sys_name, mod_name, index, status, callback, on_thread
+            if opt[:sys_name] && !opt[:sys_id]
+                @thread.work(proc {
+                    id = ::Orchestrator::ControlSystem.bucket.get("sysname-#{sys_name}")
+                    opt[:sys_id] = id
+
+                    # Grabbing system here as thread-safe and has the potential to block
+                    ::Orchestrator::System.get(id)
+                }).then(proc { |sys|
+                    mod = sys.get(opt[:mod_name], opt[:index] - 1)
+                    if mod
+                        opt[:mod_id] = mod.settings.id.to_sym
+                        opt[:mod] = mod
+                    end
+
+                    do_subscribe(opt)
+                })
+            else
+                do_subscribe(opt)
+            end
+        end
+
+        # Removes subscription callback from the lookup
+        def unsubscribe(sub)
+            if sub.is_a? ::Libuv::Q::Promise 
+                sub.then @find_subscription
+            else
+                find_subscription(sub)
+            end
+        end
+
+        # Triggers an update to be sent to listening callbacks
+        def update(mod_id, status, value)
+            mod = @subscriptions[mod_id]
+            if mod
+                subscribed = mod[status]
+                if subscribed
+                    subscribed.each do |subscription|
+                        begin
+                            subscription.notify(value)
+                        rescue => e
+                            @controller.log_unhandled_exception(e)
+                        end
+                    end
+                end
+            end
+        end
+
+        # TODO:: we also need the system class to contact each of the threads
+        def reloaded_system(sys_id)
+            subscriptions = @systems[sys_id]
+            if subscriptions
+                sys = ::Orchestrator::System.get(@system)
+
+                subscriptions.each do |sub|
+                    old_id = sub.mod_id
+
+                    # re-index the subscription
+                    mod = sys.get(sub.mod_name, sub.index - 1)
+                    sub.mod_id = mod ? mod.settings.id.to_sym : nil
+
+                    # Check for changes (order, removal, replacement)
+                    if old_id != sub.mod_id
+                        @subscriptions[old_id][sub.status].delete(sub)
+
+                        # Update to the new module
+                        if sub.mod_id
+                            @subscriptions[sub.mod_id] ||= {}
+                            @subscriptions[sub.mod_id][sub.status] ||= Set.new
+                            @subscriptions[sub.mod_id][sub.status].add(sub)
+
+                            # Check for existing status to send to subscriber
+                            value = mod.status[sub.status]
+                            sub.notify(value) unless value.nil?
+                        end
+
+                        # Perform any required cleanup
+                        if @subscriptions[old_id][sub.status].empty?
+                            @subscriptions[old_id].delete(sub.status)
+                            if @subscriptions[old_id].empty?
+                                @subscriptions.delete(old_id)
+                            end
+                        end
+                    end
+                end
+            end
+        end
+
+
+        # NOTE:: Only to be called from subscription thread
+        def exec_unsubscribe(sub)
+            # Update the system lookup if a system was specified
+            if sub.sys_id
+                subscriptions = @systems[sub.sys_id]
+                if subscriptions
+                    subscriptions.delete(sub)
+
+                    if subscriptions.empty?
+                        @systems.delete(sub.sys_id)
+                    end
+                end
+            end
+
+            # Update the module lookup
+            statuses = @subscriptions[sub.mod_id]
+            if statuses
+                subscriptions = statuses[sub.status]
+                if subscriptions
+                    subscriptions.delete(sub)
+
+                    if subscriptions.empty?
+                        statuses.delete(sub.status)
+
+                        if statuses.empty?
+                            @subscriptions.delete(sub.mod_id)
+                        end
+                    end
+                end
+            end
+        end
+
+
+        protected
+
+
+        def do_subscribe(opt)
+            # Build the subscription object (as loosely coupled as we can)
+            sub = Subscription.new(opt[:sys_name], opt[:sys_id], opt[:mod_name], opt[:mod_id], opt[:index], opt[:status], opt[:callback], opt[:on_thread])
+
+            if sub.sys_id
+                @systems[sub.sys_id] ||= Set.new
+                @systems[sub.sys_id].add(sub)
+            end
+
+            # Now if the module is added later we'll still receive updates
+            # and also support direct module status bindings
+            if sub.mod_id
+                @subscriptions[sub.mod_id] ||= {}
+                @subscriptions[sub.mod_id][sub.status] ||= Set.new
+                @subscriptions[sub.mod_id][sub.status].add(sub)
+
+                # Check for existing status to send to subscriber
+                value = opt[:mod].status[sub.status]
+                sub.notify(value) unless value.nil?
+            end
+
+            # return the subscription
+            sub
+        end
+
+        def find_subscription(sub)
+            # Find module thread
+            if sub.mod_id
+                manager = @controller.loaded?(sub.mod_id)
+                if manager
+                    thread = manager.thread
+                    thread.schedule do
+                        thread.observer.exec_unsubscribe(sub)
+                    end
+                else
+                    # Should be in our schedule
+                    exec_unsubscribe(sub)
+                end
+            else
+                exec_unsubscribe(sub)
+            end
+        end
+    end
+end
+
+module Libuv
+    class Loop
+        def observer
+            @observer ||= ::Orchestrator::Status.new(@loop)
+            @observer
+        end
+    end
+end