orchestrator 1.0.1

data/app/models/orchestrator/zone.rb

@@ -0,0 +1,47 @@
+
+module Orchestrator
+    class Zone < Couchbase::Model
+        design_document :zone
+        include ::CouchbaseId::Generator
+
+
+        attribute :name
+        attribute :description
+        attribute :settings,    default: lambda { {} }
+
+        attribute :created_at,  default: lambda { Time.now.to_i }
+
+
+
+        # Loads all the zones
+        def self.all
+            all(stale: false)
+        end
+        view :all
+
+
+        protected
+
+
+        validates :name,  presence: true
+
+
+        before_delete :remove_zone
+        def remove_zone
+            ::Orchestrator::Control.instance.zones.delete(self.id)
+            ::Orchestrator::ControlSystem.in_zone(self.id).each do |cs|
+                cs.zones.delete(self.id)
+                cs.save
+            end
+        end
+
+        # Expire both the zone cache and any systems that use the zone
+        after_save :expire_caches
+        def expire_caches
+            ::Orchestrator::Control.instance.zones[self.id] = self
+            ::Orchestrator::ControlSystem.in_zone(self.id).each do |cs|
+                cs.expire_cache
+            end
+        end
+    end
+end

data/app/models/orchestrator/zone/all/map.js

@@ -0,0 +1,6 @@
+
+function(doc, meta) {
+    if(doc.type == "zone") {
+        emit(meta.id, null);
+    }
+}

data/config/routes.rb

@@ -0,0 +1,43 @@
+Orchestrator::Engine.routes.draw do
+
+    match '/*path' => 'api#options', :via => :options
+
+    # Restful access to services
+    namespace :api do
+        # Allows multiple routes to resolve to the one controller
+        concern :mods do
+            resources :modules do # modules have settings
+                post 'start',   on: :member
+                post 'stop',    on: :member
+                get  'state',   on: :member
+            end
+        end
+
+        # Trusted Sessions - Create Trust (returns id), Update Session and Destroy Trust
+        resources :trusts
+
+        resources(:systems, {as: :control_system}) do       # systems have settings and define what zone they are in
+            post 'remove',  on: :member
+            post 'start',   on: :member
+            post 'stop',    on: :member
+            post 'exec',    on: :member
+            get  'state',   on: :member
+            get  'funcs',   on: :member
+            get  'count',   on: :member
+            get  'types',   on: :member
+
+            concerns :mods
+        end
+        resources :dependencies do  # dependencies have settings
+            post 'reload',  on: :member
+        end
+        resources :groups           # users define the groups they are in
+        resources :zones            # zones define what groups can access them
+        
+        concerns  :mods
+    end
+
+    # These are non-restful endpoints
+    # Websockets and Eventsources
+    get 'websocket', to: 'persistence#websocket', via: :all
+end

data/lib/generators/module/USAGE

@@ -0,0 +1,8 @@
+Description:
+    Generates a module scaffold for you
+
+Example:
+    rails generate module manufacturer/category/module_name
+
+    This will create:
+        app/modules/manufacturer/category/module_name.rb

data/lib/generators/module/module_generator.rb

@@ -0,0 +1,52 @@
+class ModuleGenerator < Rails::Generators::NamedBase
+    #source_root File.expand_path('../templates', __FILE__)
+    
+    def create_module_file
+        
+        name = file_name.downcase.gsub(/\s|-/, '_')
+        param = class_path
+        param.map! {|item| item.downcase.gsub(/\s|-/, '_')}
+        
+        path = File.join('app/modules', *param)
+        
+        scope = []
+        text = ""
+        param.map! {|item|
+            item = item.camelcase
+            scope << item
+            text += "module #{scope.join('::')}; end\n"
+            item
+        }
+        param << name.camelcase
+        scope = param.join('::')
+        
+        
+        create_file File.join(path, "#{name}.rb") do            
+            text += <<-FILE
+
+
+class #{scope}
+    include ::Orchestrator::Constants  # On, Off and other useful constants
+    include ::Orchestrator::Transcoder # binary, hex and string helper methods
+    # For stream tokenization use ::UV::BufferedTokenizer or ::UV::AbstractTokenizer
+
+    def on_load
+        # module has been started
+    end
+    
+    def on_unload
+        # module has been stopped
+    end
+    
+    # Called when class updated at runtime
+    def on_update
+    end
+end
+
+            FILE
+            
+            text
+        end
+        
+    end
+end

data/lib/orchestrator.rb

@@ -0,0 +1,52 @@
+require 'orchestrator/engine'
+
+
+# Gems
+require 'couchbase-id'
+require 'uv-rays'
+require 'co-elastic-query'
+
+# Optional utility modules
+require 'orchestrator/utilities/transcoder' # functions for data manipulation
+require 'orchestrator/utilities/constants'  # constants for readable code
+
+# System Main
+require 'orchestrator/dependency_manager'   # Manages code loading
+require 'orchestrator/websocket_manager'    # Websocket interface
+require 'orchestrator/datagram_server'      # UDP abstraction management
+require 'orchestrator/control'              # Module control and system loader
+require 'orchestrator/version'              # orchestrator version
+require 'orchestrator/system'               # This is the source of truth for all system information
+require 'orchestrator/status'               # Manages status subscriptions across threads
+require 'orchestrator/logger'               # Logs events of interest as well as coordinating live log feedback
+require 'orchestrator/errors'               # A list of errors that can occur within the system
+
+# Core Abstractions
+require 'orchestrator/core/module_manager'  # Base class of logic, device and service managers
+require 'orchestrator/core/schedule_proxy'  # Common proxy for all module schedules
+require 'orchestrator/core/requests_proxy'  # Sends a command to all modules of that type
+require 'orchestrator/core/request_proxy'   # Sends a command to a single module
+require 'orchestrator/core/system_proxy'    # prevents stale system objects (maintains loose coupling)
+require 'orchestrator/core/mixin'           # Common mixin functions for modules classes
+
+# Logic abstractions
+require 'orchestrator/logic/manager'        # control system manager for logic modules
+require 'orchestrator/logic/mixin'          # helper functions for logic module classes
+
+# Device abstractions
+require 'orchestrator/device/transport_makebreak'
+require 'orchestrator/device/command_queue'
+require 'orchestrator/device/transport_tcp'
+require 'orchestrator/device/transport_udp'
+require 'orchestrator/device/processor'
+require 'orchestrator/device/manager'
+require 'orchestrator/device/mixin'
+
+# Service abstractions
+require 'orchestrator/service/transport_http'
+require 'orchestrator/service/manager'
+require 'orchestrator/service/mixin'
+
+
+module Orchestrator
+end

data/lib/orchestrator/control.rb

@@ -0,0 +1,303 @@
+require 'set'
+require 'logger'
+
+
+module Orchestrator
+    class Control
+        include Singleton
+
+        #
+        #
+        # 1. Load the modules allocated to this node
+        # 2. Allocate modules to CPUs
+        #    * Modules load dependencies as required
+        #    * Logics are streamed in after devices and services
+        #
+        # Logic modules will fetch their system when they interact with other modules.
+        #  Devices and services do not have a system associated with them
+        # This makes systems very loosely coupled to the modules
+        #  which should make distributing the system slightly simpler
+        #
+        #
+
+        def initialize
+            # critical sections
+            @critical = ::Mutex.new
+            @loaded = ::ThreadSafe::Cache.new
+            @zones = ::ThreadSafe::Cache.new
+            @loader = DependencyManager.instance
+            @loop = ::Libuv::Loop.default
+            @exceptions = method(:log_unhandled_exception)
+
+            @ready = false
+
+            if Rails.env.production?
+                logger = ::Logger.new(::Rails.root.join('log/control.log').to_s, 10, 4194304)
+            else
+                logger = ::Logger.new(STDOUT)
+            end
+            logger.formatter = proc { |severity, datetime, progname, msg|
+                "#{datetime.strftime("%d/%m/%Y @ %I:%M%p")} #{severity}: #{progname} - #{msg}\n"
+            }
+            @logger = ::ActiveSupport::TaggedLogging.new(logger)
+        end
+
+
+        attr_reader :logger, :loop, :ready, :zones
+
+
+        # Start the control reactor
+        def mount
+            return @server.loaded if @server
+
+            @critical.synchronize {
+                return if @server   # Protect against multiple mounts
+
+                # Cache all the zones in the system
+                ::Orchestrator::Zone.all.each do |zone|
+                    @zones[zone.id] = zone
+                end
+
+                @server = ::SpiderGazelle::Spider.instance
+                @server.loaded.then do
+                    # Share threads with SpiderGazelle (one per core)
+                    if @server.mode == :thread
+                        @threads = @server.threads
+                    else    # We are either running no_ipc or process (unsupported for control)
+                        @threads = Set.new
+
+                        cpus = ::Libuv.cpu_count || 1
+                        cpus.times &method(:start_thread)
+
+                        @loop.signal :INT, method(:kill_workers)
+                    end
+
+                    @selector = @threads.cycle
+                end
+            }
+
+            return @server.loaded
+        end
+
+        # Boot the control system, running all defined modules
+        def boot(*args)
+            # Only boot if running as a server
+            Thread.new &method(:load_all)
+        end
+
+        # Load the modules on the loop references in round robin
+        # This method is thread safe.
+        def load(mod_settings)
+            mod_id = mod_settings.id.to_sym
+            defer = @loop.defer
+            mod = @loaded[mod_id]
+
+            if mod
+                defer.resolve(mod)
+            else
+                defer.resolve(
+                    @loader.load(mod_settings.dependency).then(proc { |klass|
+                        # We will always be on the default loop here
+                        thread = @selector.next
+
+                        # We'll resolve the promise if the module loads on the deferred thread
+                        defer = @loop.defer
+                        thread.schedule do
+                            defer.resolve(start_module(thread, klass, mod_settings))
+                        end
+
+                        # update the module cache
+                        defer.promise.then do |mod_manager|
+                            @loaded[mod_id] = mod_manager
+                        end
+                        defer.promise
+                    }, @exceptions)
+                )
+            end
+            defer.promise
+        end
+
+        # Checks if a module with the ID specified is loaded
+        def loaded?(mod_id)
+            @loaded[mod_id.to_sym]
+        end
+
+        # Starts a module running
+        def start(mod_id)
+            defer = @loop.defer
+
+            mod = loaded? mod_id
+            if mod
+                mod.thread.schedule do
+                    mod.start
+                    defer.resolve(true)
+                end
+            else
+                err = Error::ModuleNotFound.new "unable to start module '#{mod_id}', not found"
+                defer.reject(err)
+                @logger.warn err.message
+            end
+
+            defer.promise
+        end
+
+        # Stops a module running
+        def stop(mod_id)
+            defer = @loop.defer
+
+            mod = loaded? mod_id
+            if mod
+                mod.thread.schedule do
+                    mod.stop
+                    defer.resolve(true)
+                end
+            else
+                err = Error::ModuleNotFound.new "unable to stop module '#{mod_id}', not found"
+                defer.reject(err)
+                @logger.warn err.message
+            end
+
+            defer.promise
+        end
+
+        # Stop the module gracefully
+        # Then remove it from @loaded
+        def unload(mod_id)
+            stop(mod_id).then(proc {
+                @loaded.delete(mod_id.to_sym)
+                true # promise response
+            })
+        end
+
+        # Unload then
+        # Get a fresh version of the settings from the database
+        # load the module
+        def update(mod_id)
+            unload(mod_id).then(proc {
+                # Grab database model in the thread pool
+                res = @loop.work do
+                    ::Orchestrator::Module.find(mod_id)
+                end
+
+                # Load the module if model found
+                res.then(proc { |config|
+                    load(config)    # Promise chaining to here
+                })
+            })
+        end
+
+        def reload(dep_id)
+            @loop.work do
+                reload_dep(dep_id)
+            end
+        end
+
+        def notify_ready
+            # Clear the system cache (in case it has been populated at all)
+            System.clear_cache
+            @ready = true
+        end
+
+        def log_unhandled_exception(*args)
+            msg = ''
+            err = args[-1]
+            if err && err.respond_to?(:backtrace)
+                msg << "exception: #{err.message} (#{args[0..-2]})"
+                msg << "\n#{err.backtrace.join("\n")}" if err.respond_to?(:backtrace) && err.backtrace
+            else
+                msg << "unhandled exception: #{args}"
+            end
+            @logger.error msg
+            ::Libuv::Q.reject(@loop, msg)
+        end
+
+
+        protected
+
+
+        # This will always be called on the thread reactor here
+        def start_module(thread, klass, settings)
+            # Initialize the connection / logic / service handler here
+            case settings.dependency.role
+            when :device
+                Device::Manager.new(thread, klass, settings)
+            when :service
+                Service::Manager.new(thread, klass, settings)
+            else
+                Logic::Manager.new(thread, klass, settings)
+            end
+        end
+
+
+        # Grab the modules from the database and load them
+        def load_all
+            loading = []
+            wait = nil
+
+            modules = ::Orchestrator::Module.all
+            modules.each do |mod|
+                if mod.role < 3
+                    loading << load(mod)  # modules are streamed in
+                else
+                    if wait.nil?
+                        wait = ::Libuv::Q.finally(@loop, *loading)
+                        loading.clear
+
+                        # Clear here in case rest api calls have built the cache
+                        System.clear_cache
+                    end
+
+                    loading << mod
+                end
+            end
+
+            # In case there were no logic modules
+            if wait.nil?
+                wait = ::Libuv::Q.finally(@loop, *loading)
+                loading.clear
+            end
+
+            # Mark system as ready
+            wait.finally do
+                continue_loading(loading)
+            end
+        end
+
+        # Load all the logic modules after the device modules are complete
+        def continue_loading(modules)
+            loading = []
+
+            modules.each do |mod|
+                loading << load(mod)  # grab the load promises
+            end
+
+            # Once load is complete we'll accept websockets
+            ::Libuv::Q.finally(@loop, *loading).finally method(:notify_ready)
+        end
+
+
+
+        ##
+        # Methods called when we manage the threads:
+        def start_thread(num)
+            thread = Libuv::Loop.new
+            @threads << thread
+            Thread.new do
+                thread.run do |promise|
+                    promise.progress @exceptions
+
+                    thread.async do
+                        p 'noop'
+                    end
+                end
+            end
+        end
+
+        def kill_workers(*args)
+            @threads.each do |thread|
+                thread.stop
+            end
+            @loop.stop
+        end
+    end
+end