legionio 0.3.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 732846134f79449f2a5c0141e57e40379015403485296374f39263c2dcda4060
-  data.tar.gz: 7867fa05990668a0d4742134ba017fcdc19f5a00a2272f750b07e2810539e516
+  metadata.gz: 054fe7a9c7707f232d0e93719567254ff8b1ca190f0248abc3b8ba1bcba45a84
+  data.tar.gz: 658bd9bb7e565268e15fc768f7b7691eef6c77738ccd4fba1d87b6bf44e9fbb6
 SHA512:
-  metadata.gz: 932ab867c3212b6893b414e35051c90117e38ed0036301a5559f77c38f4574953a127e5194c199cff099d74336fdbaca41e3709887d465d0940a4034c830daae
-  data.tar.gz: b9d6af4416f0261cee4df3d6111eb4ec09b324ae64e7efef6f8f09f76b67db1a9897fe1e82405a80cd82b7f65ef48b4e21ea1628a2cf688ceec6c03750581c76
+  metadata.gz: 98a94510b0408b5f857adaaaa41efe74de4dcc0c509c21d5630001d9aa4a5af16cb2d7ecf5ff0174d310e6f9e159c81948c2e87c44067100d95e106db8b7e390
+  data.tar.gz: 1c21c38abdbfc70e384a09604f88a712e5a1d240409443f7f50a8df75ed4f528f856c1a23b038d89d0f7e7c167d3b645df0ef07378d77ce6b311bbb34d27bd9c

data/.rubocop.yml

@@ -62,7 +62,7 @@ Metrics/CyclomaticComplexity:
     - .debug
     - .trace
 Metrics/PerceivedComplexity:
-  Max: 15
+  Max: 17
   IgnoredMethods:
     - .unknown
     - .fatal

data/Gemfile

@@ -1,3 +1,14 @@
 source 'https://rubygems.org'
 
 gemspec
+
+gem 'lex-conditioner'
+gem 'lex-esphome'
+gem 'lex-health'
+gem 'lex-http'
+gem 'lex-influxdb'
+gem 'lex-lex'
+gem 'lex-log'
+gem 'lex-scheduler', path: '../../lex/lex-scheduler'
+gem 'lex-tasker'
+gem 'lex-transformer'

data/bin/legion

@@ -2,6 +2,7 @@
 # frozen_string_literal: true
 
 require 'optparse'
+require 'etc'
 
 options = { action: :run }
 
@@ -44,7 +45,46 @@ op.on('-v', '--version') { options[:action] = :version }
 op.separator ''
 op.parse!(ARGV)
 
-require File.expand_path('lib/legion.rb') unless options[:action] == :help
-Legion.start
-require File.expand_path('lib/legion/process.rb') unless options[:action] == :help
-Legion::Process.new(options).run!
+@children = []
+children = 5
+if children == 1
+  require File.expand_path('lib/legion.rb') unless options[:action] == :help
+  require File.expand_path('lib/legion/process.rb') unless options[:action] == :help
+  Legion.start
+  Legion::Process.new(options).run!
+else
+  children.times do
+    @children.push(
+      Process.fork do
+        require File.expand_path('lib/legion.rb') unless options[:action] == :help
+        require File.expand_path('lib/legion/process.rb') unless options[:action] == :help
+
+        Legion.start
+        sleep(1)
+        Legion::Process.new(options).run!
+      end
+    )
+    sleep(5)
+  end
+
+  @quit = false
+  trap('SIGINT') do
+    @quit = true
+    @children.each do |child|
+      Process.kill('SIGINT', child)
+    end
+    sleep(5)
+  end
+
+  trap('SIGTERM') do
+    @children.each do |child|
+      Process.kill('SIGTERM', child)
+    end
+
+    sleep(5)
+  end
+
+  sleep(1) until @quit
+end
+
+sleep(2)

data/lib/legion/extensions.rb

@@ -148,6 +148,62 @@ module Legion
 
       def find_extensions
         @extensions ||= {}
+        # puts Gem.install('lex-esphome')
+        # puts Gem::Installer.new('lex-esphome')
+        # test = Gem::Installer.new('lex-esphome')
+        # puts test.install
+        # puts Gem::Dependency.new('lex-esphome')
+        # $LOAD_PATH << "#{ENV['GEM_HOME']}/gems/lex-esphome-0.1.0/lib"
+        # require 'legion/extensions/esphome'
+        # $LOAD_PATH << "#{ENV['GEM_HOME']}/gems/lex-esphome-0.1.0/lib"
+        # ENV['MY_RUBY_HOME'] #/Users/miverso2/.rvm/rubies/ruby-2.7.0
+        # ENV['GEM_HOME'] # /Users/miverso2/.rvm/gems/ruby-2.7.0
+        # require 'rubygems/dependency_installer.rb'
+        # puts Gem::DependencyInstaller.new.install('lex-esphome')
+        # puts Gem::Installer.new('lex-esphome').install
+        # puts Gem::Installer.new('lex-esphome').ensure_loadable_spec
+        # puts Gem.search('lex-esphome')
+        # Gem.clear_paths
+        # require 'legion/extensions/esphome'
+
+        # puts Gem.use_gemdeps
+        # pp Gem.gemdeps
+        # puts 'gems things'
+        # puts Gem.loaded_specs.key? 'lex-esphome'
+        # pp Gem.loaded_specs.keys
+        #
+        # require 'rubygems'
+        # require 'rubygems/command.rb'
+        # require 'rubygems/dependency_installer.rb'
+        #
+        # Gem::Command.build_args = ARGV
+        # inst = Gem::DependencyInstaller.new
+        # puts inst.installed_gems
+        # puts inst.install 'lex-esphome', install_as_default: true
+        # puts inst.installed_gems
+        # puts '....'
+        # test = Gem::DependencyList.new
+        # test.add(inst)
+        # pp test.find_name('lex-esphome')
+        # pp test.find_name('lex-logger')
+        # pp test.specs
+        # pp Gem::DependencyList.from_specs
+        # pp Gem::DependencyList.from_specs.key? 'lex-logg'
+        # pp inst
+        #
+        # pp Gem.new.all_specs
+        # load 'legion/extensions/esphome.rb'
+        # puts Gem.register_default_spec()
+        #  :use_gemdeps,
+        #  :detect_gemdeps,
+        #  :source_date_epoch,
+        #  :gemdeps,
+        # :register_default_spec,
+        #  :find_unresolved_default_spec,
+        #  :clear_default_specs,
+        # :loaded_specs,
+        # :default_path,
+
         Gem::Specification.all_names.each do |gem|
           next unless gem[0..3] == 'lex-'
 
@@ -158,6 +214,10 @@ module Legion
                                   version:         lex[2],
                                   extension_class: "Legion::Extensions::#{lex[1].capitalize}" }
         end
+
+        # Legion::Settings[:extensions].each do |extension|
+        #   puts extension unless @extensions.key? extension
+        # end
       end
     end
   end

data/lib/legion/extensions/actors/subscription.rb

@@ -65,22 +65,26 @@ module Legion
         end
 
         def process_message(message, metadata, delivery_info)
-          payload = if metadata[:content_encoding] && metadata[:content_encoding] == 'encrypted/cs'
-                      Legion::Crypt.decrypt(message, metadata[:headers]['iv'])
-                    elsif metadata[:content_encoding] && metadata[:content_encoding] == 'encrypted/pk'
-                      Legion::Crypt.decrypt_from_keypair(metadata[:headers][:public_key], message)
+          payload = if metadata.content_encoding && metadata.content_encoding == 'encrypted/cs'
+                      Legion::Crypt.decrypt(message, metadata.headers['iv'])
+                    elsif metadata.content_encoding && metadata.content_encoding == 'encrypted/pk'
+                      Legion::Crypt.decrypt_from_keypair(metadata.headers[:public_key], message)
                     else
                       message
                     end
 
-          message = if metadata[:content_type] == 'application/json'
+          message = if metadata.content_type == 'application/json'
                       Legion::JSON.load(payload)
                     else
                       { value: payload }
                     end
           if include_metadata_in_message?
-            message = message.merge(metadata[:headers].transform_keys(&:to_sym))
-            message[:routing_key] = delivery_info[:routing_key]
+            message = message.merge(metadata.headers.transform_keys(&:to_sym)) unless metadata.headers.nil?
+            message[:routing_key] = if Legion::Transport::TYPE == 'march_hare'
+                                      metadata.routing_key
+                                    else
+                                      delivery_info[:routing_key]
+                                    end
           end
 
           message[:timestamp] = (message[:timestamp_in_ms] / 1000).round if message.key?(:timestamp_in_ms) && !message.key?(:timestamp)
@@ -98,11 +102,15 @@ module Legion
         end
 
         def subscribe
-          require 'legion/extensions/tasker/runners/updater'
           sleep(delay_start)
           consumer_tag = "#{Legion::Settings[:client][:name]}_#{lex_name}_#{runner_name}_#{Thread.current.object_id}"
           on_cancellation = block { cancel }
-          @consumer = @queue.subscribe(manual_ack: manual_ack, block: false, consumer_tag: consumer_tag, on_cancellation: on_cancellation) do |delivery_info, metadata, payload|
+
+          @consumer = @queue.subscribe(manual_ack: manual_ack, block: false, consumer_tag: consumer_tag, on_cancellation: on_cancellation) do |*rmq_message|
+            payload = rmq_message.pop
+            metadata = rmq_message.last
+            delivery_info = rmq_message.first
+
             message = process_message(payload, metadata, delivery_info)
             if use_runner?
               Legion::Runner.run(**message,

data/lib/legion/extensions/core.rb

@@ -7,6 +7,8 @@ require_relative 'helpers/task'
 require_relative 'helpers/logger'
 require_relative 'helpers/lex'
 require_relative 'helpers/transport'
+require_relative 'helpers/data'
+require_relative 'helpers/cache'
 
 require_relative 'actors/base'
 require_relative 'actors/every'
@@ -37,11 +39,33 @@ module Legion
         @messages = {}
         build_settings
         build_transport
+        build_data
         build_helpers
         build_runners
         build_actors
       end
 
+      def data_required?
+        false
+      end
+
+      def transport_required?
+        true
+      end
+
+      def cache_required?
+        false
+      end
+
+      def crypt_required?
+        false
+      end
+
+      def build_data
+        auto_generate_data
+        extension_class::Data.build
+      end
+
       def build_transport
         if File.exist? "#{extension_path}/transport/autobuild.rb"
           require "#{extension_path}/transport/autobuild"
@@ -89,6 +113,14 @@ module Legion
 
         Kernel.const_get(lex_class.to_s).const_set('Transport', Module.new { extend Legion::Extensions::Transport })
       end
+
+      def auto_generate_data
+        require 'legion/extensions/data'
+        log.debug 'running meta magic to generate a data base class'
+        return if Kernel.const_defined? "#{lex_class}::Data"
+
+        Kernel.const_get(lex_class.to_s).const_set('Data', Module.new { extend Legion::Extensions::Data })
+      end
     end
   end
 end

data/lib/legion/extensions/data.rb

@@ -1,9 +1,57 @@
+require 'legion/extensions/data/migrator'
+require 'legion/extensions/data/model'
+
 module Legion
   module Extensions
     module Data
+      include Legion::Extensions::Helpers::Data
+      include Legion::Extensions::Helpers::Logger
+
       def build
         @models = []
         @migrations = []
+        if Dir[File.expand_path("#{data_path}/migrations/*.rb")].count.positive?
+          log.debug('Has migrations, checking status')
+          run
+        end
+
+        models = Dir[File.expand_path("#{data_path}/models/*.rb")]
+        if models.count.positive?
+          log.debug('Including LEX models')
+          models.each do |file|
+            require file
+          end
+
+          models_class.constants.select do |model|
+            models_class.const_get(model).extend Legion::Extensions::Data::Model
+          end
+        end
+
+        true
+      end
+
+      def extension_model
+        Legion::Data::Model::Extension[namespace: lex_class.to_s]
+      end
+
+      def schema_version
+        extension_model.values[:schema_version]
+      end
+
+      def migrations_path
+        "#{data_path}/migrations/"
+      end
+
+      def migrate_class
+        @migrate_class ||= Legion::Extensions::Data::Migrator.new(migrations_path, lex_class.to_s)
+      end
+
+      def run
+        return true if migrate_class.is_current?
+
+        log.debug('Running LEX schema migrator')
+        results = migrate_class.run
+        extension_model.update(schema_version: results)
       end
     end
   end

data/lib/legion/extensions/data/migrator.rb

@@ -0,0 +1,28 @@
+require 'sequel/extensions/migration'
+
+module Legion
+  module Extensions
+    module Data
+      class Migrator < Sequel::IntegerMigrator
+        def initialize(path, extension, **)
+          @path = path
+          @extension = extension
+          super(Legion::Data::Connection.sequel, path)
+        end
+
+        def default_schema_column
+          :schema_version
+        end
+
+        def default_schema_table
+          :extensions
+        end
+
+        def schema_dataset
+          Legion::Data::Connection.sequel.from(default_schema_table).where(namespace: @extension)
+        end
+        alias ds schema_dataset
+      end
+    end
+  end
+end

data/lib/legion/extensions/data/model.rb

@@ -0,0 +1,8 @@
+module Legion
+  module Extensions
+    module Data
+      module Model
+      end
+    end
+  end
+end

data/lib/legion/extensions/helpers/cache.rb

@@ -0,0 +1,23 @@
+require 'legion/extensions/helpers/base'
+
+module Legion
+  module Extensions
+    module Helpers
+      module Cache
+        include Legion::Extensions::Helpers::Base
+
+        def cache_namespace
+          @cache_namespace ||= lex_name
+        end
+
+        def cache_set(key, value, ttl: 60, **)
+          Legion::Cache.set(cache_namespace + key, value, ttl: ttl)
+        end
+
+        def cache_get(key)
+          Legion::Cache.get(cache_namespace + key)
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/helpers/data.rb

@@ -0,0 +1,23 @@
+require 'legion/extensions/helpers/base'
+
+module Legion
+  module Extensions
+    module Helpers
+      module Data
+        include Legion::Extensions::Helpers::Base
+
+        def data_path
+          @data_path ||= "#{full_path}/data"
+        end
+
+        def data_class
+          @data_class ||= lex_class::Data
+        end
+
+        def models_class
+          @models_class ||= data_class::Model
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/helpers/lex.rb

@@ -6,13 +6,13 @@ module Legion
         include Legion::Extensions::Helpers::Logger
 
         def self.included(base)
-          base.send :extend, Legion::Extensions::Helpers::Core if base.class == Class
-          base.send :extend, Legion::Extensions::Helpers::Logger if base.class == Class
-          base.extend base if base.class == Module
+          base.send :extend, Legion::Extensions::Helpers::Core if base.instance_of?(Class)
+          base.send :extend, Legion::Extensions::Helpers::Logger if base.instance_of?(Class)
+          base.extend base if base.instance_of?(Module)
         end
 
         def default_settings
-          { logger: { level: 'info' } }
+          { logger: { level: 'info' }, workers: 1 }
         end
       end
     end

data/lib/legion/extensions/helpers/task.rb

@@ -6,13 +6,33 @@ module Legion
   module Extensions
     module Helpers
       module Task
-        def generate_task_log(task_id:, runner_class: to_s, function:, **payload)
+        def generate_task_log(task_id:, function:, runner_class: to_s, **payload)
+          begin
+            if Legion::Settings[:data][:connected]
+              runner_id = Legion::Data::Model::Runner[namespace: runner_class].values[:id]
+              function_id = Legion::Data::Model::Function.where(runner_id: runner_id, name: function).first.values[:id]
+              return true if Legion::Data::Model::TaskLog.insert(task_id: task_id, function_id: function_id, entry: Legion::JSON.dump(payload))
+            end
+          rescue StandardError => e
+            log.warn e.backtrace
+            log.warn("generate_task_log failed, reverting to rmq message, e: #{e.message}")
+          end
           Legion::Transport::Messages::TaskLog.new(task_id: task_id, runner_class: runner_class, function: function, entry: payload).publish
         end
 
         def task_update(task_id, status, **opts)
           return if task_id.nil? || status.nil?
 
+          begin
+            if Legion::Settings[:data][:connected]
+              task = Legion::Data::Model::Task[task_id]
+              task.update(status: status)
+              return true
+            end
+          rescue StandardError => e
+            log.warn("task_update failed, reverting to rmq message, e: #{e.message}")
+          end
+
           update_hash = { task_id: task_id, status: status }
           %i[results payload function_args payload results].each do |column|
             update_hash[column] = opts[column] if opts.key? column

data/lib/legion/extensions/helpers/transport.rb

@@ -36,6 +36,7 @@ module Legion
 
           transport_class::Exchanges.const_set(lex_const, Class.new(Legion::Transport::Exchange))
           @default_exchange = Kernel.const_get(exchange)
+          @default_exchange
         end
       end
     end

data/lib/legion/extensions/transport.rb

@@ -2,6 +2,7 @@ module Legion
   module Extensions
     module Transport
       include Legion::Extensions::Helpers::Transport
+      include Legion::Extensions::Helpers::Logger
 
       attr_accessor :exchanges, :queues, :consumers, :messages
 
@@ -88,12 +89,13 @@ module Legion
       def build_e_to_q(array)
         array.each do |binding|
           binding[:routing_key] = nil unless binding.key? :routing_key
-          binding[:to] = nil unless binding.key? :to
+          binding[:to] = nil unless binding.key?(:to)
+          binding[:from] = default_exchange if !binding.key?(:from) || binding[:from].nil?
           bind_e_to_q(**binding)
         end
       end
 
-      def bind_e_to_q(to:, from: default_exchange, routing_key: nil)
+      def bind_e_to_q(to:, from: default_exchange, routing_key: nil, **)
         if from.is_a? String
           from = "#{transport_class}::Exchanges::#{from.split('_').collect(&:capitalize).join}" unless from.include?('::')
           auto_create_exchange(from) unless Object.const_defined? from
@@ -129,9 +131,9 @@ module Legion
         to = to.is_a?(String) ? Kernel.const_get(to).new : to.new
         to.bind(from, routing_key: routing_key)
       rescue StandardError => e
-        Legion::Logging.fatal e.message
-        Legion::Logging.fatal e.backtrace
-        Legion::Logging.fatal({ from: from, to: to, routing_key: routing_key })
+        log.fatal e.message
+        log.fatal e.backtrace
+        log.fatal({ from: from, to: to, routing_key: routing_key })
       end
 
       def e_to_q

data/lib/legion/process.rb

@@ -49,9 +49,7 @@ module Legion
 
       until quit
         sleep(1)
-        if @options.key? :time_limit
-          @quit = true if Time.now - start_time > @options[:time_limit]
-        end
+        @quit = true if @options.key?(:time_limit) && Time.now - start_time > @options[:time_limit]
       end
       Legion::Logging.info('Legion is shutting down!')
       Legion.shutdown

data/lib/legion/service.rb

@@ -62,8 +62,9 @@ module Legion
       Legion::Logging.info('Legion::Service.shutdown was called')
       @shutdown = true
       Legion::Settings[:client][:shutting_down] = true
-      sleep(1)
+      sleep(0.5)
       Legion::Extensions.shutdown
+      sleep(1)
       Legion::Data.shutdown
       Legion::Cache.shutdown
       Legion::Transport::Connection.shutdown

data/lib/legion/version.rb

@@ -1,3 +1,3 @@
 module Legion
-  VERSION = '0.3.0'.freeze
+  VERSION = '0.3.1'.freeze
 end

data/settings/client.json

@@ -8,10 +8,12 @@
   },
   "data": {
     "connection": {
-      "max_connections": 25
+      "max_connections": 100,
+      "preconnect": false
     }
   },
   "crypt": {
+    "cluster_secret": "fkljwelkfjwelkfjwelkjfwe",
     "vault": {
       "read_cluster_secret": false,
       "connected": false,

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: legionio
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.1
 platform: ruby
 authors:
 - Esity
 autorequire:
 bindir: exe
 cert_chain: []
-date: 2020-08-26 00:00:00.000000000 Z
+date: 2020-10-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby-ext
@@ -365,8 +365,12 @@ files:
 - lib/legion/extensions/builders/runners.rb
 - lib/legion/extensions/core.rb
 - lib/legion/extensions/data.rb
+- lib/legion/extensions/data/migrator.rb
+- lib/legion/extensions/data/model.rb
 - lib/legion/extensions/helpers/base.rb
+- lib/legion/extensions/helpers/cache.rb
 - lib/legion/extensions/helpers/core.rb
+- lib/legion/extensions/helpers/data.rb
 - lib/legion/extensions/helpers/lex.rb
 - lib/legion/extensions/helpers/logger.rb
 - lib/legion/extensions/helpers/task.rb