legionio 0.3.5 → 1.2.0

data/lib/legion/cli/lex/templates/base/spec_helper.rb.erb

@@ -0,0 +1,11 @@
+require 'bundler/setup'
+require 'legion/extensions/<%= config[:lex] %>'
+
+RSpec.configure do |config|
+  config.example_status_persistence_file_path = '.rspec_status'
+  config.disable_monkey_patching!
+
+  config.expect_with :rspec do |c|
+    c.syntax = :expect
+  end
+end

data/lib/legion/cli/lex/templates/base/version.erb

@@ -0,0 +1,7 @@
+module Legion
+  module Extensions
+    module <%= config[:class_name] %>
+      VERSION = '0.1.1'.freeze
+    end
+  end
+end

data/lib/legion/cli/lex/templates/exchange.erb

@@ -0,0 +1,11 @@
+module Legion::Extensions::<%= config[:lex].split('_').collect(&:capitalize).join %>
+  module Transport
+    module Exchanges
+      class <%= config[:name].split('_').collect(&:capitalize).join %> < Legion::Transport::Exchange
+        def exchange_name
+          '<%= config[:name].split('_').collect(&:capitalize).join %>'
+        end
+      end
+    end
+  end
+end

data/lib/legion/cli/lex/templates/message.erb

@@ -0,0 +1,23 @@
+module Legion::Extensions::<%= config[:lex].split('_').collect(&:capitalize).join %>
+  module Transport
+    module Messages
+      class <%= config[:name].split('_').collect(&:capitalize).join %> < Legion::Transport::Message
+        def initialize(payload, status, options = {})
+          @payload = payload
+          @options = options
+          @status = status
+          @routing_key = routing_key
+          validate
+        end
+
+        def routing_key
+          "<%= config[:lex].split('_').collect(&:capitalize).join %>.<%= config[:name].split('_').collect(&:capitalize).join %>"
+        end
+
+        def message(payload = @payload, _options = {})
+          Legion::JSON.dump(payload)
+        end
+      end
+    end
+  end
+end

data/lib/legion/cli/lex/templates/queue.erb

@@ -0,0 +1,12 @@
+module Legion
+  module Extensions
+    module <%= config[:lex].split('_').collect(&:capitalize).join %>
+      module Transport
+        module Queues
+          class <%= config[:name].split('_').collect(&:capitalize).join %> < Legion::Transport::Queue
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/cli/lex/templates/queue_helper.erb

@@ -0,0 +1,24 @@
+module Legion
+  module Transport
+    class Queue
+      def initialize(**opts)
+      end
+
+      def queue_name
+        'test_queue'
+      end
+
+      def acknowledge(delivery_tag)
+        true
+      end
+
+      def reject(delivery_tag, requeue: false, **)
+        true
+      end
+
+      def queue_options
+        Concurrent::Hash.new
+      end
+    end
+  end
+end

data/lib/legion/cli/lex/templates/queue_spec.erb

@@ -0,0 +1,11 @@
+require 'spec_helper'
+require 'queue_helper'
+require 'legion/extensions/<%= config[:lex] %>/transport/queues/<%= config[:name] %>'
+
+RSpec.describe Legion::Extensions::<%= config[:lex].split('_').collect(&:capitalize).join %>::Transport::Queues::<%= config[:name].capitalize %> do
+  it { should be_a Legion::Transport::Queue }
+  it { should respond_to :queue_name }
+  it { should respond_to(:acknowledge).with(1).argument }
+  it { should respond_to(:reject).with(1).argument }
+  it { should respond_to :queue_options }
+end

data/lib/legion/cli/lex/templates/runner.erb

@@ -0,0 +1,11 @@
+module Legion
+  module Extensions
+    module <%= config[:lex].split('_').collect(&:capitalize).join %>
+      module Runners
+        module <%= config[:name].split('_').collect(&:capitalize).join %>
+          extend Legion::Extensions::Helpers::Lex if Legion::Extensions.const_defined? 'Helpers::Lex'
+        end
+      end
+    end
+  end
+end

data/lib/legion/cli/lex/templates/runner_spec.erb

@@ -0,0 +1,11 @@
+require 'spec_helper'
+require 'legion/extensions/<%= config[:lex] %>/runners/<%= config[:name] %>'
+
+RSpec.describe Legion::Extensions::<%= config[:lex].split('_').collect(&:capitalize).join %>::Runners::<%= config[:name].capitalize %> do
+  it { should be_a Module }
+  Legion::Extensions::<%= config[:lex].capitalize %>::Runners::<%= config[:name].split('_').collect(&:capitalize).join %>.extend Legion::Extensions::<%= config[:lex].split('_').collect(&:capitalize).join %>::Runners::<%= config[:name].split('_').collect(&:capitalize).join %>
+
+  describe 'Functions should work with arguments' do
+    let(:test_class) { Class.new { extend Legion::Extensions::<%= config[:lex].split('_').collect(&:capitalize).join %>::Runners::<%= config[:name].split('_').collect(&:capitalize).join %> } }
+  end
+end

data/lib/legion/cli/relationship.rb

@@ -0,0 +1,22 @@
+module Legion
+  class Cli
+    class Relationship < Thor
+      desc 'create', 'creates a new relationship'
+      def create(_name, _type)
+        trigger_id = invoke('legion:cli:function:find', [], internal: true, capture: true) # rubocop:disable Lint/UselessAssignment
+      end
+
+      desc 'activate', 'actives a relationship'
+      def active; end
+
+      desc 'deactivate', 'deactivates a relationship'
+      def deactivate; end
+
+      desc 'modify', 'modify an existing relationship'
+      def modify; end
+
+      desc 'delete', 'deletes a relationship'
+      def delete; end
+    end
+  end
+end

data/lib/legion/cli/task.rb

@@ -0,0 +1,49 @@
+module Legion
+  class Cli
+    class Task < Thor
+      package_name 'Legion'
+
+      option :limit, type: :numeric, required: true, default: 10, desc: 'how many tasks to return'
+      desc 'show', 'show'
+      option :status, type: :string, required: false, desc: 'search for specific status'
+      def show
+        Legion::Service.new(cache: false, crypt: false, extensions: false, log_level: 'error')
+        rows = [%w[id relationship function status]]
+        Legion::Data::Model::Task.limit(options[:limit]).order(:id).reverse_each do |row|
+          rows.push([row.values[:id], row.values[:relationship_id], row.values[:function_id], row.values[:status]])
+        end
+
+        print_table rows
+      end
+
+      desc 'test', 'test'
+      def status(id)
+        Legion::Service.new(cache: false, crypt: false, extensions: false, log_level: 'error')
+        say Legion::Data::Model::Task[id].values
+      end
+
+      desc 'logs', 'logs'
+      option :limit, type: :numeric, required: true, default: 10, desc: 'how many tasks to return'
+      def logs(id)
+        Legion::Service.new(cache: false, crypt: false, extensions: false, log_level: 'error')
+        rows = [%w[id node_id created entry]]
+        Legion::Data::Model::TaskLog.where(task_id: id).limit(options[:limit]).each do |row|
+          rows.push([row.values[:id], row.values[:node_id], row.values[:created], row.values[:entry]])
+        end
+        print_table rows
+      end
+
+      desc 'purge', 'purge'
+      def purge
+        Legion::Service.new(cache: false, crypt: false, extensions: false, log_level: 'error')
+        days = ask 'how many days do you want to keep?', default: 7
+        dataset = Legion::Data::Model::Task.where { created < DateTime.now - days.to_i }
+        yes? "This will delete #{dataset.count} tasks, continue?", :red
+        dataset.delete
+        say 'Done!'
+      end
+
+      default_task :show
+    end
+  end
+end

data/lib/legion/cli/trigger.rb

@@ -0,0 +1,88 @@
+module Legion
+  class Cli
+    class Trigger < Thor
+      desc 'queue', 'used to send a job directly to a worker via Legion::Transport'
+      option :extension, type: :string, required: false, desc: 'extension short name'
+      option :runner, type: :string, required: false, desc: 'runner short name'
+      option :function, type: :string, required: false, desc: 'function short name'
+      option :delay, type: :numeric, default: 0, desc: 'how long to wait before running the task'
+      def queue(*args) # rubocop:disable Metrics/AbcSize,Metrics/CyclomaticComplexity,Metrics/PerceivedComplexity,Metrics/MethodLength
+        Legion::Service.new(cache: false, crypt: false, extensions: false, log_level: 'error')
+        include Legion::Extensions::Helpers::Task
+        response = if options['extension'].is_a? String
+                     options[:extension]
+                   else
+                     ask 'trigger extension?', limited_to: Legion::Data::Model::Extension.map(:name)
+                   end
+        trigger_extension = Legion::Data::Model::Extension.where(name: response).first
+        runners = Legion::Data::Model::Runner.where(extension_id: trigger_extension.values[:id])
+        if runners.count == 1
+          trigger_runner = runners.first
+          say "Auto selecting #{trigger_runner.values[:name]} since it is the only option for runners"
+        else
+          response = options[:runner].is_a?(String) ? options[:runner] : ask('trigger runner?', limited_to: runners.map(:name))
+          trigger_runner = Legion::Data::Model::Runner.where(name: response).where(extension_id: trigger_extension.values[:id]).first
+        end
+
+        functions = Legion::Data::Model::Function.where(runner_id: trigger_runner.values[:id])
+
+        if functions.count == 1
+          trigger_function = functions.first
+          say "Auto selecting #{trigger_function.values[:name]} since it is the only option for functions"
+        else
+          response = if options[:function].is_a?(String)
+                       options[:function]
+                     else
+                       ask('trigger function?',
+                           limited_to: Legion::Data::Model::Function.where(runner_id: trigger_runner.values[:id]).map(:name))
+                     end
+          trigger_function = Legion::Data::Model::Function.where(runner_id: trigger_runner.values[:id]).where(name: response).first
+        end
+        say "#{trigger_runner.values[:namespace]}.#{trigger_function.values[:name]} selected as trigger", :green, :italicized
+        payload = {}
+        auto_opts = {}
+        unless args.count.zero?
+          args.each do |arg|
+            test = arg.split(':')
+            auto_opts[test[0].to_sym] = test[1]
+          end
+        end
+
+        Legion::JSON.load(trigger_function.values[:args]).each do |arg, required|
+          next if %w[args payload opts options].include? arg.to_s
+
+          if auto_opts.key? arg
+            payload[arg.to_sym] = auto_opts[arg]
+            next
+          end
+          response = ask "#{required == 'keyreq' ? '[required]' : '[optional]'} #{arg} value:"
+          if response.empty? && required == 'keyreq'
+            say "Error! #{arg} is required and cannot be empty", :red
+            redo
+          end
+          payload[arg.to_sym] = response unless response.empty?
+        end
+
+        status = options[:delay].zero? ? 'task.queued' : 'task.delayed'
+        task = generate_task_id(function_id: trigger_function.values[:id], status: status, runner_id: trigger_runner.values[:id], args: payload,
+                                delay: options[:delay])
+
+        unless options[:delay].zero?
+          say "Task: #{task[:task_id]} is queued and will be run in #{options[:delay]}s"
+          return true
+        end
+
+        routing_key = "#{trigger_extension.values[:exchange]}.#{trigger_runner.values[:queue]}.#{trigger_function.values[:name]}"
+        exchange = Legion::Transport::Messages::Dynamic.new(function: trigger_function.values[:name], function_id: trigger_function.values[:id],
+                                                            routing_key: routing_key, args: payload)
+        exchange.options[:task_id] = task[:task_id]
+        exchange.publish if options[:delay].zero?
+
+        say "Task: #{task[:task_id]} was queued"
+      end
+      remove_command :generate_task_id
+
+      default_task :queue
+    end
+  end
+end

data/lib/legion/cli/version.rb

@@ -0,0 +1,5 @@
+module Legion
+  class Cli
+    VERSION = '0.2.0'.freeze
+  end
+end

data/lib/legion/extensions.rb

@@ -0,0 +1,229 @@
+require 'legion/extensions/core'
+require 'legion/runner'
+
+module Legion
+  module Extensions
+    class << self
+      def setup
+        hook_extensions
+      end
+
+      def hook_extensions
+        @timer_tasks = []
+        @loop_tasks = []
+        @once_tasks = []
+        @poll_tasks = []
+        @subscription_tasks = []
+        @actors = []
+
+        find_extensions
+        load_extensions
+      end
+
+      def shutdown
+        return nil if @loaded_extensions.nil?
+
+        @subscription_tasks.each do |task|
+          task[:threadpool].shutdown
+          task[:threadpool].kill unless task[:threadpool].wait_for_termination(5)
+        end
+
+        @loop_tasks.each { |task| task[:running_class].cancel if task[:running_class].respond_to?(:cancel) }
+        @once_tasks.each { |task| task[:running_class].cancel if task[:running_class].respond_to?(:cancel) }
+        @timer_tasks.each { |task| task[:running_class].cancel if task[:running_class].respond_to?(:cancel) }
+        @poll_tasks.each { |task| task[:running_class].cancel if task[:running_class].respond_to?(:cancel) }
+
+        Legion::Logging.info 'Successfully shut down all actors'
+      end
+
+      def load_extensions
+        @extensions ||= {}
+        @loaded_extensions ||= []
+        @extensions.each do |extension, values|
+          if values.key(:enabled) && !values[:enabled]
+            Legion::Logging.info "Skipping #{extension} because it's disabled"
+            next
+          end
+
+          if Legion::Settings[:extensions].key?(extension.to_sym) && Legion::Settings[:extensions][extension.to_sym].key?(:enabled) && !Legion::Settings[:extensions][extension.to_sym][:enabled] # rubocop:disable Layout/LineLength
+            next
+          end
+
+          unless load_extension(extension, values)
+            Legion::Logging.warn("#{extension} failed to load")
+            next
+          end
+          @loaded_extensions.push(extension)
+          sleep(0.1)
+        end
+        Legion::Logging.info "#{@extensions.count} extensions loaded with subscription:#{@subscription_tasks.count},every:#{@timer_tasks.count},poll:#{@poll_tasks.count},once:#{@once_tasks.count},loop:#{@loop_tasks.count}"
+      end
+
+      def load_extension(extension, values) # rubocop:disable Metrics/PerceivedComplexity, Metrics/CyclomaticComplexity, Metrics/AbcSize
+        return unless gem_load(values[:gem_name], extension)
+
+        extension = Kernel.const_get(values[:extension_class])
+        extension.extend Legion::Extensions::Core unless extension.singleton_class.included_modules.include? Legion::Extensions::Core
+
+        min_version = Legion::Settings[:extensions][values[:extension_name]][:min_version] || nil
+        Legion::Logging.fatal values if min_version.is_a?(String) && Gem::Version.new(values[:version]) >= Gem::Version.new(min_version)
+
+        if extension.data_required? && Legion::Settings[:data][:connected] == false
+          Legion::Logging.warn "#{values[:extension_name]} requires Legion::Data but isn't enabled, skipping"
+          return false
+        end
+
+        if extension.cache_required? && Legion::Settings[:cache][:connected] == false
+          Legion::Logging.warn "#{values[:extension_name]} requires Legion::Cache but isn't enabled, skipping"
+          return false
+        end
+
+        if extension.crypt_required? && Legion::Settings[:crypt][:cs].nil?
+          Legion::Logging.warn "#{values[:extension_name]} requires Legion::Crypt but isn't ready, skipping"
+          return false
+        end
+
+        if extension.vault_required? && Legion::Settings[:crypt][:vault][:connected] == false
+          Legion::Logging.warn "#{values[:extension_name]} requires Legion::Crypt::Vault but isn't enabled, skipping"
+          return false
+        end
+
+        has_logger = extension.respond_to?(:log)
+        extension.autobuild
+
+        require 'legion/transport/messages/lex_register'
+        Legion::Transport::Messages::LexRegister.new(function: 'save', opts: extension.runners).publish
+
+        if extension.respond_to?(:meta_actors) && extension.meta_actors.is_a?(Array)
+          extension.meta_actors.each do |_key, actor|
+            extension.log.debug("hooking meta actor: #{actor}") if has_logger
+            hook_actor(**actor)
+          end
+        end
+
+        extension.actors.each do |_key, actor|
+          extension.log.debug("hooking literal actor: #{actor}") if has_logger
+          hook_actor(**actor)
+        end
+        extension.log.info "Loaded v#{extension::VERSION}"
+      rescue StandardError => e
+        Legion::Logging.error e.message
+        Legion::Logging.error e.backtrace
+        false
+      end
+
+      def hook_actor(extension:, extension_name:, actor_class:, size: 1, **opts)
+        size = if Legion::Settings[:extensions].key?(extension_name.to_sym) && Legion::Settings[:extensions][extension_name.to_sym].key?(:workers)
+                 Legion::Settings[:extensions][extension_name.to_sym][:workers]
+               elsif size.is_a? Integer
+                 size
+               else
+                 1
+               end
+
+        extension_hash = {
+          extension:       extension,
+          extension_name:  extension_name,
+          actor_class:     actor_class,
+          size:            size,
+          fallback_policy: :abort,
+          **opts
+        }
+        extension_hash[:running_class] = if actor_class.ancestors.include? Legion::Extensions::Actors::Subscription
+                                           actor_class
+                                         else
+                                           actor_class.new
+                                         end
+
+        return if extension_hash[:running_class].respond_to?(:enabled?) && !extension_hash[:running_class].enabled?
+
+        if actor_class.ancestors.include? Legion::Extensions::Actors::Every
+          @timer_tasks.push(extension_hash)
+        elsif actor_class.ancestors.include? Legion::Extensions::Actors::Once
+          @once_tasks.push(extension_hash)
+        elsif actor_class.ancestors.include? Legion::Extensions::Actors::Loop
+          @loop_tasks.push(extension_hash)
+        elsif actor_class.ancestors.include? Legion::Extensions::Actors::Poll
+          @poll_tasks.push(extension_hash)
+        elsif actor_class.ancestors.include? Legion::Extensions::Actors::Subscription
+          extension_hash[:threadpool] = Concurrent::FixedThreadPool.new(size)
+          size.times do
+            extension_hash[:threadpool].post do
+              klass = actor_class.new
+              if klass.respond_to?(:async)
+                klass.async.subscribe
+              else
+                klass.subscribe
+              end
+            end
+          end
+          @subscription_tasks.push(extension_hash)
+        else
+          Legion::Logging.fatal 'did not match any actor classes'
+        end
+      end
+
+      def gem_load(gem_name, name)
+        require "#{Gem::Specification.find_by_name(gem_name).gem_dir}/lib/legion/extensions/#{name}"
+        true
+      rescue LoadError => e
+        Legion::Logging.error e.message
+        Legion::Logging.error e.backtrace
+        Legion::Logging.error "gem_path: #{gem_path}" unless gem_path.nil?
+        false
+      end
+
+      def find_extensions
+        @extensions ||= {}
+        Gem::Specification.all_names.each do |gem|
+          next unless gem[0..3] == 'lex-'
+
+          lex = gem.split('-')
+          @extensions[lex[1]] = { full_gem_name:   gem,
+                                  gem_name:        "lex-#{lex[1]}",
+                                  extension_name:  lex[1],
+                                  version:         lex[2],
+                                  extension_class: "Legion::Extensions::#{lex[1].split('_').collect(&:capitalize).join}" }
+        end
+
+        enabled = 0
+        requested = 0
+
+        Legion::Settings[:extensions].each do |extension, values|
+          next if @extensions.key? extension.to_s
+          next if values[:enabled] == false
+
+          requested += 1
+          next if values[:auto_install] == false
+          next if ENV['_'].include? 'bundle'
+
+          Legion::Logging.warn "#{extension} is missing, attempting to install automatically.."
+          install = Gem.install("lex-#{extension}", values[:version])
+          Legion::Logging.debug(install)
+          lex = Gem::Specification.find_by_name("lex-#{extension}")
+
+          @extensions[extension.to_s] = {
+            full_gem_name:   "lex-#{extension}-#{lex.version}",
+            gem_name:        "lex-#{extension}",
+            extension_name:  extension.to_s,
+            version:         lex.version,
+            extension_class: "Legion::Extensions::#{extension.to_s.split('_').collect(&:capitalize).join}"
+          }
+
+          enabled += 1
+
+        rescue StandardError, Gem::MissingSpecError => e
+          Legion::Logging.error "Failed to auto install #{extension}, e: #{e.message}"
+        end
+        return true if requested == enabled
+
+        Legion::Logging.warn "A total of #{requested - enabled} where skipped"
+        if ENV.key?('_') && ENV['_'].include?('bundle')
+          Legion::Logging.warn 'Please add them to your Gemfile since you are using bundler'
+        else
+          Legion::Logging.warn 'You must have auto_install_missing_lex set to true to auto install missing extensions'
+        end
+      end
+    end
+  end
+end