legionio 0.1.1 → 0.2.0

data/lib/legion/extensions/helpers/core.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require_relative 'base'
+module Legion
+  module Extensions
+    module Helpers
+      module Core
+        include Legion::Extensions::Helpers::Base
+
+        def settings
+          if Legion::Settings[:extensions].key?(lex_filename.to_sym)
+            Legion::Settings[:extensions][lex_filename.to_sym]
+          else
+            { logger: { level: 'info', extended: false } }
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/helpers/lex.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module Helpers
+      module Lex
+        include Legion::Extensions::Helpers::Core
+        include Legion::Extensions::Helpers::Logger
+
+        def self.included(base)
+          base.send :extend, Legion::Extensions::Helpers::Core if base.class == Class
+          base.send :extend, Legion::Extensions::Helpers::Logger if base.class == Class
+          base.extend base if base.class == Module
+        end
+
+        def default_settings
+          { logger: { level: 'info' } }
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/helpers/logger.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module Helpers
+      module Logger
+        def log
+          return @log unless @log.nil?
+
+          logger_hash = { lex: lex_filename || nil }
+          logger_hash[:lex] = lex_filename.first if logger_hash[:lex].is_a? Array
+          if respond_to?(:settings) && settings.key?(:logger)
+            logger_hash[:level] = settings[:logger][:level] if settings[:logger].key? :level
+            logger_hash[:level] = settings[:logger][:log_file] if settings[:logger].key? :log_file
+            logger_hash[:trace] = settings[:logger][:trace] if settings[:logger].key? :trace
+            logger_hash[:extended] = settings[:logger][:extended] if settings[:logger].key? :extended
+          elsif respond_to?(:settings)
+            Legion::Logging.warn Legion::Settings[:extensions][lex_filename.to_sym]
+            Legion::Logging.warn "#{lex_name} has settings but no :logger key"
+          else
+            Legion::Logging.warn 'no settings'
+          end
+          @log = Legion::Logging::Logger.new(logger_hash)
+        end
+
+        def handle_exception(exception, task_id: nil, **opts)
+          log.error exception.message + " for task_id: #{task_id} but was logged "
+          log.error exception.backtrace[0..10]
+          log.error opts
+
+          unless task_id.nil?
+            Legion::Transport::Messages::TaskLog.new(
+              task_id:      task_id,
+              runner_class: to_s,
+              entry:        {
+                exception: true,
+                message:   exception.message,
+                **opts
+              }
+            ).publish
+          end
+
+          raise Legion::Exception::HandledTask
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/helpers/task.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require 'legion/transport'
+require 'legion/transport/messages/task_update'
+require 'legion/transport/messages/task_log'
+
+module Legion
+  module Extensions
+    module Helpers
+      module Task
+        def generate_task_log(task_id:, runner_class: to_s, function:, **payload)
+          Legion::Transport::Messages::TaskLog.new(task_id: task_id, runner_class: runner_class, function: function, entry: payload).publish
+        end
+
+        def task_update(task_id, status, **opts)
+          return if task_id.nil? || status.nil?
+
+          update_hash = { task_id: task_id, status: status }
+          %i[results payload function_args payload results].each do |column|
+            update_hash[column] = opts[column] if opts.key? column
+          end
+          Legion::Transport::Messages::TaskUpdate.new(update_hash).publish
+        rescue StandardError => e
+          log.fatal e.message
+          log.fatal e.backtrace
+          raise e
+        end
+
+        def generate_task_id(function_id:, status: 'task.queued', **opts)
+          insert = { status: status, function_id: function_id }
+          insert[:payload] = Legion::JSON.dump(opts[:payload]) if opts.key? :payload
+          insert[:function_args] = Legion::JSON.dump(opts[:args]) if opts.key? :args
+          %i[master_id parent_id relationship_id task_id].each do |column|
+            insert[column] = opts[column] if opts.key? column
+          end
+
+          { success: true, task_id: Legion::Data::Model::Task.insert(insert), **insert }
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/helpers/transport.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require_relative 'base'
+
+module Legion
+  module Extensions
+    module Helpers
+      module Transport
+        include Legion::Extensions::Helpers::Base
+
+        def transport_path
+          @transport_path ||= "#{full_path}/transport"
+        end
+
+        def transport_class
+          @transport_class ||= lex_class::Transport
+        end
+
+        def messages
+          @messages ||= transport_class::Messages
+        end
+
+        def queues
+          @queues ||= transport_class::Queues
+        end
+
+        def exchanges
+          @exchanges ||= transport_class::Exchanges
+        end
+
+        def default_exchange
+          @default_exchange ||= build_default_exchange
+        end
+
+        def build_default_exchange
+          exchange = "#{transport_class}::Exchanges::#{lex_const}"
+          return Object.const_get(exchange) if transport_class::Exchanges.const_defined? lex_const
+
+          transport_class::Exchanges.const_set(lex_const, Class.new(Legion::Transport::Exchange))
+          @default_exchange = Kernel.const_get(exchange)
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/transport.rb

@@ -0,0 +1,156 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module Transport
+      include Legion::Extensions::Helpers::Transport
+
+      attr_accessor :exchanges, :queues, :consumers, :messages
+      def build
+        @queues = []
+        @exchanges = []
+        @messages = []
+        @consumers = []
+        generate_base_modules
+        require_transport_items
+
+        build_e_to_e
+        build_e_to_q(e_to_q)
+        build_e_to_q(additional_e_to_q)
+        auto_create_dlx_exchange
+        auto_create_dlx_queue
+      rescue StandardError => e
+        Legion::Logging.error e.message
+        Legion::Logging.error e.backtrace
+      end
+
+      def generate_base_modules
+        lex_class.const_set('Transport', Module.new) unless lex_class.const_defined?('Transport')
+        %w[Queues Exchanges Messages Consumers].each do |thing|
+          next if transport_class.const_defined? thing
+
+          transport_class.const_set(thing, Module.new)
+        end
+      end
+
+      def require_transport_items
+        { 'exchanges': @exchanges, 'queues': @queues, 'consumers': @consumers, 'messages': @messages }.each do |item, obj|
+          Dir[File.expand_path("#{transport_path}/#{item}/*.rb")].sort.each do |file|
+            require file
+            file_name = file.to_s.split('/').last.split('.').first
+            obj.push(file_name) unless obj.include?(file_name)
+          end
+        end
+      end
+
+      def auto_create_exchange(exchange, default_exchange = false)
+        if Object.const_defined? exchange
+          Legion::Logging.warn "#{exchange} is already defined"
+          return
+        end
+        return build_default_exchange if default_exchange
+
+        transport_class::Exchanges.const_set(exchange.split('::').pop, Class.new(Legion::Transport::Exchange) do
+          def exchange_name
+            self.class.ancestors.first.to_s.split('::')[5].downcase
+          end
+        end)
+      end
+
+      def auto_create_queue(queue)
+        if Kernel.const_defined?(queue)
+          Legion::Logging.warn "#{queue} is already defined"
+          return
+        end
+
+        Kernel.const_set(queue, Class.new(Legion::Transport::Queue))
+      end
+
+      def auto_create_dlx_exchange
+        return if transport_class::Exchanges.const_defined? 'Dlx'
+
+        dlx = transport_class::Exchanges.const_set('Dlx', Class.new(default_exchange) do
+          def exchange_name
+            "#{super}.dlx"
+          end
+        end)
+        dlx.new
+        dlx
+      end
+
+      def auto_create_dlx_queue
+        return if transport_class::Queues.const_defined?('Dlx')
+
+        special_name = default_exchange.new.exchange_name
+        dlx_queue = Legion::Transport::Queue.new "#{special_name}.dlx"
+        dlx_queue.bind("#{special_name}.dlx", { routing_key: '#' })
+      end
+
+      def build_e_to_q(array)
+        array.each do |binding|
+          binding[:routing_key] = nil unless binding.key? :routing_key
+          binding[:to] = nil unless binding.key? :to
+          bind_e_to_q(**binding)
+        end
+      end
+
+      def bind_e_to_q(to:, from: default_exchange, routing_key: nil)
+        if from.is_a? String
+          from = "#{transport_class}::Exchanges::#{from.split('_').collect(&:capitalize).join}" unless from.include?('::')
+          auto_create_exchange(from) unless Object.const_defined? from
+        end
+
+        if to.is_a? String
+          to = "#{transport_class}::Queues::#{to.split('_').collect(&:capitalize).join}" unless to.include?('::')
+          auto_create_queue(to) unless Object.const_defined?(to)
+        end
+
+        routing_key = to.to_s.split('::').last.downcase if routing_key.nil?
+        bind(from, to, routing_key: routing_key)
+      end
+
+      def build_e_to_e
+        e_to_e.each do |binding|
+          if binding[:from].is_a? String
+            binding[:from] = "#{transport_class}::Exchanges::#{binding[:from].capitalize}" unless binding[:from].include?('::')
+            auto_create_exchange(binding[:from]) unless Object.const_defined? binding[:from]
+          end
+
+          if binding[:to].is_a? String
+            binding[:to] = "#{transport_class}::Exchanges::#{binding[:to].capitalize}" unless binding[:to].include?('::')
+            auto_create_exchange(binding[:to]) unless Object.const_defined? binding[:to]
+          end
+
+          bind(binding[:from], binding[:to], binding)
+        end
+      end
+
+      def bind(from, to, routing_key: nil, **_options)
+        from = from.is_a?(String) ? Kernel.const_get(from).new : from.new
+        to = to.is_a?(String) ? Kernel.const_get(to).new : to.new
+        to.bind(from, routing_key: routing_key)
+      rescue StandardError => e
+        Legion::Logging.fatal e.message
+        Legion::Logging.fatal e.backtrace
+        Legion::Logging.fatal({ from: from, to: to, routing_key: routing_key })
+      end
+
+      def e_to_q
+        [] if !@exchanges.count != 1
+        auto = []
+        @queues.each do |queue|
+          auto.push(from: @exchanges.first, to: queue, routing_key: queue)
+        end
+        auto
+      end
+
+      def e_to_e
+        []
+      end
+
+      def additional_e_to_q
+        []
+      end
+    end
+  end
+end

data/lib/legion/process.rb

@@ -42,6 +42,9 @@ module Legion
     end
 
     def run!
+      start_time = Time.now
+      @options[:time_limit] = @options[:time_limit].to_i if @options.key? :time_limit
+      @quit = false
       check_pid
       daemonize if daemonize?
       write_pid
@@ -49,12 +52,14 @@ module Legion
 
       until quit
         sleep(1) # in real life, something productive would happen here
+        if @options.key? :time_limit
+          @quit = true if Time.now - start_time > @options[:time_limit]
+        end
       end
-      # sleep(1)
-      # sleep(1)
       Legion::Logging.info('Legion is shutting down!')
-      # @service.shutdown
+      Legion.shutdown
       Legion::Logging.info('Legion has shutdown. Goodbye!')
+
       exit
     end
 

data/lib/legion/runner.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require 'legion/exceptions/handled_task'
+require_relative 'runner/log'
+require_relative 'runner/status'
+require 'legion/transport'
+require 'legion/transport/messages/check_subtask'
+
+module Legion
+  module Runner
+    def self.run(runner_class:, function:, task_id: nil, args: nil, check_subtask: true, generate_task: true, parent_id: nil, master_id: nil, **opts) # rubocop:disable Metrics/ParameterLists
+      runner_class = Kernel.const_get(runner_class) if runner_class.is_a? String
+
+      if task_id.nil? && generate_task
+        task_gen = Legion::Runner::Status.generate_task_id(
+          function: function,
+           runner_class: runner_class,
+         parent_id: parent_id, master_id: master_id, task_id: task_id, **opts
+        )
+        task_id = task_gen[:task_id] unless task_gen.nil?
+      end
+
+      args = opts if args.nil?
+      args[:task_id] = task_id unless task_id.nil?
+      args[:master_id] = master_id unless master_id.nil?
+      args[:parent_id] = parent_id unless parent_id.nil?
+
+      result = runner_class.send(function, args)
+    rescue Legion::Exception::HandledTask
+      status = 'task.exception'
+      result = { error: {} }
+    rescue StandardError => e
+      runner_class.handle_exception(e,
+                                    **opts,
+                                    runner_class:  runner_class,
+                                    args:          args,
+                                    function:      function,
+                                    task_id:       task_id,
+                                    generate_task: generate_task,
+                                    check_subtask: check_subtask)
+      status = 'task.exception'
+      result = { success: false, status: status, error: { message: e.message, backtrace: e.backtrace } }
+    else
+      status = 'task.completed'
+    ensure
+      Legion::Runner::Status.update(task_id: task_id, status: status) unless task_id.nil?
+      if check_subtask && status == 'task.completed'
+        Legion::Transport::Messages::CheckSubtask.new(runner_class:  runner_class,
+                                                      function:      function,
+                                                      result:        result,
+                                                      original_args: args,
+                                                      **opts).publish
+      end
+      return { success: true, status: status, result: result, task_id: task_id } # rubocop:disable Lint/EnsureReturn
+    end
+  end
+end

data/lib/legion/runner/log.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Legion
+  module Runner
+    module Log
+      def self.exception(exc, **opts)
+        Legion::Logging.error exc.message
+        Legion::Logging.error opts
+      end
+    end
+  end
+end

data/lib/legion/runner/status.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module Legion
+  module Runner
+    module Status
+      def self.update(task_id:, status: 'task.completed', **opts)
+        Legion::Logging.debug "Legion::Runner::Status.update called, #{task_id}, status: #{status}, #{opts}"
+        return if status.nil?
+
+        if Legion::Settings[:data][:connected]
+          update_db(task_id: task_id, status: status, **opts)
+        else
+          update_rmq(task_id: task_id, status: status, **opts)
+        end
+      end
+
+      def self.update_rmq(task_id:, status: 'task.completed', **opts)
+        return if status.nil?
+
+        Legion::Transport::Messages::TaskUpdate.new(task_id: task_id, status: status, **opts).publish
+      rescue StandardError => e
+        Legion::Logging.fatal e.message
+        Legion::Logging.fatal e.backtrace
+        retries ||= 0
+        Legion::Logging.fatal 'Will retry in 3 seconds' if retries < 5
+        sleep(3)
+        retry if (retries += 1) < 5
+      end
+
+      def self.update_db(task_id:, status: 'task.completed', **opts)
+        return if status.nil?
+
+        task = Legion::Data::Model::Task[task_id]
+        task.update(status: status)
+      rescue StandardError => e
+        Legion::Logging.warn e.message
+        Legion::Logging.warn 'Legion::Runner.update_status_db failed, defaulting to rabbitmq'
+        Legion::Logging.warn e.backtrace
+        update_rmq(task_id: task_id, status: status, **opts)
+      end
+
+      def self.generate_task_id(runner_class:, function:, status: 'task.queued', **opts)
+        Legion::Logging.debug "Legion::Runner::Status.generate_task_id called, #{runner_class}, #{function}, status: #{status}, #{opts}"
+        return nil unless Legion::Settings[:data][:connected]
+
+        runner = Legion::Data::Model::Runner.where(namespace: runner_class.to_s.downcase).first
+        return nil if runner.nil?
+
+        function = Legion::Data::Model::Function.where(runner_id: runner.values[:id], name: function).first
+        return nil if function.nil?
+
+        insert = { status: status, function_id: function.values[:id] }
+        insert[:parent_id] = opts[:task_id] if opts.key? :task_id
+        insert[:master_id] = opts[:task_id] if opts.key? :task_id
+        insert[:payload] = Legion::JSON.dump(opts[:payload]) if opts.key? :payload
+
+        %i[function_args master_id parent_id relationship_id].each do |column|
+          next unless opts.key? column
+
+          insert[column] = opts[column].is_a?(Hash) ? Legion::JSON.dump(opts[column]) : opts[column]
+          # insert[column] = opts[column] if opts.key? column
+        end
+
+        { success: true, task_id: Legion::Data::Model::Task.insert(insert), **insert }
+      rescue StandardError => e
+        Legion::Logging.error e.message
+        Legion::Logging.error e.backtrace
+        raise(e)
+      end
+    end
+  end
+end