legionio 0.3.5 → 0.4.0

data/lib/legion/extensions/helpers/logger.rb

@@ -0,0 +1,44 @@
+module Legion
+  module Extensions
+    module Helpers
+      module Logger
+        def log
+          return @log unless @log.nil?
+
+          logger_hash = { lex: lex_filename || nil }
+          logger_hash[:lex] = lex_filename.first if logger_hash[:lex].is_a? Array
+          if respond_to?(:settings) && settings.key?(:logger)
+            logger_hash[:level] = settings[:logger].key?(:level) ? settings[:logger][:level] : 'info'
+            logger_hash[:log_file] = settings[:logger][:log_file] if settings[:logger].key? :log_file
+            logger_hash[:trace] = settings[:logger][:trace] if settings[:logger].key? :trace
+            logger_hash[:extended] = settings[:logger][:extended] if settings[:logger].key? :extended
+          elsif respond_to?(:settings)
+            Legion::Logging.warn Legion::Settings[:extensions][lex_filename.to_sym]
+            Legion::Logging.warn "#{lex_name} has settings but no :logger key"
+          end
+          @log = Legion::Logging::Logger.new(**logger_hash)
+        end
+
+        def handle_exception(exception, task_id: nil, **opts)
+          log.error exception.message + " for task_id: #{task_id} but was logged "
+          log.error exception.backtrace[0..10]
+          log.error opts
+
+          unless task_id.nil?
+            Legion::Transport::Messages::TaskLog.new(
+              task_id:      task_id,
+              runner_class: to_s,
+              entry:        {
+                exception: true,
+                message:   exception.message,
+                **opts
+              }
+            ).publish
+          end
+
+          raise Legion::Exception::HandledTask
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/helpers/task.rb

@@ -0,0 +1,60 @@
+require 'legion/transport'
+require 'legion/transport/messages/task_update'
+require 'legion/transport/messages/task_log'
+
+module Legion
+  module Extensions
+    module Helpers
+      module Task
+        def generate_task_log(task_id:, function:, runner_class: to_s, **payload)
+          begin
+            if Legion::Settings[:data][:connected]
+              runner_id = Legion::Data::Model::Runner[namespace: runner_class].values[:id]
+              function_id = Legion::Data::Model::Function.where(runner_id: runner_id, name: function).first.values[:id]
+              return true if Legion::Data::Model::TaskLog.insert(task_id: task_id, function_id: function_id, entry: Legion::JSON.dump(payload))
+            end
+          rescue StandardError => e
+            log.warn e.backtrace
+            log.warn("generate_task_log failed, reverting to rmq message, e: #{e.message}")
+          end
+          Legion::Transport::Messages::TaskLog.new(task_id: task_id, runner_class: runner_class, function: function, entry: payload).publish
+        end
+
+        def task_update(task_id, status, **opts)
+          return if task_id.nil? || status.nil?
+
+          begin
+            if Legion::Settings[:data][:connected]
+              task = Legion::Data::Model::Task[task_id]
+              task.update(status: status)
+              return true
+            end
+          rescue StandardError => e
+            log.debug("task_update failed, reverting to rmq message, e: #{e.message}")
+          end
+
+          update_hash = { task_id: task_id, status: status }
+          %i[results payload function_args payload results].each do |column|
+            update_hash[column] = opts[column] if opts.key? column
+          end
+          Legion::Transport::Messages::TaskUpdate.new(**update_hash).publish
+        rescue StandardError => e
+          log.fatal e.message
+          log.fatal e.backtrace
+          raise e
+        end
+
+        def generate_task_id(function_id:, status: 'task.queued', **opts)
+          insert = { status: status, function_id: function_id }
+          insert[:payload] = Legion::JSON.dump(opts[:payload]) if opts.key? :payload
+          insert[:function_args] = Legion::JSON.dump(opts[:args]) if opts.key? :args
+          %i[master_id parent_id relationship_id task_id].each do |column|
+            insert[column] = opts[column] if opts.key? column
+          end
+
+          { success: true, task_id: Legion::Data::Model::Task.insert(insert), **insert }
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/helpers/transport.rb

@@ -0,0 +1,44 @@
+require_relative 'base'
+
+module Legion
+  module Extensions
+    module Helpers
+      module Transport
+        include Legion::Extensions::Helpers::Base
+
+        def transport_path
+          @transport_path ||= "#{full_path}/transport"
+        end
+
+        def transport_class
+          @transport_class ||= lex_class::Transport
+        end
+
+        def messages
+          @messages ||= transport_class::Messages
+        end
+
+        def queues
+          @queues ||= transport_class::Queues
+        end
+
+        def exchanges
+          @exchanges ||= transport_class::Exchanges
+        end
+
+        def default_exchange
+          @default_exchange ||= build_default_exchange
+        end
+
+        def build_default_exchange
+          exchange = "#{transport_class}::Exchanges::#{lex_const}"
+          return Object.const_get(exchange) if transport_class::Exchanges.const_defined? lex_const
+
+          transport_class::Exchanges.const_set(lex_const, Class.new(Legion::Transport::Exchange))
+          @default_exchange = Kernel.const_get(exchange)
+          @default_exchange
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/transport.rb

@@ -0,0 +1,159 @@
+module Legion
+  module Extensions
+    module Transport
+      include Legion::Extensions::Helpers::Transport
+      include Legion::Extensions::Helpers::Logger
+
+      attr_accessor :exchanges, :queues, :consumers, :messages
+
+      def build
+        @queues = []
+        @exchanges = []
+        @messages = []
+        @consumers = []
+        generate_base_modules
+        require_transport_items
+
+        build_e_to_e
+        build_e_to_q(e_to_q)
+        build_e_to_q(additional_e_to_q)
+        auto_create_dlx_exchange
+        auto_create_dlx_queue
+      rescue StandardError => e
+        Legion::Logging.error e.message
+        Legion::Logging.error e.backtrace
+      end
+
+      def generate_base_modules
+        lex_class.const_set('Transport', Module.new) unless lex_class.const_defined?('Transport')
+        %w[Queues Exchanges Messages Consumers].each do |thing|
+          next if transport_class.const_defined? thing
+
+          transport_class.const_set(thing, Module.new)
+        end
+      end
+
+      def require_transport_items
+        { 'exchanges': @exchanges, 'queues': @queues, 'consumers': @consumers, 'messages': @messages }.each do |item, obj|
+          Dir[File.expand_path("#{transport_path}/#{item}/*.rb")].sort.each do |file|
+            require file
+            file_name = file.to_s.split('/').last.split('.').first
+            obj.push(file_name) unless obj.include?(file_name)
+          end
+        end
+      end
+
+      def auto_create_exchange(exchange, default_exchange = false) # rubocop:disable Style/OptionalBooleanParameter
+        if Object.const_defined? exchange
+          Legion::Logging.warn "#{exchange} is already defined"
+          return
+        end
+        return build_default_exchange if default_exchange
+
+        transport_class::Exchanges.const_set(exchange.split('::').pop, Class.new(Legion::Transport::Exchange) do
+          def exchange_name
+            self.class.ancestors.first.to_s.split('::')[5].downcase
+          end
+        end)
+      end
+
+      def auto_create_queue(queue)
+        if Kernel.const_defined?(queue)
+          Legion::Logging.warn "#{queue} is already defined"
+          return
+        end
+
+        transport_class::Queues.const_set(queue.split('::').last, Class.new(Legion::Transport::Queue))
+      end
+
+      def auto_create_dlx_exchange
+        dlx = if transport_class::Exchanges.const_defined? 'Dlx'
+                transport_class::Exchanges::Dlx
+              else
+                transport_class::Exchanges.const_set('Dlx', Class.new(default_exchange) do
+                  def exchange_name
+                    "#{super}.dlx"
+                  end
+                end)
+              end
+
+        dlx.new
+      end
+
+      def auto_create_dlx_queue
+        return if transport_class::Queues.const_defined?('Dlx')
+
+        special_name = default_exchange.new.exchange_name
+        dlx_queue = Legion::Transport::Queue.new "#{special_name}.dlx", auto_delete: false
+        dlx_queue.bind("#{special_name}.dlx", { routing_key: '#' })
+      end
+
+      def build_e_to_q(array)
+        array.each do |binding|
+          binding[:routing_key] = nil unless binding.key? :routing_key
+          binding[:to] = nil unless binding.key?(:to)
+          binding[:from] = default_exchange if !binding.key?(:from) || binding[:from].nil?
+          bind_e_to_q(**binding)
+        end
+      end
+
+      def bind_e_to_q(to:, from: default_exchange, routing_key: nil, **)
+        if from.is_a? String
+          from = "#{transport_class}::Exchanges::#{from.split('_').collect(&:capitalize).join}" unless from.include?('::')
+          auto_create_exchange(from) unless Object.const_defined? from
+        end
+
+        if to.is_a? String
+          to = "#{transport_class}::Queues::#{to.split('_').collect(&:capitalize).join}" unless to.include?('::')
+          auto_create_queue(to) unless Object.const_defined?(to)
+        end
+
+        routing_key = to.to_s.split('::').last.downcase if routing_key.nil?
+        bind(from, to, routing_key: routing_key)
+      end
+
+      def build_e_to_e
+        e_to_e.each do |binding|
+          if binding[:from].is_a? String
+            binding[:from] = "#{transport_class}::Exchanges::#{binding[:from].capitalize}" unless binding[:from].include?('::')
+            auto_create_exchange(binding[:from]) unless Object.const_defined? binding[:from]
+          end
+
+          if binding[:to].is_a? String
+            binding[:to] = "#{transport_class}::Exchanges::#{binding[:to].capitalize}" unless binding[:to].include?('::')
+            auto_create_exchange(binding[:to]) unless Object.const_defined? binding[:to]
+          end
+
+          bind(binding[:from], binding[:to], binding)
+        end
+      end
+
+      def bind(from, to, routing_key: nil, **_options)
+        from = from.is_a?(String) ? Kernel.const_get(from).new : from.new
+        to = to.is_a?(String) ? Kernel.const_get(to).new : to.new
+        to.bind(from, routing_key: routing_key)
+      rescue StandardError => e
+        log.fatal e.message
+        log.fatal e.backtrace
+        log.fatal({ from: from, to: to, routing_key: routing_key })
+      end
+
+      def e_to_q
+        [] if !@exchanges.count != 1
+        auto = []
+        @queues.each do |queue|
+          auto.push(from: @exchanges.first, to: queue, routing_key: queue)
+        end
+        auto
+      end
+
+      def e_to_e
+        []
+      end
+
+      def additional_e_to_q
+        []
+      end
+    end
+  end
+end

data/lib/legion/lex.rb

@@ -0,0 +1,89 @@
+require 'thor'
+require 'legion/cli/version'
+require 'legion/cli/lex/actor'
+require 'legion/cli/lex/exchange'
+require 'legion/cli/lex/message'
+require 'legion/cli/lex/queue'
+require 'legion/cli/lex/runner'
+
+module Legion
+  class Cli
+    class LexBuilder < Thor
+      check_unknown_options!
+      include Thor::Actions
+
+      no_commands do
+        def lex
+          Dir.pwd.split('/').last.split('-').last
+        end
+      end
+
+      def self.exit_on_failure?
+        true
+      end
+
+      def self.source_root
+        File.dirname(__FILE__)
+      end
+
+      desc 'actor', 'creates and manages actors'
+      subcommand 'actor', Legion::Cli::Lex::Actor
+
+      desc 'exchange', 'creates and manages exchanges'
+      subcommand 'exchange', Legion::Cli::Lex::Exchange
+
+      desc 'messages', 'creates and manages messages'
+      subcommand 'message',  Legion::Cli::Lex::Message
+
+      desc 'queue', 'creates and manages queues'
+      subcommand 'queue', Legion::Cli::Lex::Queue
+
+      desc 'runner', 'creates and manages runners'
+      subcommand 'runner', Legion::Cli::Lex::Runner
+
+      desc 'version', 'Display Version'
+      map %w[-v --version] => :version
+      def version
+        say "Legion::CLI #{Legion::Cli::VERSION}"
+      end
+
+      method_option rspec: true
+      method_option pipeline: true
+      method_option git_init: true
+      method_option bundle_install: true
+      desc 'create :name', 'creates a new lex'
+      def create(name)
+        if Dir.pwd.include?('lex-')
+          say('already inside a lex_gen, try moving to a different directory', :red)
+          return nil
+        end
+
+        vars = { filename: "lex-#{name}", class_name: name.capitalize, lex: name }
+        filename = vars[:filename]
+        template('cli/lex/templates/base/gemspec.erb', "#{filename}/#{filename}.gemspec", vars)
+        template('cli/lex/templates/base/gemfile.erb', "#{filename}/Gemfile", vars)
+        template('cli/lex/templates/base/gitignore.erb', "#{filename}/.gitignore", vars)
+        template('cli/lex/templates/base/lic.erb', "#{filename}/LICENSE.txt", vars)
+        template('cli/lex/templates/base/rakefile.erb', "#{filename}/Rakefile", vars)
+        template('cli/lex/templates/base/rubocop.yml.erb', "#{filename}/.rubocop.yml", vars)
+        template('cli/lex/templates/base/readme.md.erb', "#{filename}/README.md", **vars)
+        template('cli/lex/templates/base/lex.erb', "#{filename}/lib/legion/extensions/#{name}.rb", vars)
+        template('cli/lex/templates/base/version.erb', "#{filename}/lib/legion/extensions/#{name}/version.rb", vars)
+        template('cli/lex/templates/base/bitbucket.yml.erb', "#{filename}/bitbucket-pipelines.yml", vars) if options[:pipeline]
+        template('cli/lex/templates/base/spec_helper.rb.erb', "#{filename}/spec/spec_helper.rb", vars)
+        template('cli/lex/templates/base/lex_spec.erb', "#{filename}/spec/legion/#{name}_spec.rb", vars)
+
+        return if !options[:git_init] && !options[:bundle_install]
+
+        run("cd lex_gen-#{filename}")
+        if options[:git_init]
+          run('git init')
+          run('git add .')
+          run('git commit -m \'Initial commit\'')
+        end
+
+        run('bundle update') if options[:bundle_install]
+      end
+    end
+  end
+end

data/lib/legion/process.rb

@@ -0,0 +1,124 @@
+require 'fileutils'
+
+module Legion
+  class Process
+    def self.run!(options)
+      Legion::Process.new(options).run!
+    end
+
+    attr_reader :options, :quit, :service
+
+    def initialize(options)
+      @options = options
+      options[:logfile] = File.expand_path(logfile) if logfile?
+      options[:pidfile] = File.expand_path(pidfile) if pidfile?
+    end
+
+    def daemonize?
+      options[:daemonize]
+    end
+
+    def logfile
+      options[:logfile]
+    end
+
+    def pidfile
+      options[:pidfile]
+    end
+
+    def logfile?
+      !logfile.nil?
+    end
+
+    def pidfile?
+      !pidfile.nil?
+    end
+
+    def info(msg)
+      puts "[#{::Process.pid}] [#{Time.now}] #{msg}"
+    end
+
+    def run!
+      start_time = Time.now
+      @options[:time_limit] = @options[:time_limit].to_i if @options.key? :time_limit
+      @quit = false
+      check_pid
+      daemonize if daemonize?
+      write_pid
+      trap_signals
+
+      until quit
+        sleep(1)
+        @quit = true if @options.key?(:time_limit) && Time.now - start_time > @options[:time_limit]
+      end
+      Legion::Logging.info('Legion is shutting down!')
+      Legion.shutdown
+      Legion::Logging.info('Legion has shutdown. Goodbye!')
+
+      exit
+    end
+
+    #==========================================================================
+    # DAEMONIZING, PID MANAGEMENT, and OUTPUT REDIRECTION
+    #==========================================================================
+
+    def daemonize
+      exit if fork
+      ::Process.setsid
+      exit if fork
+      Dir.chdir '/'
+    end
+
+    def write_pid
+      if pidfile?
+        begin
+          File.open(pidfile, ::File::CREAT | ::File::EXCL | ::File::WRONLY) { |f| f.write(::Process.pid.to_s) }
+          at_exit { File.delete(pidfile) if File.exist?(pidfile) }
+        rescue Errno::EEXIST
+          check_pid
+          retry
+        end
+      end
+      false
+    end
+
+    def check_pid
+      if pidfile?
+        case pid_status(pidfile)
+        when :running, :not_owned
+          exit(1)
+        when :dead
+          File.delete(pidfile)
+        end
+      end
+      false
+    end
+
+    def pid_status(pidfile)
+      return :exited unless File.exist?(pidfile)
+
+      pid = ::File.read(pidfile).to_i
+      return :dead if pid.zero?
+
+      ::Process.kill(0, pid)
+      :running
+    rescue Errno::ESRCH
+      :dead
+    rescue Errno::EPERM
+      :not_owned
+    end
+
+    def trap_signals
+      trap('SIGTERM') do
+        info 'sigterm'
+      end
+
+      trap('SIGHUP') do
+        info 'sithup'
+      end
+      trap('SIGINT') do
+        @quit = true
+      end
+    end
+  end
+end