legionio 0.3.5 → 1.2.0

data/lib/legion/lex.rb

@@ -0,0 +1,89 @@
+require 'thor'
+require 'legion/cli/version'
+require 'legion/cli/lex/actor'
+require 'legion/cli/lex/exchange'
+require 'legion/cli/lex/message'
+require 'legion/cli/lex/queue'
+require 'legion/cli/lex/runner'
+
+module Legion
+  class Cli
+    class LexBuilder < Thor
+      check_unknown_options!
+      include Thor::Actions
+
+      no_commands do
+        def lex
+          Dir.pwd.split('/').last.split('-').last
+        end
+      end
+
+      def self.exit_on_failure?
+        true
+      end
+
+      def self.source_root
+        File.dirname(__FILE__)
+      end
+
+      desc 'actor', 'creates and manages actors'
+      subcommand 'actor', Legion::Cli::Lex::Actor
+
+      desc 'exchange', 'creates and manages exchanges'
+      subcommand 'exchange', Legion::Cli::Lex::Exchange
+
+      desc 'messages', 'creates and manages messages'
+      subcommand 'message',  Legion::Cli::Lex::Message
+
+      desc 'queue', 'creates and manages queues'
+      subcommand 'queue', Legion::Cli::Lex::Queue
+
+      desc 'runner', 'creates and manages runners'
+      subcommand 'runner', Legion::Cli::Lex::Runner
+
+      desc 'version', 'Display Version'
+      map %w[-v --version] => :version
+      def version
+        say "Legion::CLI #{Legion::Cli::VERSION}"
+      end
+
+      method_option rspec: true
+      method_option pipeline: true
+      method_option git_init: true
+      method_option bundle_install: true
+      desc 'create :name', 'creates a new lex'
+      def create(name)
+        if Dir.pwd.include?('lex-')
+          say('already inside a lex_gen, try moving to a different directory', :red)
+          return nil
+        end
+
+        vars = { filename: "lex-#{name}", class_name: name.capitalize, lex: name }
+        filename = vars[:filename]
+        template('cli/lex/templates/base/gemspec.erb', "#{filename}/#{filename}.gemspec", vars)
+        template('cli/lex/templates/base/gemfile.erb', "#{filename}/Gemfile", vars)
+        template('cli/lex/templates/base/gitignore.erb', "#{filename}/.gitignore", vars)
+        template('cli/lex/templates/base/lic.erb', "#{filename}/LICENSE.txt", vars)
+        template('cli/lex/templates/base/rakefile.erb', "#{filename}/Rakefile", vars)
+        template('cli/lex/templates/base/rubocop.yml.erb', "#{filename}/.rubocop.yml", vars)
+        template('cli/lex/templates/base/readme.md.erb', "#{filename}/README.md", **vars)
+        template('cli/lex/templates/base/lex.erb', "#{filename}/lib/legion/extensions/#{name}.rb", vars)
+        template('cli/lex/templates/base/version.erb', "#{filename}/lib/legion/extensions/#{name}/version.rb", vars)
+        template('cli/lex/templates/base/bitbucket.yml.erb', "#{filename}/bitbucket-pipelines.yml", vars) if options[:pipeline]
+        template('cli/lex/templates/base/spec_helper.rb.erb', "#{filename}/spec/spec_helper.rb", vars)
+        template('cli/lex/templates/base/lex_spec.erb', "#{filename}/spec/legion/#{name}_spec.rb", vars)
+
+        return if !options[:git_init] && !options[:bundle_install]
+
+        run("cd lex_gen-#{filename}")
+        if options[:git_init]
+          run('git init')
+          run('git add .')
+          run('git commit -m \'Initial commit\'')
+        end
+
+        run('bundle update') if options[:bundle_install]
+      end
+    end
+  end
+end

data/lib/legion/process.rb

@@ -0,0 +1,124 @@
+require 'fileutils'
+
+module Legion
+  class Process
+    def self.run!(options)
+      Legion::Process.new(options).run!
+    end
+
+    attr_reader :options, :quit, :service
+
+    def initialize(options)
+      @options = options
+      options[:logfile] = File.expand_path(logfile) if logfile?
+      options[:pidfile] = File.expand_path(pidfile) if pidfile?
+    end
+
+    def daemonize?
+      options[:daemonize]
+    end
+
+    def logfile
+      options[:logfile]
+    end
+
+    def pidfile
+      options[:pidfile]
+    end
+
+    def logfile?
+      !logfile.nil?
+    end
+
+    def pidfile?
+      !pidfile.nil?
+    end
+
+    def info(msg)
+      puts "[#{::Process.pid}] [#{Time.now}] #{msg}"
+    end
+
+    def run!
+      start_time = Time.now
+      @options[:time_limit] = @options[:time_limit].to_i if @options.key? :time_limit
+      @quit = false
+      check_pid
+      daemonize if daemonize?
+      write_pid
+      trap_signals
+
+      until quit
+        sleep(1)
+        @quit = true if @options.key?(:time_limit) && Time.now - start_time > @options[:time_limit]
+      end
+      Legion::Logging.info('Legion is shutting down!')
+      Legion.shutdown
+      Legion::Logging.info('Legion has shutdown. Goodbye!')
+
+      exit
+    end
+
+    #==========================================================================
+    # DAEMONIZING, PID MANAGEMENT, and OUTPUT REDIRECTION
+    #==========================================================================
+
+    def daemonize
+      exit if fork
+      ::Process.setsid
+      exit if fork
+      Dir.chdir '/'
+    end
+
+    def write_pid
+      if pidfile?
+        begin
+          File.open(pidfile, ::File::CREAT | ::File::EXCL | ::File::WRONLY) { |f| f.write(::Process.pid.to_s) }
+          at_exit { File.delete(pidfile) if File.exist?(pidfile) }
+        rescue Errno::EEXIST
+          check_pid
+          retry
+        end
+      end
+      false
+    end
+
+    def check_pid
+      if pidfile?
+        case pid_status(pidfile)
+        when :running, :not_owned
+          exit(1)
+        when :dead
+          File.delete(pidfile)
+        end
+      end
+      false
+    end
+
+    def pid_status(pidfile)
+      return :exited unless File.exist?(pidfile)
+
+      pid = ::File.read(pidfile).to_i
+      return :dead if pid.zero?
+
+      ::Process.kill(0, pid)
+      :running
+    rescue Errno::ESRCH
+      :dead
+    rescue Errno::EPERM
+      :not_owned
+    end
+
+    def trap_signals
+      trap('SIGTERM') do
+        info 'sigterm'
+      end
+
+      trap('SIGHUP') do
+        info 'sithup'
+      end
+      trap('SIGINT') do
+        @quit = true
+      end
+    end
+  end
+end

data/lib/legion/runner.rb

@@ -0,0 +1,57 @@
+require_relative 'runner/log'
+require_relative 'runner/status'
+require 'legion/transport'
+require 'legion/transport/messages/check_subtask'
+
+module Legion
+  module Runner
+    def self.run(runner_class:, function:, task_id: nil, args: nil, check_subtask: true, generate_task: true, parent_id: nil, master_id: nil, catch_exceptions: false, **opts) # rubocop:disable Layout/LineLength, Metrics/CyclomaticComplexity, Metrics/ParameterLists
+      runner_class = Kernel.const_get(runner_class) if runner_class.is_a? String
+
+      if task_id.nil? && generate_task
+        task_gen = Legion::Runner::Status.generate_task_id(
+          function: function,
+          runner_class: runner_class,
+          parent_id: parent_id, master_id: master_id, task_id: task_id, **opts
+        )
+        task_id = task_gen[:task_id] unless task_gen.nil?
+      end
+
+      args = opts if args.nil?
+      args[:task_id] = task_id unless task_id.nil?
+      args[:master_id] = master_id unless master_id.nil?
+      args[:parent_id] = parent_id unless parent_id.nil?
+
+      # result = Fiber.new { Fiber.yield runner_class.send(function, **args) }
+      raise 'No Function defined' if function.nil?
+
+      result = runner_class.send(function, **args)
+    rescue Legion::Exception::HandledTask
+      status = 'task.exception'
+      result = { error: {} }
+    rescue StandardError => e
+      runner_class.handle_exception(e,
+                                    **opts,
+                                    runner_class:  runner_class,
+                                    args:          args,
+                                    function:      function,
+                                    task_id:       task_id,
+                                    generate_task: generate_task,
+                                    check_subtask: check_subtask)
+      status = 'task.exception'
+      result = { success: false, status: status, error: { message: e.message, backtrace: e.backtrace } }
+      raise e unless catch_exceptions
+    ensure
+      status = 'task.completed' if status.nil?
+      Legion::Runner::Status.update(task_id: task_id, status: status) unless task_id.nil?
+      if check_subtask && status == 'task.completed'
+        Legion::Transport::Messages::CheckSubtask.new(runner_class:  runner_class,
+                                                      function:      function,
+                                                      result:        result,
+                                                      original_args: args,
+                                                      **opts).publish
+      end
+      return { success: true, status: status, result: result, task_id: task_id } # rubocop:disable Lint/EnsureReturn
+    end
+  end
+end

data/lib/legion/runner/log.rb

@@ -0,0 +1,10 @@
+module Legion
+  module Runner
+    module Log
+      def self.exception(exc, **opts)
+        Legion::Logging.error exc.message
+        Legion::Logging.error opts
+      end
+    end
+  end
+end

data/lib/legion/runner/status.rb

@@ -0,0 +1,69 @@
+module Legion
+  module Runner
+    module Status
+      def self.update(task_id:, status: 'task.completed', **opts)
+        Legion::Logging.debug "Legion::Runner::Status.update called, #{task_id}, status: #{status}, #{opts}"
+        return if status.nil?
+
+        if Legion::Settings[:data][:connected]
+          update_db(task_id: task_id, status: status, **opts)
+        else
+          update_rmq(task_id: task_id, status: status, **opts)
+        end
+      end
+
+      def self.update_rmq(task_id:, status: 'task.completed', **opts)
+        return if status.nil?
+
+        Legion::Transport::Messages::TaskUpdate.new(task_id: task_id, status: status, **opts).publish
+      rescue StandardError => e
+        Legion::Logging.fatal e.message
+        Legion::Logging.fatal e.backtrace
+        retries ||= 0
+        Legion::Logging.fatal 'Will retry in 3 seconds' if retries < 5
+        sleep(3)
+        retry if (retries += 1) < 5
+      end
+
+      def self.update_db(task_id:, status: 'task.completed', **opts)
+        return if status.nil?
+
+        task = Legion::Data::Model::Task[task_id]
+        task.update(status: status)
+      rescue StandardError => e
+        Legion::Logging.warn e.message
+        Legion::Logging.warn 'Legion::Runner.update_status_db failed, defaulting to rabbitmq'
+        Legion::Logging.warn e.backtrace
+        update_rmq(task_id: task_id, status: status, **opts)
+      end
+
+      def self.generate_task_id(runner_class:, function:, status: 'task.queued', **opts)
+        Legion::Logging.debug "Legion::Runner::Status.generate_task_id called, #{runner_class}, #{function}, status: #{status}, #{opts}"
+        return nil unless Legion::Settings[:data][:connected]
+
+        runner = Legion::Data::Model::Runner.where(namespace: runner_class.to_s.downcase).first
+        return nil if runner.nil?
+
+        function = Legion::Data::Model::Function.where(runner_id: runner.values[:id], name: function).first
+        return nil if function.nil?
+
+        insert = { status: status, function_id: function.values[:id] }
+        insert[:parent_id] = opts[:task_id] if opts.key? :task_id
+        insert[:master_id] = opts[:task_id] if opts.key? :task_id
+        insert[:payload] = Legion::JSON.dump(opts[:payload]) if opts.key? :payload
+
+        %i[function_args master_id parent_id relationship_id].each do |column|
+          next unless opts.key? column
+
+          insert[column] = opts[column].is_a?(Hash) ? Legion::JSON.dump(opts[column]) : opts[column]
+        end
+
+        { success: true, task_id: Legion::Data::Model::Task.insert(insert), **insert }
+      rescue StandardError => e
+        Legion::Logging.error e.message
+        Legion::Logging.error e.backtrace
+        raise(e)
+      end
+    end
+  end
+end

data/lib/legion/service.rb

@@ -0,0 +1,130 @@
+module Legion
+  class Service
+    def modules
+      [Legion::Crypt, Legion::Transport, Legion::Cache, Legion::Data, Legion::Supervision].freeze
+    end
+
+    def initialize(transport: true, cache: true, data: true, supervision: true, extensions: true, crypt: true, log_level: 'info') # rubocop:disable Metrics/ParameterLists
+      setup_logging(log_level: log_level)
+      Legion::Logging.debug('Starting Legion::Service')
+      setup_settings
+      Legion::Logging.info("node name: #{Legion::Settings[:client][:name]}")
+
+      if crypt
+        require 'legion/crypt'
+        Legion::Crypt.start
+      end
+
+      setup_transport if transport
+
+      require 'legion/cache' if cache
+
+      setup_data if data
+      setup_supervision if supervision
+      require 'legion/runner'
+      load_extensions if extensions
+
+      Legion::Crypt.cs if crypt
+      Legion::Settings[:client][:ready] = true
+    end
+
+    def setup_data
+      if RUBY_ENGINE == 'truffleruby'
+        Legion::Logging.error 'Legion::Data does not support truffleruby, please use MRI for any LEX that require it '
+        Legion::Settings[:data][:connected] = false
+        return false
+      end
+
+      require 'legion/data'
+      Legion::Settings.merge_settings(:data, Legion::Data::Settings.default)
+      Legion::Data.setup
+    rescue LoadError
+      Legion::Logging.info 'Legion::Data gem is not installed, please install it manually with gem install legion-data'
+    rescue StandardError => e
+      Legion::Logging.warn "Legion::Data failed to load, starting without it. e: #{e.message}"
+    end
+
+    # noinspection RubyArgCount
+    def default_paths
+      [
+        '/etc/legionio',
+        "#{ENV['home']}/legionio",
+        '~/legionio',
+        './settings'
+      ]
+    end
+
+    def setup_settings(default_dir = __dir__)
+      require 'legion/settings'
+      config_directory = default_dir
+      default_paths.each do |path|
+        next unless Dir.exist? path
+
+        Legion::Logging.info "Using #{path} for settings"
+        config_directory = path
+        break
+      end
+
+      Legion::Logging.info "Using directory #{config_directory} for settings"
+      Legion::Settings.load(config_dir: config_directory)
+      Legion::Logging.info('Legion::Settings Loaded')
+    end
+
+    def setup_logging(log_level: 'info', **_opts)
+      require 'legion/logging'
+      Legion::Logging.setup(log_level: log_level, level: log_level, trace: true)
+    end
+
+    def setup_transport
+      require 'legion/transport'
+      Legion::Settings.merge_settings('transport', Legion::Transport::Settings.default)
+      Legion::Transport::Connection.setup
+    end
+
+    def setup_supervision
+      require 'legion/supervision'
+      @supervision = Legion::Supervision.setup
+    end
+
+    def shutdown
+      Legion::Logging.info('Legion::Service.shutdown was called')
+      @shutdown = true
+      Legion::Settings[:client][:shutting_down] = true
+      sleep(0.5)
+      Legion::Extensions.shutdown
+      sleep(1)
+      Legion::Data.shutdown if Legion::Settings[:data][:connected]
+      Legion::Cache.shutdown
+      Legion::Transport::Connection.shutdown
+      Legion::Crypt.shutdown
+    end
+
+    def reload
+      Legion::Logging.info 'Legion::Service.reload was called'
+      Legion::Extensions.shutdown
+      sleep(1)
+      Legion::Data.shutdown
+      Legion::Cache.shutdown
+      Legion::Transport::Connection.shutdown
+      Legion::Crypt.shutdown
+      Legion::Settings[:client][:ready] = false
+
+      sleep(5)
+      setup_settings
+      Legion::Crypt.start
+      setup_transport
+      setup_data
+      setup_supervision
+      load_extensions
+
+      Legion::Crypt.cs
+      Legion::Settings[:client][:ready] = true
+      Legion::Logging.info 'Legion has been reloaded'
+    end
+
+    def load_extensions
+      require 'legion/runner'
+      Legion::Extensions.hook_extensions
+    end
+  end
+end