legionio 0.3.5 → 1.2.0

data/lib/legion/extensions/helpers/cache.rb

@@ -0,0 +1,23 @@
+require 'legion/extensions/helpers/base'
+
+module Legion
+  module Extensions
+    module Helpers
+      module Cache
+        include Legion::Extensions::Helpers::Base
+
+        def cache_namespace
+          @cache_namespace ||= lex_name
+        end
+
+        def cache_set(key, value, ttl: 60, **)
+          Legion::Cache.set(cache_namespace + key, value, ttl: ttl)
+        end
+
+        def cache_get(key)
+          Legion::Cache.get(cache_namespace + key)
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/helpers/core.rb

@@ -0,0 +1,41 @@
+require_relative 'base'
+module Legion
+  module Extensions
+    module Helpers
+      module Core
+        include Legion::Extensions::Helpers::Base
+
+        def settings
+          if Legion::Settings[:extensions].key?(lex_filename.to_sym)
+            Legion::Settings[:extensions][lex_filename.to_sym]
+          else
+            { logger: { level: 'info', extended: false, internal: false } }
+          end
+        end
+
+        # looks local, then in crypt, then settings, then cache, then env
+        def find_setting(name, **opts)
+          log.debug ".find_setting(#{name}) called"
+          return opts[name.to_sym] if opts.key? name.to_sym
+
+          string_name = "#{lex_name}_#{name.to_s.downcase}"
+          if Legion::Settings[:crypt][:vault][:connected] && Legion::Crypt.exist?(lex_name)
+            log.debug "looking for #{string_name} in Legion::Crypt"
+            crypt_result = Legion::Crypt.get(lex_name)
+            return crypt_result[name.to_sym] if crypt_result.is_a?(Hash) && crypt_result.key?(name.to_sym)
+          end
+          return settings[name.to_sym] if settings.key? name.to_sym
+
+          if Legion::Settings[:cache][:connected]
+            log.debug "looking for #{string_name} in Legion::Cache"
+            cache_result = Legion::Cache.get(string_name)
+            return cache_result unless cache_result.nil?
+          end
+
+          ENV[string_name] if ENV.key? string_name
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/helpers/data.rb

@@ -0,0 +1,23 @@
+require 'legion/extensions/helpers/base'
+
+module Legion
+  module Extensions
+    module Helpers
+      module Data
+        include Legion::Extensions::Helpers::Base
+
+        def data_path
+          @data_path ||= "#{full_path}/data"
+        end
+
+        def data_class
+          @data_class ||= lex_class::Data
+        end
+
+        def models_class
+          @models_class ||= data_class::Model
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/helpers/lex.rb

@@ -0,0 +1,48 @@
+module Legion
+  module Extensions
+    module Helpers
+      module Lex
+        include Legion::Extensions::Helpers::Core
+        include Legion::Extensions::Helpers::Logger
+
+        def function_example(function, example)
+          function_set(function, :example, example)
+        end
+
+        def function_options(function, options)
+          function_set(function, :options, options)
+        end
+
+        def function_desc(function, desc)
+          function_set(function, :desc, desc)
+        end
+
+        def function_set(function, key, value)
+          unless respond_to? function
+            log.debug "function_#{key} called but function doesn't exist, f: #{function}"
+            return nil
+          end
+          settings[:functions] = {} if settings[:functions].nil?
+          settings[:functions][function] = {} if settings[:functions][function].nil?
+          settings[:functions][function][key] = value
+        end
+
+        def runner_desc(desc)
+          settings[:runners] = {} if settings[:runners].nil?
+          settings[:runners][actor_name.to_sym] = {} if settings[:runners][actor_name.to_sym].nil?
+          settings[:runners][actor_name.to_sym][:desc] = desc
+        end
+
+        def self.included(base)
+          base.send :extend, Legion::Extensions::Helpers::Core if base.instance_of?(Class)
+          base.send :extend, Legion::Extensions::Helpers::Logger if base.instance_of?(Class)
+          base.extend base if base.instance_of?(Module)
+        end
+
+        def default_settings
+          { logger: { level: 'info' }, workers: 1, runners: {}, functions: {} }
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/helpers/logger.rb

@@ -0,0 +1,44 @@
+module Legion
+  module Extensions
+    module Helpers
+      module Logger
+        def log
+          return @log unless @log.nil?
+
+          logger_hash = { lex: lex_filename || nil }
+          logger_hash[:lex] = lex_filename.first if logger_hash[:lex].is_a? Array
+          if respond_to?(:settings) && settings.key?(:logger)
+            logger_hash[:level] = settings[:logger].key?(:level) ? settings[:logger][:level] : 'info'
+            logger_hash[:log_file] = settings[:logger][:log_file] if settings[:logger].key? :log_file
+            logger_hash[:trace] = settings[:logger][:trace] if settings[:logger].key? :trace
+            logger_hash[:extended] = settings[:logger][:extended] if settings[:logger].key? :extended
+          elsif respond_to?(:settings)
+            Legion::Logging.warn Legion::Settings[:extensions][lex_filename.to_sym]
+            Legion::Logging.warn "#{lex_name} has settings but no :logger key"
+          end
+          @log = Legion::Logging::Logger.new(**logger_hash)
+        end
+
+        def handle_exception(exception, task_id: nil, **opts)
+          log.error exception.message + " for task_id: #{task_id} but was logged "
+          log.error exception.backtrace[0..10]
+          log.error opts
+
+          unless task_id.nil?
+            Legion::Transport::Messages::TaskLog.new(
+              task_id:      task_id,
+              runner_class: to_s,
+              entry:        {
+                exception: true,
+                message:   exception.message,
+                **opts
+              }
+            ).publish
+          end
+
+          raise Legion::Exception::HandledTask
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/helpers/task.rb

@@ -0,0 +1,60 @@
+require 'legion/transport'
+require 'legion/transport/messages/task_update'
+require 'legion/transport/messages/task_log'
+
+module Legion
+  module Extensions
+    module Helpers
+      module Task
+        def generate_task_log(task_id:, function:, runner_class: to_s, **payload)
+          begin
+            if Legion::Settings[:data][:connected]
+              runner_id = Legion::Data::Model::Runner[namespace: runner_class].values[:id]
+              function_id = Legion::Data::Model::Function.where(runner_id: runner_id, name: function).first.values[:id]
+              return true if Legion::Data::Model::TaskLog.insert(task_id: task_id, function_id: function_id, entry: Legion::JSON.dump(payload))
+            end
+          rescue StandardError => e
+            log.warn e.backtrace
+            log.warn("generate_task_log failed, reverting to rmq message, e: #{e.message}")
+          end
+          Legion::Transport::Messages::TaskLog.new(task_id: task_id, runner_class: runner_class, function: function, entry: payload).publish
+        end
+
+        def task_update(task_id, status, use_database: true, **opts)
+          return if task_id.nil? || status.nil?
+
+          begin
+            if Legion::Settings[:data][:connected] && use_database
+              task = Legion::Data::Model::Task[task_id]
+              task.update(status: status)
+              return true
+            end
+          rescue StandardError => e
+            log.debug("task_update failed, reverting to rmq message, e: #{e.message}")
+          end
+
+          update_hash = { task_id: task_id, status: status }
+          %i[results payload function_args payload results].each do |column|
+            update_hash[column] = opts[column] if opts.key? column
+          end
+          Legion::Transport::Messages::TaskUpdate.new(**update_hash).publish
+        rescue StandardError => e
+          log.fatal e.message
+          log.fatal e.backtrace
+          raise e
+        end
+
+        def generate_task_id(function_id:, status: 'task.queued', **opts)
+          insert = { status: status, function_id: function_id }
+          insert[:payload] = Legion::JSON.dump(opts[:payload]) if opts.key? :payload
+          insert[:function_args] = Legion::JSON.dump(opts[:args]) if opts.key? :args
+          %i[master_id parent_id relationship_id task_id].each do |column|
+            insert[column] = opts[column] if opts.key? column
+          end
+
+          { success: true, task_id: Legion::Data::Model::Task.insert(insert), **insert }
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/helpers/transport.rb

@@ -0,0 +1,44 @@
+require_relative 'base'
+
+module Legion
+  module Extensions
+    module Helpers
+      module Transport
+        include Legion::Extensions::Helpers::Base
+
+        def transport_path
+          @transport_path ||= "#{full_path}/transport"
+        end
+
+        def transport_class
+          @transport_class ||= lex_class::Transport
+        end
+
+        def messages
+          @messages ||= transport_class::Messages
+        end
+
+        def queues
+          @queues ||= transport_class::Queues
+        end
+
+        def exchanges
+          @exchanges ||= transport_class::Exchanges
+        end
+
+        def default_exchange
+          @default_exchange ||= build_default_exchange
+        end
+
+        def build_default_exchange
+          exchange = "#{transport_class}::Exchanges::#{lex_const}"
+          return Object.const_get(exchange) if transport_class::Exchanges.const_defined? lex_const
+
+          transport_class::Exchanges.const_set(lex_const, Class.new(Legion::Transport::Exchange))
+          @default_exchange = Kernel.const_get(exchange)
+          @default_exchange
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/transport.rb

@@ -0,0 +1,159 @@
+module Legion
+  module Extensions
+    module Transport
+      include Legion::Extensions::Helpers::Transport
+      include Legion::Extensions::Helpers::Logger
+
+      attr_accessor :exchanges, :queues, :consumers, :messages
+
+      def build
+        @queues = []
+        @exchanges = []
+        @messages = []
+        @consumers = []
+        generate_base_modules
+        require_transport_items
+
+        build_e_to_e
+        build_e_to_q(e_to_q)
+        build_e_to_q(additional_e_to_q)
+        auto_create_dlx_exchange
+        auto_create_dlx_queue
+      rescue StandardError => e
+        Legion::Logging.error e.message
+        Legion::Logging.error e.backtrace
+      end
+
+      def generate_base_modules
+        lex_class.const_set('Transport', Module.new) unless lex_class.const_defined?('Transport')
+        %w[Queues Exchanges Messages Consumers].each do |thing|
+          next if transport_class.const_defined? thing
+
+          transport_class.const_set(thing, Module.new)
+        end
+      end
+
+      def require_transport_items
+        { exchanges: @exchanges, queues: @queues, consumers: @consumers, messages: @messages }.each do |item, obj|
+          Dir[File.expand_path("#{transport_path}/#{item}/*.rb")].sort.each do |file|
+            require file
+            file_name = file.to_s.split('/').last.split('.').first
+            obj.push(file_name) unless obj.include?(file_name)
+          end
+        end
+      end
+
+      def auto_create_exchange(exchange, default_exchange = false) # rubocop:disable Style/OptionalBooleanParameter
+        if Object.const_defined? exchange
+          Legion::Logging.warn "#{exchange} is already defined"
+          return
+        end
+        return build_default_exchange if default_exchange
+
+        transport_class::Exchanges.const_set(exchange.split('::').pop, Class.new(Legion::Transport::Exchange) do
+          def exchange_name
+            self.class.ancestors.first.to_s.split('::')[5].downcase
+          end
+        end)
+      end
+
+      def auto_create_queue(queue)
+        if Kernel.const_defined?(queue)
+          Legion::Logging.warn "#{queue} is already defined"
+          return
+        end
+
+        transport_class::Queues.const_set(queue.split('::').last, Class.new(Legion::Transport::Queue))
+      end
+
+      def auto_create_dlx_exchange
+        dlx = if transport_class::Exchanges.const_defined? 'Dlx'
+                transport_class::Exchanges::Dlx
+              else
+                transport_class::Exchanges.const_set('Dlx', Class.new(default_exchange) do
+                  def exchange_name
+                    "#{super}.dlx"
+                  end
+                end)
+              end
+
+        dlx.new
+      end
+
+      def auto_create_dlx_queue
+        return if transport_class::Queues.const_defined?('Dlx')
+
+        special_name = default_exchange.new.exchange_name
+        dlx_queue = Legion::Transport::Queue.new "#{special_name}.dlx", auto_delete: false
+        dlx_queue.bind("#{special_name}.dlx", { routing_key: '#' })
+      end
+
+      def build_e_to_q(array)
+        array.each do |binding|
+          binding[:routing_key] = nil unless binding.key? :routing_key
+          binding[:to] = nil unless binding.key?(:to)
+          binding[:from] = default_exchange if !binding.key?(:from) || binding[:from].nil?
+          bind_e_to_q(**binding)
+        end
+      end
+
+      def bind_e_to_q(to:, from: default_exchange, routing_key: nil, **)
+        if from.is_a? String
+          from = "#{transport_class}::Exchanges::#{from.split('_').collect(&:capitalize).join}" unless from.include?('::')
+          auto_create_exchange(from) unless Object.const_defined? from
+        end
+
+        if to.is_a? String
+          to = "#{transport_class}::Queues::#{to.split('_').collect(&:capitalize).join}" unless to.include?('::')
+          auto_create_queue(to) unless Object.const_defined?(to)
+        end
+
+        routing_key = to.to_s.split('::').last.downcase if routing_key.nil?
+        bind(from, to, routing_key: routing_key)
+      end
+
+      def build_e_to_e
+        e_to_e.each do |binding|
+          if binding[:from].is_a? String
+            binding[:from] = "#{transport_class}::Exchanges::#{binding[:from].capitalize}" unless binding[:from].include?('::')
+            auto_create_exchange(binding[:from]) unless Object.const_defined? binding[:from]
+          end
+
+          if binding[:to].is_a? String
+            binding[:to] = "#{transport_class}::Exchanges::#{binding[:to].capitalize}" unless binding[:to].include?('::')
+            auto_create_exchange(binding[:to]) unless Object.const_defined? binding[:to]
+          end
+
+          bind(binding[:from], binding[:to], binding)
+        end
+      end
+
+      def bind(from, to, routing_key: nil, **_options)
+        from = from.is_a?(String) ? Kernel.const_get(from).new : from.new
+        to = to.is_a?(String) ? Kernel.const_get(to).new : to.new
+        to.bind(from, routing_key: routing_key)
+      rescue StandardError => e
+        log.fatal e.message
+        log.fatal e.backtrace
+        log.fatal({ from: from, to: to, routing_key: routing_key })
+      end
+
+      def e_to_q
+        [] if !@exchanges.count != 1
+        auto = []
+        @queues.each do |queue|
+          auto.push(from: @exchanges.first, to: queue, routing_key: queue)
+        end
+        auto
+      end
+
+      def e_to_e
+        []
+      end
+
+      def additional_e_to_q
+        []
+      end
+    end
+  end
+end