lex-tasker 0.1.0 → 0.2.1

data/lib/legion/extensions/tasker/actors/log.rb

@@ -0,0 +1,13 @@
+module Legion::Extensions::Tasker
+  module Actor
+    class Log < Legion::Extensions::Actors::Subscription
+      def check_subtask?
+        false
+      end
+
+      def generate_task?
+        false
+      end
+    end
+  end
+end

data/lib/legion/extensions/tasker/actors/task_manager.rb

@@ -0,0 +1,21 @@
+module Legion::Extensions::Tasker
+  module Actor
+    class TaskManager < Legion::Extensions::Actors::Subscription
+      def use_runner?
+        true
+      end
+
+      def check_subtask?
+        true
+      end
+
+      def generate_task?
+        true
+      end
+
+      def prefetch
+        1
+      end
+    end
+  end
+end

data/lib/legion/extensions/tasker/actors/updater.rb

@@ -0,0 +1,21 @@
+module Legion::Extensions::Tasker
+  module Actor
+    class Updater < Legion::Extensions::Actors::Subscription
+      def runner_function
+        'update_status'
+      end
+
+      def check_subtask?
+        false
+      end
+
+      def generate_task?
+        false
+      end
+
+      def use_runner?
+        false
+      end
+    end
+  end
+end

data/lib/legion/extensions/tasker/helpers/base.rb

@@ -0,0 +1,11 @@
+module Legion
+  module Extensions
+    module Tasker
+      module Helpers
+        module Base
+          def send_task(**opts); end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/tasker/helpers/fetch_delayed.rb

@@ -0,0 +1,66 @@
+module Legion
+  module Extensions
+    module Tasker
+      module Helpers
+        module FetchDelayed
+          def find_trigger(runner_class:, function:, **)
+            sql = "SELECT `functions`.`id` as `function_id`, `runner_id`, `runners`.`namespace`
+                   FROM `legion`.`functions`
+                     INNER JOIN `legion`.`runners` ON (`functions`.`runner_id` = `runners`.`id`)
+                    WHERE `functions`.`name` = '#{function}'
+                     AND `runners`.`namespace` = '#{runner_class}' LIMIT 1;"
+
+            cache = Legion::Cache.get(sql)
+            return cache unless cache.nil?
+
+            results = Legion::Data::Connection.sequel.fetch(sql).first
+            Legion::Cache.set(sql, results) if results.is_a?(Hash) && results.count.positive?
+            results
+          end
+
+          def find_delayed(**)
+            sql = '
+              SELECT tasks.id, tasks.relationship_id, tasks.function_id, tasks.created,
+                relationships.delay as relationship_delay, relationships.chain_id,
+                functions.name as function_name,
+                runners.namespace as class, runners.id as runner_id,
+                CONCAT( `exchange`, ".",`queue`,".",`functions`.`name`) AS runner_routing_key
+              FROM legion.tasks
+                INNER JOIN legion.functions ON (functions.id = tasks.function_id)
+                INNER JOIN legion.runners ON (runners.id = functions.runner_id)
+                INNER JOIN `legion`.`extensions` ON (`runners`.`extension_id` = `extensions`.`id`)
+                LEFT JOIN legion.relationships ON (relationships.id = tasks.relationship_id)
+              WHERE status = \'task.delayed\';'
+
+            Legion::Data::Connection.sequel.fetch(sql).all
+          end
+
+          def find_subtasks(trigger_id:, **)
+            sql = "
+              SELECT
+               `relationships`.`id` as `relationship_id`, `debug`,
+               `chain_id`, `allow_new_chains`, `delay`, `trigger_id`, `action_id`, `conditions`, `transformation`,
+               `runners`.`namespace`, `runners`.`id` as `runner_id`, `runners`.`queue`,
+               `runners`.`namespace` as runner_class,
+               `functions`.`id` as `function_id`, `functions`.`name` as `function`,
+               `extensions`.`exchange`,
+               CONCAT( `exchange`, '.',`queue`,'.',`functions`.`name`) AS runner_routing_key
+              FROM `legion`.`relationships`
+                INNER JOIN `legion`.`functions` ON (`functions`.`id` = `relationships`.`action_id`)
+                INNER JOIN `legion`.`runners` ON (`functions`.`runner_id` = `runners`.`id`)
+                INNER JOIN `legion`.`extensions` ON (`runners`.`extension_id` = `extensions`.`id`)
+              WHERE `relationships`.`trigger_id` = #{trigger_id} AND `relationships`.`active` = 1;
+            "
+
+            cache = Legion::Cache.get(sql)
+            return cache unless cache.nil?
+
+            results = Legion::Data::Connection.sequel.fetch(sql).all
+            Legion::Cache.set(sql, results, 5) if results.is_a?(Array) && results.count.positive?
+            results
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/tasker/helpers/find_subtask.rb

@@ -0,0 +1,49 @@
+module Legion
+  module Extensions
+    module Tasker
+      module Helpers
+        module FindSubtask
+          def find_trigger(runner_class:, function:, **)
+            sql = "SELECT `functions`.`id` as `function_id`, `runner_id`, `runners`.`namespace`
+                   FROM `legion`.`functions`
+                     INNER JOIN `legion`.`runners` ON (`functions`.`runner_id` = `runners`.`id`)
+                    WHERE `functions`.`name` = '#{function}'
+                     AND `runners`.`namespace` = '#{runner_class}' LIMIT 1;"
+
+            cache = Legion::Cache.get(sql)
+            return cache unless cache.nil?
+
+            results = Legion::Data::Connection.sequel.fetch(sql).first
+            Legion::Cache.set(sql, results) if results.is_a?(Hash) && results.count.positive?
+            results
+          end
+
+          def find_subtasks(trigger_id:, **)
+            sql = "
+              SELECT
+               `relationships`.`id` as `relationship_id`, `debug`,
+               `chain_id`, `allow_new_chains`, `delay`, `trigger_id`, `action_id`, `conditions`, `transformation`,
+               `runners`.`namespace`, `runners`.`id` as `runner_id`, `runners`.`queue`,
+               `runners`.`namespace` as runner_class,
+               `functions`.`id` as `function_id`, `functions`.`name` as `function`,
+               `extensions`.`exchange`,
+               CONCAT( `exchange`, '.',`queue`,'.',`functions`.`name`) AS runner_routing_key
+              FROM `legion`.`relationships`
+                INNER JOIN `legion`.`functions` ON (`functions`.`id` = `relationships`.`action_id`)
+                INNER JOIN `legion`.`runners` ON (`functions`.`runner_id` = `runners`.`id`)
+                INNER JOIN `legion`.`extensions` ON (`runners`.`extension_id` = `extensions`.`id`)
+              WHERE `relationships`.`trigger_id` = #{trigger_id} AND `relationships`.`active` = 1;
+            "
+
+            cache = Legion::Cache.get(sql)
+            return cache unless cache.nil?
+
+            results = Legion::Data::Connection.sequel.fetch(sql).all
+            Legion::Cache.set(sql, results, 5) if results.is_a?(Array) && results.count.positive?
+            results
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/tasker/runners/check_subtask.rb

@@ -0,0 +1,97 @@
+require 'legion/transport/messages/subtask'
+
+module Legion::Extensions::Tasker
+  module Runners
+    module CheckSubtask
+      include Legion::Extensions::Helpers::Lex
+      extend Legion::Extensions::Tasker::Helpers::FindSubtask
+
+      def check_subtasks(runner_class:, function:, **opts)
+        trigger = find_trigger(runner_class: runner_class, function: function)
+
+        find_subtasks(trigger_id: trigger[:function_id]).each do |relationship|
+          unless relationship[:allow_new_chains]
+            next if relationship[:chain_id].nil?
+            next unless opts.key? :chain_id
+            next unless relationship[:chain_id] == opts[:chain_id]
+          end
+
+          task_hash = relationship
+          task_hash[:status] = relationship[:delay].zero? ? 'conditioner.queued' : 'task.delayed'
+          task_hash[:payload] = opts
+
+          if opts.key? :master_id
+            task_hash[:master_id] = opts[:master_id]
+          elsif opts.key? :parent_id
+            task_hash[:master_id] = opts[:parent_id]
+          elsif opts.key? :task_id
+            task_hash[:master_id] = opts[:task_id]
+          end
+
+          task_hash[:parent_id] = opts[:task_id] if opts.key? :task_id
+          task_hash[:routing_key] = if relationship[:conditions].is_a?(String) && relationship[:conditions].length > 4
+                                      'task.subtask.conditioner'
+                                    elsif relationship[:transformation].is_a?(String) && relationship[:transformation].length > 4 # rubocop:disable Layout/LineLength
+                                      'task.subtask.transformation'
+                                    else
+                                      relationship[:runner_routing_key]
+                                    end
+
+          if opts[:result].is_a? Array
+            opts[:result].each do |result|
+              send_task(results:             result,
+                        trigger_runner_id:   trigger[:runner_id],
+                        trigger_function_id: trigger[:function_id],
+                        **task_hash)
+            end
+          else
+            results = if opts[:results].is_a? Hash
+                        opts[:results]
+                      elsif opts[:result].is_a? Hash
+                        opts[:result]
+                      else
+                        opts
+                      end
+            send_task(
+              results:             results,
+              trigger_runner_id:   trigger[:runner_id],
+              trigger_function_id: trigger[:function_id],
+              **task_hash
+            )
+          end
+        end
+      end
+
+      def send_task(**opts)
+        opts[:results] = opts[:result] if opts.key?(:result) && !opts.key?(:results)
+        opts[:success] = if opts.key?(:result) && opts.key?(:success)
+                           opts[:result][:success]
+                         elsif opts.key?(:success)
+                           opts[:success]
+                         else
+                           1
+                         end
+
+        # opts[:task_id] = Legion::Runner::Status.generate_task_id(**opts)[:task_id]
+        opts[:task_id] = insert_task(**opts)
+        return { status: true } unless opts[:delay].zero?
+
+        Legion::Transport::Messages::SubTask.new(**opts).publish
+      end
+
+      def insert_task(relationship_id:, function_id:, status: 'task.queued', master_id: nil, parent_id: nil, **opts)
+        insert_hash = { relationship_id: relationship_id, function_id: function_id, status: status }
+        insert_hash[:master_id] = if master_id.is_a? Integer
+                                    master_id
+                                  elsif parent_id.is_a? Integer
+                                    parent_id
+                                  end
+        insert_hash[:parent_id] = parent_id if parent_id.is_a? Integer
+        insert_hash[:payload] = Legion::JSON.dump(opts)
+        # insert_hash[:function_args] = nil
+        # insert_hash[:results] = nil
+        Legion::Data::Model::Task.insert(insert_hash)
+      end
+    end
+  end
+end

data/lib/legion/extensions/tasker/runners/fetch_delayed.rb

@@ -0,0 +1,71 @@
+module Legion::Extensions::Tasker::Runners
+  module FetchDelayed
+    extend Legion::Extensions::Tasker::Helpers::FetchDelayed
+    include Legion::Extensions::Helpers::Task
+
+    def fetch(**_opts)
+      find_delayed.each do |task|
+        if task[:relationship_delay].is_a?(Integer) && task[:relationship_delay].positive?
+          next if Time.now < task[:created] + task[:relationship_delay] # rubocop:disable Style/SoleNestedConditional
+        end
+
+        if task[:task_delay].is_a?(Integer) && task[:task_delay].positive?
+          next if Time.now < task[:created] + task[:task_delay] # rubocop:disable Style/SoleNestedConditional
+        end
+
+        subtask_hash = {
+          relationship_id: task[:relationship_id],
+          chain_id:        task[:chain_id],
+          function_id:     task[:function_id],
+          function:        task[:function_name],
+          runner_id:       task[:runner_id],
+          runner_class:    task[:runner_class],
+          task_id:         task[:id],
+          exchange:        task[:exchange],
+          queue:           task[:queue]
+        }
+
+        subtask_hash[:conditions] = task[:conditions] if task[:conditions].is_a?(String)
+        subtask_hash[:transformation] = task[:transformation] if task[:transformation].is_a?(String)
+
+        subtask_hash[:routing_key] = if task[:conditions].is_a?(String) && task[:conditions].length > 4
+                                       'task.subtask.conditioner'
+                                     elsif task[:transformation].is_a?(String) && task[:transformation].length > 4
+                                       'task.subtask.transformation'
+                                     else
+                                       task[:runner_routing_key]
+                                     end
+
+        send_task(**subtask_hash)
+        case subtask_hash[:routing_key]
+        when 'task.subtask.conditioner'
+          task_update(task[:id], 'conditioner.queued')
+        when 'task.subtask.transformation'
+          task_update(task[:id], 'transformer.queued')
+        else
+          task_update(task[:id], 'task.queued')
+        end
+      end
+    end
+
+    def send_task(**opts)
+      opts[:results] = opts[:result] if opts.key?(:result) && !opts.key?(:results)
+      opts[:success] = if opts.key?(:result) && opts.key?(:success)
+                         opts[:result][:success]
+                       elsif opts.key?(:success)
+                         opts[:success]
+                       else
+                         1
+                       end
+      log.debug 'pushing delayed task to worker'
+      Legion::Transport::Messages::Task.new(**opts).publish
+    end
+
+    def push(**_opts)
+      Legion::Extensions::Tasker::Transport::Messages::FetchDelayed.new.publish
+      { success: true }
+    end
+
+    include Legion::Extensions::Helpers::Lex
+  end
+end

data/lib/legion/extensions/tasker/runners/log.rb

@@ -0,0 +1,50 @@
+module Legion::Extensions::Tasker
+  module Runners
+    module Log
+      include Legion::Extensions::Helpers::Lex
+
+      def add_log(task_id:, entry:, function: nil, runner_class: nil, **opts)
+        entry = JSON.dump(entry) unless entry.is_a? String
+        insert = { task_id: task_id, entry: entry }
+        if opts.key?(:node_id)
+          insert[:node_id] = payload[:node_id]
+        elsif opts.key?(:name)
+          node = Legion::Data::Model::Node.where(opts[:name]).first
+          insert[:node_id] = node.values[:id] unless node.nil?
+        end
+        insert[:function_id] = opts[:function_id] if opts.key? :function_id
+
+        unless function.nil? && runner_class.nil?
+          runner = Legion::Data::Model::Runner.where(namespace: runner_class).first
+          unless runner.values.nil?
+            insert[:function_id] = runner.functions_dataset.where(name: function).first.values[:id]
+          end
+        end
+
+        id = Legion::Data::Model::TaskLog.insert(insert)
+
+        { success: !id.nil?, id: id }
+      end
+
+      def delete_log(id:, **_opts)
+        delete = Legion::Data::Model::TaskLog[id].delete
+        { success: delete.positive?, count: delete, deleted_id: id }
+      end
+
+      def delete_task_logs(task_id:, **_opts)
+        delete = Legion::Data::Model::TaskLog.where(task_id: task_id).delete
+        { success: delete.positive?, count: delete, deleted_task_id: task_id }
+      end
+
+      def delete_node_logs(node_id:, **_opts)
+        delete = Legion::Data::Model::TaskLog.where(node_id: node_id).delete
+        { success: delete.positive?, count: delete, deleted_node_id: node_id }
+      end
+
+      def delete_all(**_opts)
+        delete = Legion::Data::Model::TaskLog.all.delete
+        { success: delete.positive?, count: delete }
+      end
+    end
+  end
+end

data/lib/legion/extensions/tasker/runners/task_manager.rb

@@ -0,0 +1,28 @@
+module Legion
+  module Extensions
+    module Tasker
+      module Runners
+        module TaskManager
+          def purge_old(age: 31, limit: 100, status: 'task.completed', **_opts)
+            log.debug("purging old completed tasks with an age > #{age} days, limit: #{limit}")
+            dataset = Legion::Data::Model::Task
+                      .where(Sequel.lit("created <= DATE_SUB(SYSDATE(), INTERVAL #{age} DAY)"))
+                      .limit(limit)
+            dataset.where(status: status) unless ['*', nil, ''].include? status
+            log.debug "Deleting #{dataset.count} records" if dataset.count.positive?
+            dataset&.delete
+          end
+
+          def expire_queued(age: 1, limit: 10, **) # rubocop:disable Lint/UnusedMethodArgument
+            dataset = Legion::Data::Model::Task # rubocop:disable Lint/UselessAssignment
+                      .where(status: ['conditioner.queued', 'transformer.queued', 'task.queued'])
+                      .limit(limit)
+          end
+
+          include Legion::Extensions::Helpers::Task
+          include Legion::Extensions::Helpers::Lex
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/tasker/runners/updater.rb

@@ -0,0 +1,31 @@
+module Legion::Extensions::Tasker
+  module Runners
+    module Updater
+      include Legion::Extensions::Helpers::Lex
+
+      def update_status(task_id:, **opts)
+        task = Legion::Data::Model::Task[task_id]
+        if task.nil? || task.values.nil?
+          return { success: false, changed: false, task_id: task_id, message: 'task nil' }
+        end
+
+        log.unknown task.class
+
+        update_hash = {}
+        %i[status function_args payload results].each do |column|
+          next unless opts.key? column
+
+          update_hash[column] = if opts[column].is_a? String
+                                  opts[column]
+                                else
+                                  to_json opts[column]
+                                end
+        end
+        { success: true, changed: false, task_id: task_id } if update_hash.count.zero?
+        task.update(update_hash)
+
+        { success: true, changed: true, task_id: task_id, updates: update_hash }
+      end
+    end
+  end
+end