lex-tasker 0.2.1 → 0.3.0

data/lib/legion/extensions/tasker/helpers/task_finder.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module Tasker
+      module Helpers
+        module TaskFinder
+          def find_trigger(runner_class:, function:, **)
+            cache_key = "tasker:find_trigger:#{runner_class}:#{function}"
+            cached = Legion::Cache.get(cache_key)
+            return cached unless cached.nil?
+
+            result = Legion::Data::Model::Function
+                     .join(:runners, id: :runner_id)
+                     .where(Sequel[:functions][:name]    => function,
+                            Sequel[:runners][:namespace] => runner_class)
+                     .select(Sequel[:functions][:id].as(:function_id),
+                             Sequel[:functions][:runner_id],
+                             Sequel[:runners][:namespace])
+                     .first
+
+            Legion::Cache.set(cache_key, result) if result
+            result
+          end
+
+          def find_subtasks(trigger_id:, **)
+            cache_key = "tasker:find_subtasks:#{trigger_id}"
+            cached = Legion::Cache.get(cache_key)
+            return cached unless cached.nil?
+
+            results = subtask_query(trigger_id).all.map do |row|
+              row[:runner_routing_key] = "#{row[:exchange]}.#{row[:queue]}.#{row[:function]}"
+              row
+            end
+
+            Legion::Cache.set(cache_key, results, 5) if results.is_a?(Array) && results.any?
+            results
+          end
+
+          def find_delayed(**)
+            Legion::Data::Model::Task
+              .join(:functions, id: :function_id)
+              .join(:runners, id: Sequel[:functions][:runner_id])
+              .join(:extensions, id: Sequel[:runners][:extension_id])
+              .left_join(:relationships, id: Sequel[:tasks][:relationship_id])
+              .where(Sequel[:tasks][:status] => 'task.delayed')
+              .select(
+                Sequel[:tasks][:id],
+                Sequel[:tasks][:relationship_id],
+                Sequel[:tasks][:function_id],
+                Sequel[:tasks][:created],
+                Sequel[:relationships][:delay].as(:relationship_delay),
+                Sequel[:relationships][:chain_id],
+                Sequel[:functions][:name].as(:function_name),
+                Sequel[:runners][:namespace].as(:runner_class),
+                Sequel[:runners][:id].as(:runner_id),
+                Sequel[:runners][:queue],
+                Sequel[:extensions][:exchange],
+                Sequel[:tasks][:task_delay]
+              ).all.map do |task|
+                task[:runner_routing_key] = "#{task[:exchange]}.#{task[:queue]}.#{task[:function_name]}"
+                task
+              end
+          end
+
+          private
+
+          def subtask_query(trigger_id)
+            Legion::Data::Model::Relationship
+              .join(:functions, id: :action_id)
+              .join(:runners, id: Sequel[:functions][:runner_id])
+              .join(:extensions, id: Sequel[:runners][:extension_id])
+              .where(Sequel[:relationships][:trigger_id] => trigger_id,
+                     Sequel[:relationships][:active]     => true)
+              .select(
+                Sequel[:relationships][:id].as(:relationship_id),
+                Sequel[:relationships][:debug],
+                Sequel[:relationships][:chain_id],
+                Sequel[:relationships][:allow_new_chains],
+                Sequel[:relationships][:delay],
+                Sequel[:relationships][:trigger_id],
+                Sequel[:relationships][:action_id],
+                Sequel[:relationships][:conditions],
+                Sequel[:relationships][:transformation],
+                Sequel[:runners][:namespace],
+                Sequel[:runners][:id].as(:runner_id),
+                Sequel[:runners][:queue],
+                Sequel[:runners][:namespace].as(:runner_class),
+                Sequel[:functions][:id].as(:function_id),
+                Sequel[:functions][:name].as(:function),
+                Sequel[:extensions][:exchange]
+              )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/tasker/runners/check_subtask.rb

@@ -1,96 +1,108 @@
+# frozen_string_literal: true
+
 require 'legion/transport/messages/subtask'
 
-module Legion::Extensions::Tasker
-  module Runners
-    module CheckSubtask
-      include Legion::Extensions::Helpers::Lex
-      extend Legion::Extensions::Tasker::Helpers::FindSubtask
+module Legion
+  module Extensions
+    module Tasker
+      module Runners
+        module CheckSubtask
+          include Legion::Extensions::Helpers::Lex
+          include Legion::Extensions::Tasker::Helpers::TaskFinder
+
+          def check_subtasks(runner_class:, function:, **opts)
+            trigger = find_trigger(runner_class: runner_class, function: function)
+            return { success: true, subtasks: 0 } unless trigger
+
+            find_subtasks(trigger_id: trigger[:function_id]).each do |relationship|
+              next unless chain_matches?(relationship, opts)
+
+              task_hash = build_task_hash(relationship, opts)
+              dispatch_task(task_hash, trigger, opts)
+            end
+          end
+
+          def chain_matches?(relationship, opts)
+            return true if relationship[:allow_new_chains]
 
-      def check_subtasks(runner_class:, function:, **opts)
-        trigger = find_trigger(runner_class: runner_class, function: function)
+            !relationship[:chain_id].nil? && opts.key?(:chain_id) && relationship[:chain_id] == opts[:chain_id]
+          end
 
-        find_subtasks(trigger_id: trigger[:function_id]).each do |relationship|
-          unless relationship[:allow_new_chains]
-            next if relationship[:chain_id].nil?
-            next unless opts.key? :chain_id
-            next unless relationship[:chain_id] == opts[:chain_id]
+          def build_task_hash(relationship, opts)
+            task_hash = relationship.dup
+            task_hash[:status] = relationship[:delay].to_i.zero? ? 'conditioner.queued' : 'task.delayed'
+            task_hash[:payload] = opts
+            task_hash[:master_id] = resolve_master_id(opts)
+            task_hash[:parent_id] = opts[:task_id] if opts.key?(:task_id)
+            task_hash[:routing_key] = subtask_routing_key(relationship)
+            task_hash
           end
 
-          task_hash = relationship
-          task_hash[:status] = relationship[:delay].zero? ? 'conditioner.queued' : 'task.delayed'
-          task_hash[:payload] = opts
+          def resolve_master_id(opts)
+            return opts[:master_id] if opts.key?(:master_id)
+            return opts[:parent_id] if opts.key?(:parent_id)
 
-          if opts.key? :master_id
-            task_hash[:master_id] = opts[:master_id]
-          elsif opts.key? :parent_id
-            task_hash[:master_id] = opts[:parent_id]
-          elsif opts.key? :task_id
-            task_hash[:master_id] = opts[:task_id]
+            opts[:task_id] if opts.key?(:task_id)
           end
 
-          task_hash[:parent_id] = opts[:task_id] if opts.key? :task_id
-          task_hash[:routing_key] = if relationship[:conditions].is_a?(String) && relationship[:conditions].length > 4
-                                      'task.subtask.conditioner'
-                                    elsif relationship[:transformation].is_a?(String) && relationship[:transformation].length > 4 # rubocop:disable Layout/LineLength
-                                      'task.subtask.transformation'
-                                    else
-                                      relationship[:runner_routing_key]
-                                    end
-
-          if opts[:result].is_a? Array
-            opts[:result].each do |result|
-              send_task(results:             result,
-                        trigger_runner_id:   trigger[:runner_id],
-                        trigger_function_id: trigger[:function_id],
-                        **task_hash)
+          def subtask_routing_key(relationship)
+            if relationship[:conditions].is_a?(String) && relationship[:conditions].length > 4
+              'task.subtask.conditioner'
+            elsif relationship[:transformation].is_a?(String) && relationship[:transformation].length > 4
+              'task.subtask.transformation'
+            else
+              relationship[:runner_routing_key]
             end
-          else
-            results = if opts[:results].is_a? Hash
-                        opts[:results]
-                      elsif opts[:result].is_a? Hash
-                        opts[:result]
-                      else
-                        opts
-                      end
-            send_task(
-              results:             results,
-              trigger_runner_id:   trigger[:runner_id],
-              trigger_function_id: trigger[:function_id],
-              **task_hash
-            )
           end
-        end
-      end
 
-      def send_task(**opts)
-        opts[:results] = opts[:result] if opts.key?(:result) && !opts.key?(:results)
-        opts[:success] = if opts.key?(:result) && opts.key?(:success)
-                           opts[:result][:success]
-                         elsif opts.key?(:success)
-                           opts[:success]
-                         else
-                           1
-                         end
-
-        # opts[:task_id] = Legion::Runner::Status.generate_task_id(**opts)[:task_id]
-        opts[:task_id] = insert_task(**opts)
-        return { status: true } unless opts[:delay].zero?
-
-        Legion::Transport::Messages::SubTask.new(**opts).publish
-      end
+          def dispatch_task(task_hash, trigger, opts)
+            trigger_info = { trigger_runner_id: trigger[:runner_id], trigger_function_id: trigger[:function_id] }
+            results_value = opts[:result] || opts[:results]
+
+            if results_value.is_a?(Array)
+              results_value.each do |result|
+                send_task(results: result, **trigger_info, **task_hash)
+              end
+            else
+              send_task(results: resolve_results(opts), **trigger_info, **task_hash)
+            end
+          end
+
+          def resolve_results(opts)
+            return opts[:results] if opts[:results].is_a?(Hash)
+            return opts[:result] if opts[:result].is_a?(Hash)
+
+            opts
+          end
+
+          def send_task(**opts)
+            opts[:results] = opts[:result] if opts.key?(:result) && !opts.key?(:results)
+            opts[:success] = if opts.key?(:result) && opts.key?(:success)
+                               opts[:result][:success]
+                             elsif opts.key?(:success)
+                               opts[:success]
+                             else
+                               1
+                             end
 
-      def insert_task(relationship_id:, function_id:, status: 'task.queued', master_id: nil, parent_id: nil, **opts)
-        insert_hash = { relationship_id: relationship_id, function_id: function_id, status: status }
-        insert_hash[:master_id] = if master_id.is_a? Integer
-                                    master_id
-                                  elsif parent_id.is_a? Integer
-                                    parent_id
-                                  end
-        insert_hash[:parent_id] = parent_id if parent_id.is_a? Integer
-        insert_hash[:payload] = Legion::JSON.dump(opts)
-        # insert_hash[:function_args] = nil
-        # insert_hash[:results] = nil
-        Legion::Data::Model::Task.insert(insert_hash)
+            opts[:task_id] = insert_task(**opts)
+            return { status: true } unless opts[:delay].zero?
+
+            Legion::Transport::Messages::SubTask.new(**opts).publish
+          end
+
+          def insert_task(relationship_id:, function_id:, **opts)
+            status = opts.fetch(:status, 'task.queued')
+            master_id = opts[:master_id]
+            parent_id = opts[:parent_id]
+
+            insert_hash = { relationship_id: relationship_id, function_id: function_id, status: status }
+            insert_hash[:master_id] = master_id.is_a?(Integer) ? master_id : (parent_id if parent_id.is_a?(Integer))
+            insert_hash[:parent_id] = parent_id if parent_id.is_a?(Integer)
+            insert_hash[:payload] = Legion::JSON.dump(opts)
+            Legion::Data::Model::Task.insert(insert_hash)
+          end
+        end
       end
     end
   end

data/lib/legion/extensions/tasker/runners/fetch_delayed.rb

@@ -1,71 +1,87 @@
-module Legion::Extensions::Tasker::Runners
-  module FetchDelayed
-    extend Legion::Extensions::Tasker::Helpers::FetchDelayed
-    include Legion::Extensions::Helpers::Task
+# frozen_string_literal: true
 
-    def fetch(**_opts)
-      find_delayed.each do |task|
-        if task[:relationship_delay].is_a?(Integer) && task[:relationship_delay].positive?
-          next if Time.now < task[:created] + task[:relationship_delay] # rubocop:disable Style/SoleNestedConditional
-        end
+module Legion
+  module Extensions
+    module Tasker
+      module Runners
+        module FetchDelayed
+          include Legion::Extensions::Tasker::Helpers::TaskFinder
+          include Legion::Extensions::Helpers::Task
 
-        if task[:task_delay].is_a?(Integer) && task[:task_delay].positive?
-          next if Time.now < task[:created] + task[:task_delay] # rubocop:disable Style/SoleNestedConditional
-        end
+          def fetch(**_opts)
+            find_delayed.each do |task|
+              next if delayed_by?(task[:relationship_delay], task[:created])
+              next if delayed_by?(task[:task_delay], task[:created])
 
-        subtask_hash = {
-          relationship_id: task[:relationship_id],
-          chain_id:        task[:chain_id],
-          function_id:     task[:function_id],
-          function:        task[:function_name],
-          runner_id:       task[:runner_id],
-          runner_class:    task[:runner_class],
-          task_id:         task[:id],
-          exchange:        task[:exchange],
-          queue:           task[:queue]
-        }
+              subtask_hash = build_delayed_hash(task)
+              send_task(**subtask_hash)
+              update_delayed_status(task[:id], subtask_hash[:routing_key])
+            end
+          end
 
-        subtask_hash[:conditions] = task[:conditions] if task[:conditions].is_a?(String)
-        subtask_hash[:transformation] = task[:transformation] if task[:transformation].is_a?(String)
+          def delayed_by?(delay, created)
+            delay.is_a?(Integer) && delay.positive? && Time.now < created + delay
+          end
 
-        subtask_hash[:routing_key] = if task[:conditions].is_a?(String) && task[:conditions].length > 4
-                                       'task.subtask.conditioner'
-                                     elsif task[:transformation].is_a?(String) && task[:transformation].length > 4
-                                       'task.subtask.transformation'
-                                     else
-                                       task[:runner_routing_key]
-                                     end
+          def build_delayed_hash(task)
+            subtask_hash = {
+              relationship_id: task[:relationship_id],
+              chain_id:        task[:chain_id],
+              function_id:     task[:function_id],
+              function:        task[:function_name],
+              runner_id:       task[:runner_id],
+              runner_class:    task[:runner_class],
+              task_id:         task[:id],
+              exchange:        task[:exchange],
+              queue:           task[:queue]
+            }
 
-        send_task(**subtask_hash)
-        case subtask_hash[:routing_key]
-        when 'task.subtask.conditioner'
-          task_update(task[:id], 'conditioner.queued')
-        when 'task.subtask.transformation'
-          task_update(task[:id], 'transformer.queued')
-        else
-          task_update(task[:id], 'task.queued')
-        end
-      end
-    end
+            subtask_hash[:conditions] = task[:conditions] if task[:conditions].is_a?(String)
+            subtask_hash[:transformation] = task[:transformation] if task[:transformation].is_a?(String)
+            subtask_hash[:routing_key] = delayed_routing_key(task)
+            subtask_hash
+          end
 
-    def send_task(**opts)
-      opts[:results] = opts[:result] if opts.key?(:result) && !opts.key?(:results)
-      opts[:success] = if opts.key?(:result) && opts.key?(:success)
-                         opts[:result][:success]
-                       elsif opts.key?(:success)
-                         opts[:success]
-                       else
-                         1
-                       end
-      log.debug 'pushing delayed task to worker'
-      Legion::Transport::Messages::Task.new(**opts).publish
-    end
+          def delayed_routing_key(task)
+            if task[:conditions].is_a?(String) && task[:conditions].length > 4
+              'task.subtask.conditioner'
+            elsif task[:transformation].is_a?(String) && task[:transformation].length > 4
+              'task.subtask.transformation'
+            else
+              task[:runner_routing_key]
+            end
+          end
 
-    def push(**_opts)
-      Legion::Extensions::Tasker::Transport::Messages::FetchDelayed.new.publish
-      { success: true }
-    end
+          def update_delayed_status(task_id, routing_key)
+            status = case routing_key
+                     when 'task.subtask.conditioner' then 'conditioner.queued'
+                     when 'task.subtask.transformation' then 'transformer.queued'
+                     else 'task.queued'
+                     end
+            task_update(task_id, status)
+          end
+
+          def send_task(**opts)
+            opts[:results] = opts[:result] if opts.key?(:result) && !opts.key?(:results)
+            opts[:success] = if opts.key?(:result) && opts.key?(:success)
+                               opts[:result][:success]
+                             elsif opts.key?(:success)
+                               opts[:success]
+                             else
+                               1
+                             end
+            log.debug 'pushing delayed task to worker'
+            Legion::Transport::Messages::Task.new(**opts).publish
+          end
+
+          def push(**_opts)
+            Legion::Extensions::Tasker::Transport::Messages::FetchDelayed.new.publish
+            { success: true }
+          end
 
-    include Legion::Extensions::Helpers::Lex
+          include Legion::Extensions::Helpers::Lex
+        end
+      end
+    end
   end
 end

data/lib/legion/extensions/tasker/runners/log.rb

@@ -1,49 +1,55 @@
-module Legion::Extensions::Tasker
-  module Runners
-    module Log
-      include Legion::Extensions::Helpers::Lex
-
-      def add_log(task_id:, entry:, function: nil, runner_class: nil, **opts)
-        entry = JSON.dump(entry) unless entry.is_a? String
-        insert = { task_id: task_id, entry: entry }
-        if opts.key?(:node_id)
-          insert[:node_id] = payload[:node_id]
-        elsif opts.key?(:name)
-          node = Legion::Data::Model::Node.where(opts[:name]).first
-          insert[:node_id] = node.values[:id] unless node.nil?
-        end
-        insert[:function_id] = opts[:function_id] if opts.key? :function_id
-
-        unless function.nil? && runner_class.nil?
-          runner = Legion::Data::Model::Runner.where(namespace: runner_class).first
-          unless runner.values.nil?
-            insert[:function_id] = runner.functions_dataset.where(name: function).first.values[:id]
+# frozen_string_literal: true
+
+require 'json'
+
+module Legion
+  module Extensions
+    module Tasker
+      module Runners
+        module Log
+          include Legion::Extensions::Helpers::Lex
+
+          def add_log(task_id:, entry:, function: nil, runner_class: nil, **opts)
+            entry = ::JSON.dump(entry) unless entry.is_a? String
+            insert = { task_id: task_id, entry: entry }
+            if opts.key?(:node_id)
+              insert[:node_id] = opts[:node_id]
+            elsif opts.key?(:name)
+              node = Legion::Data::Model::Node.where(name: opts[:name]).first
+              insert[:node_id] = node.values[:id] unless node.nil?
+            end
+            insert[:function_id] = opts[:function_id] if opts.key? :function_id
+
+            unless function.nil? && runner_class.nil?
+              runner = Legion::Data::Model::Runner.where(namespace: runner_class).first
+              insert[:function_id] = runner.functions_dataset.where(name: function).first.values[:id] unless runner.nil?
+            end
+
+            id = Legion::Data::Model::TaskLog.insert(insert)
+
+            { success: !id.nil?, id: id }
           end
-        end
-
-        id = Legion::Data::Model::TaskLog.insert(insert)
 
-        { success: !id.nil?, id: id }
-      end
-
-      def delete_log(id:, **_opts)
-        delete = Legion::Data::Model::TaskLog[id].delete
-        { success: delete.positive?, count: delete, deleted_id: id }
-      end
+          def delete_log(id:, **_opts)
+            delete = Legion::Data::Model::TaskLog[id].delete
+            { success: delete.positive?, count: delete, deleted_id: id }
+          end
 
-      def delete_task_logs(task_id:, **_opts)
-        delete = Legion::Data::Model::TaskLog.where(task_id: task_id).delete
-        { success: delete.positive?, count: delete, deleted_task_id: task_id }
-      end
+          def delete_task_logs(task_id:, **_opts)
+            delete = Legion::Data::Model::TaskLog.where(task_id: task_id).delete
+            { success: delete.positive?, count: delete, deleted_task_id: task_id }
+          end
 
-      def delete_node_logs(node_id:, **_opts)
-        delete = Legion::Data::Model::TaskLog.where(node_id: node_id).delete
-        { success: delete.positive?, count: delete, deleted_node_id: node_id }
-      end
+          def delete_node_logs(node_id:, **_opts)
+            delete = Legion::Data::Model::TaskLog.where(node_id: node_id).delete
+            { success: delete.positive?, count: delete, deleted_node_id: node_id }
+          end
 
-      def delete_all(**_opts)
-        delete = Legion::Data::Model::TaskLog.all.delete
-        { success: delete.positive?, count: delete }
+          def delete_all(**_opts)
+            delete = Legion::Data::Model::TaskLog.dataset.delete
+            { success: delete.positive?, count: delete }
+          end
+        end
       end
     end
   end

data/lib/legion/extensions/tasker/runners/task_manager.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Legion
   module Extensions
     module Tasker
@@ -5,18 +7,23 @@ module Legion
         module TaskManager
           def purge_old(age: 31, limit: 100, status: 'task.completed', **_opts)
             log.debug("purging old completed tasks with an age > #{age} days, limit: #{limit}")
+            cutoff = Time.now - (age * 86_400)
             dataset = Legion::Data::Model::Task
-                      .where(Sequel.lit("created <= DATE_SUB(SYSDATE(), INTERVAL #{age} DAY)"))
+                      .where(Sequel.lit('created <= ?', cutoff))
                       .limit(limit)
-            dataset.where(status: status) unless ['*', nil, ''].include? status
-            log.debug "Deleting #{dataset.count} records" if dataset.count.positive?
+            dataset = dataset.where(status: status) unless ['*', nil, ''].include?(status)
+            log.debug "Deleting #{dataset.count} records" if dataset.any?
             dataset&.delete
           end
 
-          def expire_queued(age: 1, limit: 10, **) # rubocop:disable Lint/UnusedMethodArgument
-            dataset = Legion::Data::Model::Task # rubocop:disable Lint/UselessAssignment
+          def expire_queued(age: 1, limit: 10, **)
+            cutoff = Time.now - (age * 3600)
+            dataset = Legion::Data::Model::Task
                       .where(status: ['conditioner.queued', 'transformer.queued', 'task.queued'])
+                      .where(Sequel.lit('created <= ?', cutoff))
                       .limit(limit)
+            count = dataset.update(status: 'task.expired')
+            { success: true, expired: count }
           end
 
           include Legion::Extensions::Helpers::Task

data/lib/legion/extensions/tasker/runners/updater.rb

@@ -1,30 +1,33 @@
-module Legion::Extensions::Tasker
-  module Runners
-    module Updater
-      include Legion::Extensions::Helpers::Lex
+# frozen_string_literal: true
 
-      def update_status(task_id:, **opts)
-        task = Legion::Data::Model::Task[task_id]
-        if task.nil? || task.values.nil?
-          return { success: false, changed: false, task_id: task_id, message: 'task nil' }
-        end
+module Legion
+  module Extensions
+    module Tasker
+      module Runners
+        module Updater
+          include Legion::Extensions::Helpers::Lex
 
-        log.unknown task.class
+          def update_status(task_id:, **opts)
+            task = Legion::Data::Model::Task[task_id]
+            return { success: false, changed: false, task_id: task_id, message: 'task nil' } if task.nil? || task.values.nil?
 
-        update_hash = {}
-        %i[status function_args payload results].each do |column|
-          next unless opts.key? column
+            update_hash = {}
+            %i[status function_args payload results].each do |column|
+              next unless opts.key? column
 
-          update_hash[column] = if opts[column].is_a? String
-                                  opts[column]
-                                else
-                                  to_json opts[column]
-                                end
-        end
-        { success: true, changed: false, task_id: task_id } if update_hash.count.zero?
-        task.update(update_hash)
+              update_hash[column] = if opts[column].is_a? String
+                                      opts[column]
+                                    else
+                                      to_json opts[column]
+                                    end
+            end
+            return { success: true, changed: false, task_id: task_id } if update_hash.none?
 
-        { success: true, changed: true, task_id: task_id, updates: update_hash }
+            task.update(update_hash)
+
+            { success: true, changed: true, task_id: task_id, updates: update_hash }
+          end
+        end
       end
     end
   end

data/lib/legion/extensions/tasker/transport/exchanges/task.rb

@@ -1,7 +1,13 @@
-module Legion::Extensions::Tasker
-  module Transport
-    module Exchanges
-      class Tasker < Legion::Transport::Exchanges::Task
+# frozen_string_literal: true
+
+module Legion
+  module Extensions
+    module Tasker
+      module Transport
+        module Exchanges
+          class Tasker < Legion::Transport::Exchanges::Task
+          end
+        end
       end
     end
   end