lex-tasker 0.1.3 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a86403f5cd4a6634b29ca4000c36734bb53d4f5913682e2c4f60c97defa6f590
-  data.tar.gz: 4dfce03dd9c7ac9b8d5ccfe21d085328fce549c25367b088a33b3df563f6f0b5
+  metadata.gz: 8086ae496d1e741099cd0f91cd01cfaa91820518d3a93e00262ae04eeea471a5
+  data.tar.gz: 219a9d573a6e17923f01b2528fb91c1235dc1e62cf7b2dcdf9624dd99abc8d75
 SHA512:
-  metadata.gz: 2017fff11b5c75e1701cce065e7cc38b61bd2b7dbd7cb0de72d0441df9a74e93bac47a9ad9191c3c10c80548b6aa13847ffc26f8189b35fc87ae72311c3e1a3b
-  data.tar.gz: d501e798ea13a21333a2f67975d412338a929a23c5e2345af28f88d19be2e0420af9d0ec781cb06033ce2536dd40b33d04c96ada2aa85a8491b5f99fc7a5b0c6
+  metadata.gz: c7c51a2a36129f803d34b2b7aaecc43c06bdd619fabd8dd8ee37b0cf9a38a1ff68b8c0983efd3b742e22332bfc05afdb8507a04c7a62f0500ec22fb621e9ba6e
+  data.tar.gz: 7a98de6ea1d7b663992457e2c61df4a46c984ac41af4f30218404ea3798ecd0da3b8a880a86128637bb36105f971bc09a5c577934e1a7b2de633edefcb1ad14e

data/.circleci/config.yml

@@ -5,57 +5,35 @@ orbs:
 jobs:
   "rubocop":
     docker:
-      - image: circleci/ruby:2.5-node
+      - image: circleci/ruby:2.7
     steps:
       - checkout
       - ruby/load-cache
       - ruby/install-deps
+      - run:
+          name: Update bundler
+          command: gem update bundler
       - run:
           name: Run Rubocop
           command: bundle exec rubocop
       - ruby/save-cache
-  "ruby-two-five":
-    docker:
-      - image: circleci/ruby:2.5
-      - image: memcached:1.5-alpine
-    steps:
-      - checkout
-      - ruby/load-cache
-      - ruby/install-deps
-      - ruby/run-tests
-      - ruby/save-cache
-  "ruby-two-six":
-    docker:
-      - image: circleci/ruby:2.6
-      - image: memcached:1.5-alpine
-    steps:
-      - checkout
-      - ruby/load-cache
-      - ruby/install-deps
-      - ruby/run-tests
-      - ruby/save-cache
-  "ruby-two-seven":
+  "rspec":
     docker:
       - image: circleci/ruby:2.7
-      - image: memcached:1.5-alpine
     steps:
       - checkout
       - ruby/load-cache
       - ruby/install-deps
+      - run:
+          name: Update bundler
+          command: gem update bundler
       - ruby/run-tests
       - ruby/save-cache
-
 workflows:
   version: 2
   rubocop-rspec:
     jobs:
       - rubocop
-      - ruby-two-five:
-          requires:
-            - rubocop
-      - ruby-two-six:
-          requires:
-            - ruby-two-five
-      - ruby-two-seven:
+      - rspec:
           requires:
-            - ruby-two-five
+            - rubocop

data/Gemfile.lock

@@ -0,0 +1,69 @@
+PATH
+  remote: .
+  specs:
+    lex-tasker (0.2.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    ast (2.4.1)
+    codecov (0.2.12)
+      json
+      simplecov
+    diff-lcs (1.4.4)
+    docile (1.3.2)
+    json (2.3.1)
+    parallel (1.19.2)
+    parser (2.7.2.0)
+      ast (~> 2.4.1)
+    rainbow (3.0.0)
+    rake (13.0.1)
+    regexp_parser (1.8.2)
+    rexml (3.2.4)
+    rspec (3.10.0)
+      rspec-core (~> 3.10.0)
+      rspec-expectations (~> 3.10.0)
+      rspec-mocks (~> 3.10.0)
+    rspec-core (3.10.0)
+      rspec-support (~> 3.10.0)
+    rspec-expectations (3.10.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.10.0)
+    rspec-mocks (3.10.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.10.0)
+    rspec-support (3.10.0)
+    rspec_junit_formatter (0.4.1)
+      rspec-core (>= 2, < 4, != 2.12.0)
+    rubocop (1.1.0)
+      parallel (~> 1.10)
+      parser (>= 2.7.1.5)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8)
+      rexml
+      rubocop-ast (>= 1.0.1)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 2.0)
+    rubocop-ast (1.1.0)
+      parser (>= 2.7.1.5)
+    ruby-progressbar (1.10.1)
+    simplecov (0.19.1)
+      docile (~> 1.1)
+      simplecov-html (~> 0.11)
+    simplecov-html (0.12.3)
+    unicode-display_width (1.7.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bundler (>= 2)
+  codecov
+  lex-tasker!
+  rake
+  rspec
+  rspec_junit_formatter
+  rubocop
+
+BUNDLED WITH
+   2.1.4

data/README.md

@@ -1,28 +1,3 @@
 # Legion::Extensions::Tasker
 
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/legion/extensions/tasker`. To experiment with that code, run `bin/console` for an interactive prompt.
-
-
-## Installation
-
-Add this line to your application's Gemfile:
-
-```ruby
-gem 'lex-tasker'
-```
-
-And then execute:
-
-    $ bundle
-
-Or install it yourself as:
-
-    $ gem install lex-tasker
-
-## Usage
-
-TODO: Write usage instructions here
-
-## License
-
-The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+A core LEX

data/bitbucket-pipelines.yml

@@ -0,0 +1,69 @@
+image: ruby:2.7
+
+pipelines:
+  tags:
+    "v*":
+      - step:
+          name: Rubocop
+          caches:
+            - bundler
+          script:
+            - gem install rubocop
+            - gem update rubocop
+            - rubocop
+      - step:
+          name: RSpec
+          caches:
+            - bundler
+          script:
+            - gem install bundler
+            - gem update bundler
+            - bundle update
+            - bundle exec rspec
+      - step:
+          name: Push to RubyGems
+          deployment: RubyGems
+          script:
+            - gem install bundler gem-release rspec
+            - bundle install
+            - (umask  077 ; echo $gem_creds | base64 --decode > ~/.gem/credentials)
+            - gem release
+          artifacts:
+            - pkg/**
+  branches:
+    master:
+      - step:
+          caches:
+            - bundler
+          script:
+            - gem install rubocop
+            - gem update rubocop
+            - rubocop
+      - step:
+          caches:
+            - bundler
+          script:
+            - gem install bundler
+            - gem update bundler
+            - bundle update
+            - bundle exec rspec
+    develop:
+      - step:
+          caches:
+            - bundler
+          script:
+            - gem install rubocop
+            - gem update rubocop
+            - rubocop
+      - step:
+          caches:
+            - bundler
+          script:
+            - gem install bundler
+            - gem update bundler
+            - bundle update
+            - bundle exec rspec
+
+definitions:
+  caches:
+    bundler: /usr/local/bundle

data/lex-tasker.gemspec

@@ -5,7 +5,7 @@ require 'legion/extensions/tasker/version'
 Gem::Specification.new do |spec|
   spec.name          = 'lex-tasker'
   spec.version       = Legion::Extensions::Tasker::VERSION
-  spec.authors       = ['Miverson']
+  spec.authors       = ['Esity']
   spec.email         = ['matthewdiverson@gmail.com']
 
   spec.summary       = 'LEX::Tasker manages tasks status from Legion'
@@ -14,18 +14,20 @@ Gem::Specification.new do |spec|
   spec.license       = 'MIT'
   spec.required_ruby_version = Gem::Requirement.new('>= 2.5.0')
 
-  # Specify which files should be added to the gem when it is released.
-  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
   spec.files = Dir.chdir(File.expand_path(__dir__)) do
     `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
   end
-  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.metadata['homepage_uri'] = spec.homepage
+  spec.metadata['source_code_uri'] = 'https://bitbucket.org/legion-io/lex-tasker/src'
+  spec.metadata['documentation_uri'] = 'https://legionio.atlassian.net/wiki/spaces/LEX/pages/7733408'
+  spec.metadata['changelog_uri'] = 'https://legionio.atlassian.net/wiki/spaces/LEX/pages/12156936'
+  spec.metadata['bug_tracker_uri'] = 'https://bitbucket.org/legion-io/lex-tasker/issues'
   spec.require_paths = ['lib']
 
-  spec.add_development_dependency 'bundler'
+  spec.add_development_dependency 'bundler', '>= 2'
   spec.add_development_dependency 'codecov'
   spec.add_development_dependency 'rake'
   spec.add_development_dependency 'rspec'
+  spec.add_development_dependency 'rspec_junit_formatter'
   spec.add_development_dependency 'rubocop'
-  spec.add_development_dependency 'rubocop-performance'
 end

data/lib/legion/extensions/tasker.rb

@@ -1,10 +1,17 @@
 require 'legion/extensions/tasker/version'
-require 'legion/extensions'
 
 module Legion
   module Extensions
     module Tasker
-      extend Legion::Extensions::Core
+      extend Legion::Extensions::Core if Legion::Extensions.const_defined? :Core
+
+      def self.data_required?
+        true
+      end
+
+      def data_required?
+        true
+      end
     end
   end
 end

data/lib/legion/extensions/tasker/actors/fetch_delayed.rb

@@ -5,9 +5,21 @@ module Legion::Extensions::Tasker
         'fetch'
       end
 
+      def runner_class
+        Legion::Extensions::Tasker::Runners::FetchDelayed
+      end
+
       def use_runner?
         false
       end
+
+      def check_subtask?
+        false
+      end
+
+      def generate_task?
+        false
+      end
     end
   end
 end

data/lib/legion/extensions/tasker/actors/log.rb

@@ -1,10 +1,6 @@
 module Legion::Extensions::Tasker
   module Actor
     class Log < Legion::Extensions::Actors::Subscription
-      def runner_function
-        'add_log'
-      end
-
       def check_subtask?
         false
       end

data/lib/legion/extensions/tasker/actors/task_manager.rb

@@ -0,0 +1,21 @@
+module Legion::Extensions::Tasker
+  module Actor
+    class TaskManager < Legion::Extensions::Actors::Subscription
+      def use_runner?
+        true
+      end
+
+      def check_subtask?
+        true
+      end
+
+      def generate_task?
+        true
+      end
+
+      def prefetch
+        1
+      end
+    end
+  end
+end

data/lib/legion/extensions/tasker/helpers/base.rb

@@ -0,0 +1,11 @@
+module Legion
+  module Extensions
+    module Tasker
+      module Helpers
+        module Base
+          def send_task(**opts); end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/tasker/helpers/fetch_delayed.rb

@@ -0,0 +1,66 @@
+module Legion
+  module Extensions
+    module Tasker
+      module Helpers
+        module FetchDelayed
+          def find_trigger(runner_class:, function:, **)
+            sql = "SELECT `functions`.`id` as `function_id`, `runner_id`, `runners`.`namespace`
+                   FROM `legion`.`functions`
+                     INNER JOIN `legion`.`runners` ON (`functions`.`runner_id` = `runners`.`id`)
+                    WHERE `functions`.`name` = '#{function}'
+                     AND `runners`.`namespace` = '#{runner_class}' LIMIT 1;"
+
+            cache = Legion::Cache.get(sql)
+            return cache unless cache.nil?
+
+            results = Legion::Data::Connection.sequel.fetch(sql).first
+            Legion::Cache.set(sql, results) if results.is_a?(Hash) && results.count.positive?
+            results
+          end
+
+          def find_delayed(**)
+            sql = '
+              SELECT tasks.id, tasks.relationship_id, tasks.function_id, tasks.created,
+                relationships.delay as relationship_delay, relationships.chain_id,
+                functions.name as function_name,
+                runners.namespace as class, runners.id as runner_id,
+                CONCAT( `exchange`, ".",`queue`,".",`functions`.`name`) AS runner_routing_key
+              FROM legion.tasks
+                INNER JOIN legion.functions ON (functions.id = tasks.function_id)
+                INNER JOIN legion.runners ON (runners.id = functions.runner_id)
+                INNER JOIN `legion`.`extensions` ON (`runners`.`extension_id` = `extensions`.`id`)
+                LEFT JOIN legion.relationships ON (relationships.id = tasks.relationship_id)
+              WHERE status = \'task.delayed\';'
+
+            Legion::Data::Connection.sequel.fetch(sql)
+          end
+
+          def find_subtasks(trigger_id:, **)
+            sql = "
+              SELECT
+               `relationships`.`id` as `relationship_id`, `debug`,
+               `chain_id`, `allow_new_chains`, `delay`, `trigger_id`, `action_id`, `conditions`, `transformation`,
+               `runners`.`namespace`, `runners`.`id` as `runner_id`, `runners`.`queue`,
+               `runners`.`namespace` as runner_class,
+               `functions`.`id` as `function_id`, `functions`.`name` as `function`,
+               `extensions`.`exchange`,
+               CONCAT( `exchange`, '.',`queue`,'.',`functions`.`name`) AS runner_routing_key
+              FROM `legion`.`relationships`
+                INNER JOIN `legion`.`functions` ON (`functions`.`id` = `relationships`.`action_id`)
+                INNER JOIN `legion`.`runners` ON (`functions`.`runner_id` = `runners`.`id`)
+                INNER JOIN `legion`.`extensions` ON (`runners`.`extension_id` = `extensions`.`id`)
+              WHERE `relationships`.`trigger_id` = #{trigger_id} AND `relationships`.`active` = 1;
+            "
+
+            cache = Legion::Cache.get(sql)
+            return cache unless cache.nil?
+
+            results = Legion::Data::Connection.sequel.fetch(sql)
+            Legion::Cache.set(sql, results, ttl: 5) if results.is_a?(Hash) && results.count.positive?
+            results
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/tasker/helpers/find_subtask.rb

@@ -0,0 +1,49 @@
+module Legion
+  module Extensions
+    module Tasker
+      module Helpers
+        module FindSubtask
+          def find_trigger(runner_class:, function:, **)
+            sql = "SELECT `functions`.`id` as `function_id`, `runner_id`, `runners`.`namespace`
+                   FROM `legion`.`functions`
+                     INNER JOIN `legion`.`runners` ON (`functions`.`runner_id` = `runners`.`id`)
+                    WHERE `functions`.`name` = '#{function}'
+                     AND `runners`.`namespace` = '#{runner_class}' LIMIT 1;"
+
+            cache = Legion::Cache.get(sql)
+            return cache unless cache.nil?
+
+            results = Legion::Data::Connection.sequel.fetch(sql).first
+            Legion::Cache.set(sql, results) if results.is_a?(Hash) && results.count.positive?
+            results
+          end
+
+          def find_subtasks(trigger_id:, **)
+            sql = "
+              SELECT
+               `relationships`.`id` as `relationship_id`, `debug`,
+               `chain_id`, `allow_new_chains`, `delay`, `trigger_id`, `action_id`, `conditions`, `transformation`,
+               `runners`.`namespace`, `runners`.`id` as `runner_id`, `runners`.`queue`,
+               `runners`.`namespace` as runner_class,
+               `functions`.`id` as `function_id`, `functions`.`name` as `function`,
+               `extensions`.`exchange`,
+               CONCAT( `exchange`, '.',`queue`,'.',`functions`.`name`) AS runner_routing_key
+              FROM `legion`.`relationships`
+                INNER JOIN `legion`.`functions` ON (`functions`.`id` = `relationships`.`action_id`)
+                INNER JOIN `legion`.`runners` ON (`functions`.`runner_id` = `runners`.`id`)
+                INNER JOIN `legion`.`extensions` ON (`runners`.`extension_id` = `extensions`.`id`)
+              WHERE `relationships`.`trigger_id` = #{trigger_id} AND `relationships`.`active` = 1;
+            "
+
+            cache = Legion::Cache.get(sql)
+            return cache unless cache.nil?
+
+            results = Legion::Data::Connection.sequel.fetch(sql)
+            Legion::Cache.set(sql, results, ttl: 5) if results.is_a?(Hash) && results.count.positive?
+            results
+          end
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/tasker/runners/check_subtask.rb

@@ -4,114 +4,93 @@ module Legion::Extensions::Tasker
   module Runners
     module CheckSubtask
       include Legion::Extensions::Helpers::Lex
+      extend Legion::Extensions::Tasker::Helpers::FindSubtask
 
       def check_subtasks(runner_class:, function:, **opts)
-        runner_record = Legion::Data::Model::Runner[namespace: runner_class]
-        return if runner_record.nil?
+        trigger = find_trigger(runner_class: runner_class, function: function)
 
-        function_record = runner_record.functions_dataset[name: function]
-        return if function_record.nil?
-
-        relationships = function_record.trigger_relationships_dataset.where(:active)
-        relationships.where(chain_id: opts[:chain_id] || :allow_new_chains) if opts.key? :chain_id
-        return { success: true, count: relationships.count } if relationships.count.zero?
-
-        relationships.each do |relationship|
-          unless relationship.values[:allow_new_chains]
-            next if relationship.chain.nil?
+        find_subtasks(trigger_id: trigger[:function_id]).each do |relationship|
+          unless relationship[:allow_new_chains]
+            next if relationship[:chain_id].nil?
             next unless opts.key? :chain_id
-            next unless relationship.values[:chain_id] == opts[:chain_id]
+            next unless relationship[:chain_id] == opts[:chain_id]
           end
 
-          action_function = relationship.action
-          action_runner = action_function.runner
-
-          status = relationship.values[:delay].zero? ? 'conditioner.queued' : 'task.delayed'
-
-          task_id_hash = { runner_class:    action_runner.values[:namespace],
-                           function:        action_function.values[:name],
-                           status:          status,
-                           relationship_id: relationship.values[:id] }
-          task_id_hash[:payload] = opts
+          task_hash = relationship
+          task_hash[:status] = relationship[:delay].zero? ? 'conditioner.queued' : 'task.delayed'
+          task_hash[:payload] = opts
 
           if opts.key? :master_id
-            task_id_hash[:master_id] = opts[:master_id]
+            task_hash[:master_id] = opts[:master_id]
           elsif opts.key? :parent_id
-            task_id_hash[:master_id] = opts[:parent_id]
+            task_hash[:master_id] = opts[:parent_id]
           elsif opts.key? :task_id
-            task_id_hash[:master_id] = opts[:task_id]
+            task_hash[:master_id] = opts[:task_id]
           end
 
-          task_id_hash[:parent_id] = opts[:task_id] if opts.key? :task_id
+          task_hash[:parent_id] = opts[:task_id] if opts.key? :task_id
+          task_hash[:routing_key] = if relationship[:conditions].is_a?(String) && relationship[:conditions].length > 4
+                                      'task.subtask.conditioner'
+                                    elsif relationship[:transformation].is_a?(String) && relationship[:transformation].length > 4 # rubocop:disable Layout/LineLength
+                                      'task.subtask.transformation'
+                                    else
+                                      relationship[:runner_routing_key]
+                                    end
+
           if opts[:result].is_a? Array
             opts[:result].each do |result|
-              send_task(task_id_hash,
-                        relationship:    relationship,
-                        runner_record:   runner_record,
-                        function_record: function_record,
-                        action_function: action_function,
-                        action_runner:   action_runner,
-                        result:          result)
+              send_task(results:             result,
+                        trigger_runner_id:   trigger[:runner_id],
+                        trigger_function_id: trigger[:function_id],
+                        **task_hash)
             end
           else
-            send_task(task_id_hash,
-                      relationship:    relationship,
-                      runner_record:   runner_record,
-                      function_record: function_record,
-                      action_function: action_function,
-                      action_runner:   action_runner,
-                      **opts)
+            results = if opts[:results].is_a? Hash
+                        opts[:results]
+                      elsif opts[:result].is_a? Hash
+                        opts[:result]
+                      else
+                        opts
+                      end
+            send_task(
+              results:             results,
+              trigger_runner_id:   trigger[:runner_id],
+              trigger_function_id: trigger[:function_id],
+              **task_hash
+            )
           end
         end
-      rescue StandardError => e
-        Legion::Logging.fatal e.message
-        Legion::Logging.fatal e.backtrace
-        Legion::Logging.fatal runner_class
-        Legion::Logging.fatal function
-        Legion::Logging.fatal opts.keys
-        Legion::Logging.fatal opts[:entry]
       end
 
-      def send_task(task_id_hash, relationship:, runner_record:, function_record:, action_function:, action_runner:, **opts) # rubocop:disable Layout/LineLength
-        task_id = Legion::Runner::Status.generate_task_id(**task_id_hash)[:task_id]
-
-        return { status: true } unless relationship.values[:delay].zero?
+      def send_task(**opts)
+        opts[:results] = opts[:result] if opts.key?(:result) && !opts.key?(:results)
+        opts[:success] = if opts.key?(:result) && opts.key?(:success)
+                           opts[:result][:success]
+                         elsif opts.key?(:success)
+                           opts[:success]
+                         else
+                           1
+                         end
 
-        subtask_hash = {
-          relationship_id:     relationship.values[:id],
-          chain_id:            relationship.values[:chain_id],
-          trigger_runner_id:   runner_record.values[:id],
-          trigger_function_id: function_record.values[:id],
-          function_id:         action_function.values[:id],
-          function:            action_function.values[:name],
-          runner_id:           action_runner.values[:id],
-          runner_class:        action_runner.values[:namespace],
-          conditions:          relationship.values[:conditions],
-          transformation:      relationship.values[:transformation],
-          debug:               relationship.values[:debug] && 1 || 0,
-          task_id:             task_id,
-          results:             opts[:result]
-        }
+        # opts[:task_id] = Legion::Runner::Status.generate_task_id(**opts)[:task_id]
+        opts[:task_id] = insert_task(**opts)
+        return { status: true } unless opts[:delay].zero?
 
-        subtask_hash[:routing_key] = if subtask_hash[:conditions].is_a?(String) && subtask_hash[:conditioners].length > 4 # rubocop:disable Layout/LineLength
-                                       'task.subtask.conditioner'
-                                     elsif subtask_hash[:transformation].is_a?(String) && subtask_hash[:transformation].length > 4 # rubocop:disable Layout/LineLength
-                                       'task.subtask.transform'
-                                     else
-                                       "#{runner_record.extension.values[:exchange]}.#{runner_record.values[:queue]}.#{subtask_hash[:function]}" # rubocop:disable Layout/LineLength
-                                     end
+        Legion::Transport::Messages::SubTask.new(**opts).publish
+      end
 
-        subtask_hash[:success] = if opts.nil?
-                                   1
-                                 elsif opts.key?(:result)
-                                   # opts[:result][:success]
-                                   1
-                                 elsif opts.key?(:success)
-                                   opts[:success]
-                                 else
-                                   1
-                                 end
-        Legion::Transport::Messages::SubTask.new(**subtask_hash).publish
+      def insert_task(relationship_id:, function_id:, status: 'task.queued', master_id: nil, parent_id: nil, **opts)
+        insert_hash = { relationship_id: relationship_id, function_id: function_id, status: status }
+        insert_hash[:master_id] = if master_id.is_a? Integer
+                                    master_id
+                                  elsif parent_id.is_a? Integer
+                                    parent_id
+                                  end
+        insert_hash[:parent_id] = parent_id if parent_id.is_a? Integer
+        insert_hash[:payload] = Legion::JSON.dump(opts)
+        # insert_hash[:function_args] = nil
+        # insert_hash[:results] = nil
+        Legion::Data::Model::Task.insert(insert_hash)
       end
     end
   end

data/lib/legion/extensions/tasker/runners/fetch_delayed.rb

@@ -1,50 +1,71 @@
 module Legion::Extensions::Tasker::Runners
   module FetchDelayed
-    include Legion::Extensions::Helpers::Lex
+    extend Legion::Extensions::Tasker::Helpers::FetchDelayed
+    include Legion::Extensions::Helpers::Task
 
     def fetch(**_opts)
-      tasks = Legion::Data::Model::Task.where(status: 'task.delayed')
-      tasks_pushed = []
-      log.debug "tasks.count = #{tasks.count}"
-      tasks.each do |task|
-        relationship = task.relationship
-        next if !task.relationship.nil? && Time.now < task.values[:created] + relationship.values[:delay]
-
-        # next if Time.now < task.values[:created] + task.values[:delay]
-
-        subtask = Legion::Transport::Messages::SubTask.new(
-          relationship_id:     relationship.values[:id],
-          chain_id:            relationship.values[:chain_id],
-          trigger_runner_id:   relationship.trigger.runner.values[:id],
-          trigger_function_id: relationship.values[:trigger_id],
-          function_id:         relationship.action.values[:id],
-          function:            relationship.action.values[:name],
-          runner_id:           relationship.action.values[:runner_id],
-          runner_class:        relationship.action.runner.values[:namespace],
-          conditions:          relationship.values[:conditions],
-          transformation:      relationship.values[:transformation],
-          # debug:                relationship.values[:debug],
-          task_id:             task.values[:id]
-          # results:              task.values[:payload]
-        )
-        subtask.publish
-        task.update(status: 'conditioner.queued')
-        tasks_pushed.push(task.values[:id])
-      rescue StandardError => e
-        task.update(status: 'task.push_exception')
-        log.error e.message
-        log.error e.backtrace
+      find_delayed.each do |task|
+        if task[:relationship_delay].is_a?(Integer) && task[:relationship_delay].positive?
+          next if Time.now < task[:created] + task[:relationship_delay] # rubocop:disable Style/SoleNestedConditional
+        end
+
+        if task[:task_delay].is_a?(Integer) && task[:task_delay].positive?
+          next if Time.now < task[:created] + task[:task_delay] # rubocop:disable Style/SoleNestedConditional
+        end
+
+        subtask_hash = {
+          relationship_id: task[:relationship_id],
+          chain_id:        task[:chain_id],
+          function_id:     task[:function_id],
+          function:        task[:function_name],
+          runner_id:       task[:runner_id],
+          runner_class:    task[:runner_class],
+          task_id:         task[:id],
+          exchange:        task[:exchange],
+          queue:           task[:queue]
+        }
+
+        subtask_hash[:conditions] = task[:conditions] if task[:conditions].is_a?(String)
+        subtask_hash[:transformation] = task[:transformation] if task[:transformation].is_a?(String)
+
+        subtask_hash[:routing_key] = if task[:conditions].is_a?(String) && task[:conditions].length > 4
+                                       'task.subtask.conditioner'
+                                     elsif task[:transformation].is_a?(String) && task[:transformation].length > 4
+                                       'task.subtask.transformation'
+                                     else
+                                       task[:runner_routing_key]
+                                     end
+
+        send_task(**subtask_hash)
+        case subtask_hash[:routing_key]
+        when 'task.subtask.conditioner'
+          task_update(task[:id], 'conditioner.queued')
+        when 'task.subtask.transformation'
+          task_update(task[:id], 'transformer.queued')
+        else
+          task_update(task[:id], 'task.queued')
+        end
       end
+    end
 
-      { success: true, count: tasks_pushed.count, tasks: tasks_pushed }
-    rescue StandardError => e
-      Legion::Logging.error e.message
-      Legion::Logging.error e.backtrace
+    def send_task(**opts)
+      opts[:results] = opts[:result] if opts.key?(:result) && !opts.key?(:results)
+      opts[:success] = if opts.key?(:result) && opts.key?(:success)
+                         opts[:result][:success]
+                       elsif opts.key?(:success)
+                         opts[:success]
+                       else
+                         1
+                       end
+      log.debug 'pushing delayed task to worker'
+      Legion::Transport::Messages::Task.new(**opts).publish
     end
 
     def push(**_opts)
       Legion::Extensions::Tasker::Transport::Messages::FetchDelayed.new.publish
       { success: true }
     end
+
+    include Legion::Extensions::Helpers::Lex
   end
 end

data/lib/legion/extensions/tasker/runners/task_manager.rb

@@ -0,0 +1,28 @@
+module Legion
+  module Extensions
+    module Tasker
+      module Runners
+        module TaskManager
+          def purge_old(age: 31, limit: 100, status: 'task.completed', **_opts)
+            log.debug("purging old completed tasks with an age > #{age} days, limit: #{limit}")
+            dataset = Legion::Data::Model::Task
+                      .where(Sequel.lit("created <= DATE_SUB(SYSDATE(), INTERVAL #{age} DAY)"))
+                      .limit(limit)
+            dataset.where(status: status) unless ['*', nil, ''].include? status
+            log.debug "Deleting #{dataset.count} records" if dataset.count.positive?
+            dataset&.delete
+          end
+
+          def expire_queued(age: 1, limit: 10, **) # rubocop:disable Lint/UnusedMethodArgument
+            dataset = Legion::Data::Model::Task # rubocop:disable Lint/UselessAssignment
+                      .where(status: ['conditioner.queued', 'transformer.queued', 'task.queued'])
+                      .limit(limit)
+          end
+
+          include Legion::Extensions::Helpers::Task
+          include Legion::Extensions::Helpers::Lex
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/tasker/runners/updater.rb

@@ -5,6 +5,12 @@ module Legion::Extensions::Tasker
 
       def update_status(task_id:, **opts)
         task = Legion::Data::Model::Task[task_id]
+        if task.nil? || task.values.nil?
+          return { success: false, changed: false, task_id: task_id, message: 'task nil' }
+        end
+
+        log.unknown task.class
+
         update_hash = {}
         %i[status function_args payload results].each do |column|
           next unless opts.key? column

data/lib/legion/extensions/tasker/transport.rb

@@ -19,6 +19,10 @@ module Legion::Extensions::Tasker
         }, {
           to:          'fetch_delayed',
           routing_key: 'fetch.delayed'
+        }, {
+          from:        'tasker',
+          to:          'task_manager',
+          routing_key: 'task.task_manager.#'
         }
       ]
     end

data/lib/legion/extensions/tasker/transport/queues/check_subtask.rb

@@ -7,7 +7,10 @@ module Legion::Extensions::Tasker
         end
 
         def queue_options
-          { auto_delete: false }
+          {
+            auto_delete: false,
+            arguments:   { 'x-max-priority': 255 }
+          }
         end
       end
     end

data/lib/legion/extensions/tasker/transport/queues/fetch_delayed.rb

@@ -5,6 +5,16 @@ module Legion::Extensions::Tasker
         def queue_name
           'task.fetch_delayed'
         end
+
+        def queue_options
+          {
+            arguments: {
+              'x-single-active-consumer': true,
+              'x-max-priority':           255,
+              'x-message-ttl':            1
+            }
+          }
+        end
       end
     end
   end

data/lib/legion/extensions/tasker/transport/queues/task_mananger.rb

@@ -0,0 +1,15 @@
+module Legion::Extensions::Tasker
+  module Transport
+    module Queues
+      class TaskManager < Legion::Transport::Queue
+        def queue_name
+          'tasker.task_manager'
+        end
+
+        def queue_options
+          { arguments: { 'x-single-active-consumer': true } }
+        end
+      end
+    end
+  end
+end

data/lib/legion/extensions/tasker/version.rb

@@ -1,7 +1,7 @@
 module Legion
   module Extensions
     module Tasker
-      VERSION = '0.1.3'.freeze
+      VERSION = '0.2.0'.freeze
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: lex-tasker
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.2.0
 platform: ruby
 authors:
-- Miverson
-autorequire:
+- Esity
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-10-15 00:00:00.000000000 Z
+date: 2020-11-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '2'
 - !ruby/object:Gem::Dependency
   name: codecov
   requirement: !ruby/object:Gem::Requirement
@@ -67,7 +67,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: rubocop
+  name: rspec_junit_formatter
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -81,7 +81,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: rubocop-performance
+  name: rubocop
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -106,21 +106,28 @@ files:
 - ".rspec"
 - ".rubocop.yml"
 - Gemfile
+- Gemfile.lock
 - LICENSE.txt
 - README.md
 - Rakefile
 - bin/console
 - bin/setup
+- bitbucket-pipelines.yml
 - lex-tasker.gemspec
 - lib/legion/extensions/tasker.rb
 - lib/legion/extensions/tasker/actors/check_subtask.rb
 - lib/legion/extensions/tasker/actors/fetch_delayed.rb
 - lib/legion/extensions/tasker/actors/fetch_delayed_push.rb
 - lib/legion/extensions/tasker/actors/log.rb
+- lib/legion/extensions/tasker/actors/task_manager.rb
 - lib/legion/extensions/tasker/actors/updater.rb
+- lib/legion/extensions/tasker/helpers/base.rb
+- lib/legion/extensions/tasker/helpers/fetch_delayed.rb
+- lib/legion/extensions/tasker/helpers/find_subtask.rb
 - lib/legion/extensions/tasker/runners/check_subtask.rb
 - lib/legion/extensions/tasker/runners/fetch_delayed.rb
 - lib/legion/extensions/tasker/runners/log.rb
+- lib/legion/extensions/tasker/runners/task_manager.rb
 - lib/legion/extensions/tasker/runners/updater.rb
 - lib/legion/extensions/tasker/transport.rb
 - lib/legion/extensions/tasker/transport/exchanges/task.rb
@@ -130,13 +137,19 @@ files:
 - lib/legion/extensions/tasker/transport/queues/lex_register.rb
 - lib/legion/extensions/tasker/transport/queues/subtask.rb
 - lib/legion/extensions/tasker/transport/queues/task_log.rb
+- lib/legion/extensions/tasker/transport/queues/task_mananger.rb
 - lib/legion/extensions/tasker/transport/queues/updater.rb
 - lib/legion/extensions/tasker/version.rb
 homepage: https://bitbucket.org/legion-io/lex-tasker
 licenses:
 - MIT
-metadata: {}
-post_install_message:
+metadata:
+  homepage_uri: https://bitbucket.org/legion-io/lex-tasker
+  source_code_uri: https://bitbucket.org/legion-io/lex-tasker/src
+  documentation_uri: https://legionio.atlassian.net/wiki/spaces/LEX/pages/7733408
+  changelog_uri: https://legionio.atlassian.net/wiki/spaces/LEX/pages/12156936
+  bug_tracker_uri: https://bitbucket.org/legion-io/lex-tasker/issues
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -151,8 +164,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key:
+rubygems_version: 3.1.4
+signing_key: 
 specification_version: 4
 summary: LEX::Tasker manages tasks status from Legion
 test_files: []