qyu 1.0.0

data/lib/qyu/workers/concerns/callback.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Qyu
+  module Workers
+    module Concerns
+      module Callback
+        # Meant to add callbacks to Qyu::Worker
+        #
+        # Usage:
+        #
+        # Qyu::Worker.new do
+        #   callback :execute, :after do
+        #     # Do something after execution
+        #   end
+        # end
+        #
+
+        def callback(method, type, &block)
+          @_callbacks ||= {}
+          @_callbacks[method] ||= {}
+          @_callbacks[method][type] = block
+        end
+
+        def run_callbacks(method, &block)
+          find_callback(method, :before)&.call
+          find_callback(method, :around) ? find_callback(method, :around).call(block) : yield
+          find_callback(method, :after)&.call
+        end
+
+        private
+
+        def find_callback(method, type)
+          @_callbacks.dig(method, type) if @_callbacks.is_a?(Hash)
+        end
+      end
+    end
+  end
+end

data/lib/qyu/workers/concerns/failure_queue.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Qyu
+  module Workers
+    module Concerns
+      # Qyu::Concerns::FailureQueue
+      module FailureQueue
+        # Adds ability to workers enqueue failed task to another queue
+        #
+        # Qyu::Worker.new do
+        #   failure_queue true
+        #   # or
+        #   failure_queue false
+        # end
+        #
+
+        def failure_queue(fq)
+          @failure_queue = fq
+        end
+      end
+    end
+  end
+end

data/lib/qyu/workers/concerns/payload_validator.rb

@@ -0,0 +1,124 @@
+# frozen_string_literal: true
+
+module Qyu
+  module Workers
+    module Concerns
+      # Qyu::Concerns::PayloadValidator
+      module PayloadValidator
+        # Adds ability to workers to perform validations on
+        # params for the task to be processed
+        #
+        # Usage:
+        #
+        # Qyu::Worker.new do
+        #   validates :user_id,    presence: true, type: :integer, unless: :no_user
+        #   validates :name,       presence: true, type: :string
+        #   validates :account_id, absence: true,  if: :customer_id
+        #   validates :account_id, presence: true, unless: :customer_id
+        # end
+        #
+
+        def validates(parameter, opts = {})
+          @_validations ||= {}
+          @_validations[parameter.to_s] = Qyu::Utils.stringify_hash_keys(opts)
+        end
+
+        def validate_payload!(model)
+          return unless @_validations
+          payload = Qyu::Utils.stringify_hash_keys(model.payload || {})
+          validation_errors = {}
+          @_validations.each do |attribute, opts|
+            # example: attribute :name
+            # example opts { presence: true, type: integer }
+            next unless if_validation(payload, opts['if'])
+            next unless unless_validation(payload, opts['unless'])
+            opts.map do |option, value|
+              error = run_validation(option, payload[attribute.to_s], value)
+              # next if error is nil
+              next unless error
+              validation_errors["#{attribute}.#{option}"] = error
+            end
+          end
+
+          if validation_errors.size.positive?
+            fail Qyu::Errors::PayloadValidationError, validation_errors
+          end
+          nil
+        end
+
+        private
+
+        def run_validation(option, param, value)
+          # Skip if and unless conditionals (return nil)
+          return if option.eql?('if')
+          return if option.eql?('unless')
+          # supported options are presence, absence and type
+          __send__(option, param, value)
+        rescue NoMethodError
+          raise Qyu::Errors::UnknownValidationOption, option
+        end
+
+        def presence(param, value)
+          return unless value
+          return unless param.nil?
+          { expected: 'present' }
+        end
+
+        def absence(param, value)
+          return unless value
+          return if param.nil?
+          { expected: 'absent' }
+        end
+
+        def type(param, data_type)
+          __send__("#{data_type}_type_validator", param)
+        end
+
+        # DataType validators
+        def integer_type_validator(param)
+          type_validator('integer', [Integer], param.class)
+        end
+
+        def string_type_validator(param)
+          type_validator('string', [String, Symbol], param.class)
+        end
+
+        def number_type_validator(param)
+          type_validator('number', [Integer, Float], param.class)
+        end
+
+        def boolean_type_validator(param)
+          type_validator('boolean', [TrueClass, FalseClass], param.class)
+        end
+
+        def hash_type_validator(param)
+          type_validator('hash', [Hash], param.class)
+        end
+
+        def array_type_validator(param)
+          type_validator('array', [Array], param.class)
+        end
+
+        def type_validator(type_name, data_types, param_class)
+          return if data_types.include?(param_class)
+          { expected: type_name, got: param_class.name.downcase }
+        end
+
+        # Conditonal validation
+        def if_validation(payload, key)
+          # TODO: support block passing "return yield if block_given?"
+          return true if key.nil?
+          return true if payload[key.to_s]
+          false
+        end
+
+        def unless_validation(payload, key)
+          # TODO: support block passing "return yield if block_given?"
+          return true if key.nil?
+          return true unless payload[key.to_s]
+          false
+        end
+      end
+    end
+  end
+end

data/lib/qyu/workers/sync.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module Qyu
+  module Workers
+    class Sync < Base
+      def work(queue_name)
+        super do |task|
+          job = task.job
+          task_names_to_wait_for = job.tasks_to_wait_for(task)
+          log(:debug, "Task names to wait for: #{task_names_to_wait_for}")
+          task_names_to_wait_for.each do |task_name|
+            sync_condition = job.sync_condition(task, task_name)
+            log(:debug, "Task: #{task_name}, Sync condition: #{sync_condition}")
+            if respond_to?(sync_condition['function'], true)
+              __send__(sync_condition['function'], job, task, task_name, sync_condition['param'])
+            else
+              fail Qyu::Errors::NotImplementedError
+            end
+          end
+        end
+      end
+
+      private
+
+      def eq_completed(job, task, task_name_to_wait_for, sync_param_name)
+        sync_param_value = task.payload[sync_param_name]
+        log(:debug, "Task: #{task_name_to_wait_for}, Sync param value: #{sync_param_value}")
+        parent_task_id = task.parent_task_id
+        log(:debug, "Task: #{task_name_to_wait_for}, Parent task ID: #{parent_task_id}")
+        task_ids = job.find_task_ids_by_name_and_ancestor_task_id(task_name_to_wait_for, parent_task_id)
+        log(:debug, "Task: #{task_name_to_wait_for}, Task IDs: #{task_ids}")
+
+        if task_ids.size < sync_param_value
+          log(:debug, 'Re-enqueuing sync task')
+          fail Qyu::Errors::UnsyncError
+        end
+
+        check_completion!(task_ids)
+      end
+
+      def completed(job, task, task_name_to_wait_for, _sync_param_name)
+        parent_task_id = task.parent_task_id
+        log(:debug, "Task: #{task_name_to_wait_for}, Parent task ID: #{parent_task_id}")
+        task_ids = job.find_task_ids_by_name_and_ancestor_task_id(task_name_to_wait_for, parent_task_id)
+        log(:debug, "Task: #{task_name_to_wait_for}, Task IDs: #{task_ids}")
+        if task_ids.empty?
+          log(:debug, 'Re-enqueuing sync task')
+          fail Qyu::Errors::UnsyncError
+        end
+        check_completion!(task_ids)
+      end
+
+      def check_completion!(task_ids)
+        task_ids.each do |task_id|
+          state = Qyu::Status.find(task_id)
+          log(:debug, "[CHECK_COMPLETION] Task ID: #{task_id}, Status: #{state.status}")
+          next if state.completed?
+          fail Qyu::Errors::UnsyncError
+        end
+      end
+    end
+  end
+end

data/qyu.gemspec

@@ -0,0 +1,36 @@
+
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "qyu/version"
+
+Gem::Specification.new do |spec|
+  spec.name                  = "qyu"
+  spec.version               = Qyu::VERSION
+  spec.authors               = ['Elod Peter', 'Mohamed Osama']
+  spec.email                 = ['bejmuller@gmail.com', 'mohamed.o.alnagdy@gmail.com']
+
+  spec.summary               = 'Distributed task execution system for complex workflows'
+  spec.description           = 'Distributed task execution system for complex workflows'
+  spec.homepage              = 'https://github.com/FindHotel/qyu'
+  spec.required_ruby_version = '>= 2.4'
+
+  spec.files         = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency 'activesupport', '~> 5.1'
+
+  spec.add_development_dependency 'bundler', '~> 1.16'
+  spec.add_development_dependency 'dotenv'
+  spec.add_development_dependency 'rack-test'
+  spec.add_development_dependency 'pry'
+  spec.add_development_dependency 'pry-byebug'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'rspec', '~> 3.5'
+  spec.add_development_dependency 'simplecov'
+  spec.add_development_dependency 'sinatra'
+  spec.add_development_dependency 'timecop', '~> 0.8'
+end

metadata

@@ -0,0 +1,278 @@
+--- !ruby/object:Gem::Specification
+name: qyu
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Elod Peter
+- Mohamed Osama
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2018-02-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.1'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+- !ruby/object:Gem::Dependency
+  name: dotenv
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rack-test
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry-byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.5'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sinatra
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: timecop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+description: Distributed task execution system for complex workflows
+email:
+- bejmuller@gmail.com
+- mohamed.o.alnagdy@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- CODE_OF_CONDUCT.md
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- bin/console
+- bin/server
+- bin/setup
+- examples/bin/simple
+- examples/config.rb
+- examples/simple/create_workflow.rb
+- examples/simple/enqueue_job.rb
+- examples/simple/worker.rb
+- lib/qyu.rb
+- lib/qyu/config.rb
+- lib/qyu/errors.rb
+- lib/qyu/errors/base.rb
+- lib/qyu/errors/could_not_fetch_task.rb
+- lib/qyu/errors/invalid_queue_name.rb
+- lib/qyu/errors/invalid_task_attributes.rb
+- lib/qyu/errors/job_not_found.rb
+- lib/qyu/errors/lock_already_acquired.rb
+- lib/qyu/errors/lock_not_acquired.rb
+- lib/qyu/errors/message_not_received.rb
+- lib/qyu/errors/not_implemented_error.rb
+- lib/qyu/errors/payload_validation_error.rb
+- lib/qyu/errors/task_not_found.rb
+- lib/qyu/errors/task_status_update_failed.rb
+- lib/qyu/errors/unknown_validation_option.rb
+- lib/qyu/errors/unsync_error.rb
+- lib/qyu/errors/workflow_descriptor_validation_error.rb
+- lib/qyu/errors/workflow_not_found.rb
+- lib/qyu/factory.rb
+- lib/qyu/models.rb
+- lib/qyu/models/concerns/workflow_descriptor_validator.rb
+- lib/qyu/models/enums/status.rb
+- lib/qyu/models/job.rb
+- lib/qyu/models/task.rb
+- lib/qyu/models/workflow.rb
+- lib/qyu/queue.rb
+- lib/qyu/queue/base.rb
+- lib/qyu/queue/memory/adapter.rb
+- lib/qyu/store.rb
+- lib/qyu/store/base.rb
+- lib/qyu/store/memory/adapter.rb
+- lib/qyu/ui.rb
+- lib/qyu/ui/helpers/pagination.rb
+- lib/qyu/ui/public/bootstrap.min.css
+- lib/qyu/ui/public/paper-dashboard.css
+- lib/qyu/ui/public/script.js
+- lib/qyu/ui/public/style.css
+- lib/qyu/ui/views/footer.erb
+- lib/qyu/ui/views/helpers/pagination.erb
+- lib/qyu/ui/views/jobs.erb
+- lib/qyu/ui/views/kaminari/_first_page.html.erb
+- lib/qyu/ui/views/kaminari/_gap.html.erb
+- lib/qyu/ui/views/kaminari/_last_page.html.erb
+- lib/qyu/ui/views/kaminari/_next_page.html.erb
+- lib/qyu/ui/views/kaminari/_page.html.erb
+- lib/qyu/ui/views/kaminari/_paginator.html.erb
+- lib/qyu/ui/views/kaminari/_prev_page.html.erb
+- lib/qyu/ui/views/layout.erb
+- lib/qyu/ui/views/navbar.erb
+- lib/qyu/ui/views/pagination.erb
+- lib/qyu/ui/views/show_job.erb
+- lib/qyu/ui/views/sidebar.erb
+- lib/qyu/ui/views/task_row.erb
+- lib/qyu/utils.rb
+- lib/qyu/version.rb
+- lib/qyu/workers.rb
+- lib/qyu/workers/base.rb
+- lib/qyu/workers/concerns/callback.rb
+- lib/qyu/workers/concerns/failure_queue.rb
+- lib/qyu/workers/concerns/payload_validator.rb
+- lib/qyu/workers/sync.rb
+- qyu.gemspec
+homepage: https://github.com/FindHotel/qyu
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '2.4'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.4
+signing_key: 
+specification_version: 4
+summary: Distributed task execution system for complex workflows
+test_files: []