qyu 1.0.1 → 1.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA256:
-  metadata.gz: f98e891b68c01847f3ca695ed2fb92823c2c2bbfc57081e84fe47ee8709adde6
-  data.tar.gz: 2a166836847513af0227096430f9a7e08c0755fcc2eb20f77c96cc5726b5ffc5
+SHA1:
+  metadata.gz: 856ddbf8f80f887606bc72bc83bc901f5c8516a2
+  data.tar.gz: 026e1996fb30e4e3d8fe45febefabae331196a61
 SHA512:
-  metadata.gz: bbcfd3dfffc5e084a1b250c15d06772b6b6acb8c0178d02e3fbdbe3538946397c27bd55ab18e11f91fe83cdb926612d00df9ca48eebcb7496e54e8dacd7d271a
-  data.tar.gz: 0a5ddbd2392e6a317c094d4ae5126b63f411f953cc50cda9ec57fc5d38621a42b0536c3143dd9baf802f0e313f08894838ced017eeeba95c9823ba1f5a3b809f
+  metadata.gz: 56aef9a753f450f32e9e33fa398b6758bee60dcc62e7b79aa04527d623f40416a117bc343ac94c45f020c9c54da706ae6cefa86e687ae0a001e59df144f83131
+  data.tar.gz: 8e7b894defa30e3e183d90db0be04ac8b88a6a58bc1b3b6d66adca8505d9b530a8cd6d029240b1a8b6b1252fed6adcd823ef8f365906a618a99378a3ee01ccf7

data/.gitignore

@@ -54,3 +54,6 @@ Gemfile.lock
 
 # byebug history
 .byebug_history
+
+# macOS Finder
+.DS_Store

data/CHANGELOG.md

@@ -0,0 +1,31 @@
+# Changelog
+- All notable changes to this project will be documented in this file.
+- This project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
+
+---
+
+[Unreleased]: https://github.com/QyuTeam/qyu/compare/v1.0.2...HEAD
+## [Unreleased]
+
+[v1.0.2]: https://github.com/QyuTeam/qyu/compare/v1.0.1...v1.0.2
+## [v1.0.2]
+### Added
+- Added first `CHANGELOG.md`
+- [TESTS] WorkflowDescriptorValidator tests
+
+### Changed
+- [BUGFIX] WorkflowDescriptorValidator: Validation was not working due to a missed `&&`
+- [BUGFIX] Fix error name `Qyu::Errors::WorkflowDescriptorValidationError`
+
+---
+
+[Template]: https://github.com/QyuTeam/qyu/compare/release-1...release-2
+## [Template]
+### Added
+- Added something
+
+### Changed
+- Changed something
+
+### Removed
+- Removed something

data/README.md

@@ -2,6 +2,8 @@
 
 [![Gem Version](https://img.shields.io/gem/v/qyu.svg)](https://rubygems.org/gems/qyu)
 [![Build Status](https://travis-ci.org/FindHotel/qyu.svg)](https://travis-ci.org/FindHotel/qyu)
+[![Maintainability](https://api.codeclimate.com/v1/badges/88b8e0a8621d1da5c237/maintainability)](https://codeclimate.com/github/QyuTeam/qyu/maintainability)
+[![Press](https://img.shields.io/badge/RubyWeekly-Blog%20Post-green.svg)](https://rubyweekly.com/link/44495/web)
 
 ## Requirements:
 * Ruby 2.4.0 or newer
@@ -39,7 +41,49 @@ Qyu.configure(
 ```
 
 ## Usage
-TODO: Write usage instructions here
+[1] Connect all instances to the same state store and message queue  
+[2] Create a workflow  
+[3] Initialize a worker as follows  
+
+```ruby
+w = Qyu::Worker.new do
+  # callbacks
+  callback :execute, :before do
+    Qyu.logger.info 'Waiting for task..'
+  end
+
+  callback :execute, :after do
+    Qyu.logger.info 'Done'
+  end
+
+  # payload validation
+  validates :times, presence: true, type: :integer
+
+  # failure queue
+  failure_queue false
+end
+
+w.work('queue-name') do |task|
+  # to get the payload passed to the task
+  task.payload
+  # = { 'param_1' => true, 'param_2': [10, 11, 12], 'param_3' => 2 }
+
+  # get the job
+  task.job
+
+  # to manually start a task
+  task.job.create_task(task, 'next-task-name', payload)
+rescue StandardError => ex
+  # If you rescue the error for debugging or reporting purposes, you have to raise it at the end
+  #
+  # do something
+  #
+  raise ex
+end
+```
+
+[4] Start worker  
+[5] Start creating Jobs using the previously created workflow
 
 ## Plugins
 The memory queue and store is just for testing purposes. For production; use one of the following:
@@ -87,4 +131,4 @@ To install this gem onto your local machine, run `bundle exec rake install`. To
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/FindHotel/qyu.
+Bug reports and pull requests are welcome on GitHub at https://github.com/QyuTeam/qyu.

data/lib/qyu/config.rb

@@ -7,14 +7,25 @@ module Qyu
 
     class ServiceConfig
       class << self
+        # Register a service (Queue/Store)
+        #
+        # @param adapter_class [Class] service Class to Register
+        # @return [Class] service Class to Register
         def register(adapter_class)
           types[adapter_class::TYPE] = adapter_class
         end
 
+        # Validate provided config
+        #
+        # @param config [Hash] configuration
+        # @return [Boolean]
         def valid?(config)
           types[config[:type]].valid_config?(config)
         end
 
+        # Get registered services
+        #
+        # @return [Hash] registereds services
         def types
           @__types ||= {}
         end

data/lib/qyu/errors/workflow_descriptor_validation_error.rb

@@ -2,11 +2,11 @@
 
 module Qyu
   module Errors
-    # Qyu::Errors::WorkflowDescriptorValidatorationError
-    class WorkflowDescriptorValidatorationError < Base
+    # Qyu::Errors::WorkflowDescriptorValidationError
+    class WorkflowDescriptorValidationError < Base
       attr_reader :validation_errors
       def initialize(validation_errors)
-        super('Invalid Job descriptor.')
+        super("invalid workflow descriptor: #{validation_errors.join(', ')}")
         @validation_errors = validation_errors
       end
     end

data/lib/qyu/factory.rb

@@ -2,17 +2,27 @@
 
 module Qyu
   # Qyu::Factory
-  class Factory
+  class Factory #:nodoc:
     class ServiceFactory
       class << self
+        # Register a service (Queue/Store)
+        #
+        # @param adapter_class [Class] service Class to Register
+        # @return [Class] service Class to Register
         def register(adapter_class)
           types[adapter_class::TYPE] = adapter_class
         end
 
+        # Get registered services
+        #
+        # @return [Hash] registereds services
         def types
           @__types ||= {}
         end
 
+        # Initialize Queue/Store service with provided configuration
+        #
+        # @return [Object] an adapter
         def get(config)
           Qyu.logger.info "Got factory #{types[config[:type]]}"
           types[config[:type]].new(config)

data/lib/qyu/models/concerns/workflow_descriptor_validator.rb

@@ -12,6 +12,9 @@ module Qyu
         @errors = []
       end
 
+      # validates a workflow's descriptor
+      #
+      # @return [Boolean]
       def valid?
         validate
         @errors.empty?
@@ -28,19 +31,19 @@ module Qyu
 
         tasks.keys.each do |task_name|
           unless validate_queue_presence(task_name)
-            @errors << "#{task_name} must have a valid queue."
+            @errors << "#{task_name} must have a valid queue"
           end
           unless validate_task_keys(task_name)
-            @errors << "#{task_name} must only contain the following keys: #{ALLOWED_KEYS}."
+            @errors << "#{task_name} must only contain the following keys: #{ALLOWED_KEYS}"
           end
           unless validate_task_reference_formats(task_name)
-            @errors << "#{task_name} must follow the reference declaration format."
+            @errors << "#{task_name} must follow the reference declaration format"
           end
-          unless validate_task_references(task_name)
-            @errors << "#{task_name} must list existing tasks in its references."
+          unless validate_referenced_tasks(task_name)
+            @errors << "#{task_name} must list existing tasks in its references"
           end
           unless validate_sync_condition_params(task_name)
-            @errors << "#{task_name} must pass the correct parameters to the sync task."
+            @errors << "#{task_name} must pass the correct parameters to the sync task"
           end
         end
       rescue => ex
@@ -81,17 +84,17 @@ module Qyu
       end
 
       def validate_task_reference_formats(task_name)
-        (tasks[task_name]['starts'].nil? || tasks[task_name]['starts'].is_a?(Array)) &&
-          (tasks[task_name]['starts_manually'].nil? || tasks[task_name]['starts_manually'].is_a?(Array)) &&
-          (tasks[task_name]['starts_with_params'].nil? || tasks[task_name]['starts_with_params'].is_a?(Hash))
-        (tasks[task_name]['waits_for'].nil? || tasks[task_name]['waits_for'].is_a?(Hash))
+        validate_format(task_name, 'starts', Array) &&
+        validate_format(task_name, 'starts_manually', Array) &&
+        validate_format(task_name, 'starts_with_params', Hash) &&
+        validate_format(task_name, 'waits_for', Hash)
       end
 
-      def validate_task_references(task_name)
-        (tasks[task_name]['starts'] || []).all? { |t_name| tasks[t_name].is_a?(Hash) } &&
-          (tasks[task_name]['starts_manually'] || []).all? { |t_name| tasks[t_name].is_a?(Hash) } &&
-          (tasks[task_name]['starts_with_params'] || {}).all? { |t_name, _| tasks[t_name].is_a?(Hash) }
-        (tasks[task_name]['waits_for'] || {}).all? { |t_name, _| tasks[t_name].is_a?(Hash) }
+      def validate_referenced_tasks(task_name)
+        validate_presence_of_reference_tasks(task_name, 'starts', Array) &&
+        validate_presence_of_reference_tasks(task_name, 'starts_manually', Array) &&
+        validate_presence_of_reference_tasks(task_name, 'starts_with_params', Hash) &&
+        validate_presence_of_reference_tasks(task_name, 'waits_for', Hash)
       end
 
       def validate_sync_condition_params(task_name)
@@ -105,6 +108,28 @@ module Qyu
         end
       end
 
+      # checks whether a task reference key is present and in valid format
+      #
+      # @param task_name [String] name of task currently being validated
+      # @param reference_key [String] reference key to validate tasks in it
+      # @param klass [Class] class to validate against
+      # @return [Boolean]
+      def validate_format(task_name, reference_key, klass)
+        (tasks[task_name][reference_key].nil? || tasks[task_name][reference_key].is_a?(klass))
+      end
+
+      # validates that a task descriptor is present and is a Hash
+      #
+      # @param task_name [String] name of task currently being validated
+      # @param reference_key [String] reference key to validate tasks in it
+      # @param klass [Class] how this reference key is represented
+      # @return [Boolean]
+      def validate_presence_of_reference_tasks(task_name, reference_key, klass)
+        task_names = (tasks[task_name][reference_key] || klass.new)
+        task_names = task_names.keys if klass.eql?(Hash)
+        task_names.all? { |t_name| tasks[t_name].is_a?(Hash) }
+      end
+
       def entry_points
         @descriptor['starts']
       end

data/lib/qyu/models/enums/status.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 module Qyu
+  # Qyu::Status
   class Status
     COMPLETED       = 'completed'
     QUEUED          = 'queued'

data/lib/qyu/models/job.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 module Qyu
+  # Qyu::Job
   class Job
     attr_reader :descriptor, :payload, :id, :created_at, :updated_at
 

data/lib/qyu/models/task.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 module Qyu
+  # Qyu::Task
   # A Task represents a unit of work in a workflow.
   # Conceptually a Task:
   # - may not exist outside the context of a queue.

data/lib/qyu/models/workflow.rb

@@ -1,13 +1,14 @@
 # frozen_string_literal: true
 
 module Qyu
+  # Qyu::Workflow
   class Workflow
     attr_reader :id, :name, :descriptor, :created_at, :updated_at
 
     class << self
       def create(name:, descriptor:)
         validator = Qyu::Concerns::WorkflowDescriptorValidator.new(descriptor)
-        fail Qyu::Errors::WorkflowDescriptorValidatorationError, validator.errors unless validator.valid?
+        fail Qyu::Errors::WorkflowDescriptorValidationError, validator.errors unless validator.valid?
         id = persist(name, descriptor)
         time = Time.now
         new(id, name, descriptor, time, time)

data/lib/qyu/queue/memory/adapter.rb

@@ -3,6 +3,7 @@
 module Qyu
   module Queue
     module Memory
+      # Qyu::Queue::Memory::Adapter
       class Adapter < Qyu::Queue::Base
         TYPE = :memory
 

data/lib/qyu/store/memory/adapter.rb

@@ -3,6 +3,7 @@
 module Qyu
   module Store
     module Memory
+      # Qyu::Store::Memory::Adapter
       class Adapter < Qyu::Store::Base
         TYPE = :memory
 

data/lib/qyu/ui.rb

@@ -3,7 +3,7 @@
 require 'sinatra'
 require 'qyu/ui/helpers/pagination'
 
-module Qyu
+module Qyu #:nodoc: all
   class UI < Sinatra::Base
     set :port, ENV['PORT'] || 3000
     set :host, ENV['HOST'] || '0.0.0.0'

data/lib/qyu/ui/helpers/pagination.rb

@@ -3,6 +3,7 @@
 module Qyu
   module Helpers
     module Pagination
+      # Qyu::Helpers::Pagination::PaginatableArray
       class PaginatableArray < Array
         attr_accessor :limit, :offset, :total_count, :page
 

data/lib/qyu/utils.rb

@@ -1,15 +1,28 @@
 # frozen_string_literal: true
 
 module Qyu
+  # Qyu::Utils
   module Utils
+    # Calculates end time after a number of seconds
+    #
+    # @param seconds [Integer] number of seconds after time
+    # @param start_time [Time] time to start from
+    # @return [Time] end time
     def self.seconds_after_time(seconds, start_time = Time.now)
       start_time + seconds
     end
 
+    # Generates a unique UUID
+    #
+    # @return [String] UUID
     def self.uuid
       SecureRandom.uuid
     end
 
+    # Convert all hash keys to strings
+    #
+    # @param object [Hash] Hash to stringify its keys
+    # @return [String] Hash with string keys
     def self.stringify_hash_keys(object)
       object.map { |k, v| [k.to_s, v] }.to_h
     end

data/lib/qyu/version.rb

@@ -1,3 +1,3 @@
 module Qyu
-  VERSION = '1.0.1'
+  VERSION = '1.0.2'
 end

data/lib/qyu/workers/base.rb

@@ -2,6 +2,24 @@
 
 module Qyu
   module Workers
+    # Qyu::Workers::Base
+    # A Worker is sitting on a queue, waiting for something.
+    #     Qyu::Worker#work(queue_name)
+    #
+    # Worker lifecycle:
+    #   - Start an infinte loop:
+    #         while (true)
+    #   - Fetch a message (Task) from its queue:
+    #         t = Task.fetch(queue_name)
+    #   - Check the completion:
+    #         if t.completed? t.acknowledge_message
+    #   - Lock it:
+    #         t.lock! && t.mark_working
+    #   - Works: yield(t)
+    #   - Create the next steps/tasks:
+    #         t.job.create_next_tasks(t, t.job.payload (...))
+    #   - Finish:
+    #         t.unlock! && t.mark_finished && t.acknowledge_message
     class Base
       include Concerns::Callback
       include Concerns::PayloadValidator

data/lib/qyu/workers/concerns/callback.rb

@@ -3,18 +3,36 @@
 module Qyu
   module Workers
     module Concerns
+      # Qyu::Workers::Concerns::Callback
+      #
+      # Meant to add callbacks to Qyu::Worker
+      #
+      # Usage:
+      #
+      # Qyu::Worker.new do
+      #   callback :execute, :after do
+      #     # Do something after execution
+      #   end
+      # end
+      #
       module Callback
-        # Meant to add callbacks to Qyu::Worker
+        # Adds a callback option to worker
         #
-        # Usage:
-        #
-        # Qyu::Worker.new do
         #   callback :execute, :after do
         #     # Do something after execution
         #   end
-        # end
         #
-
+        #   callback :execute, :before do
+        #     # Do something before execution
+        #   end
+        #
+        #   callback :execute, :around do
+        #     # Do something around execution
+        #   end
+        #
+        # @param [Symbol] just :execute for now
+        # @param [Symbol] callback type
+        # @param [block] block to execute as callback
         def callback(method, type, &block)
           @_callbacks ||= {}
           @_callbacks[method] ||= {}

data/lib/qyu/workers/concerns/failure_queue.rb

@@ -3,17 +3,24 @@
 module Qyu
   module Workers
     module Concerns
-      # Qyu::Concerns::FailureQueue
+      # Qyu::Workers::Concerns::FailureQueue
+      #
+      # Adds ability to workers enqueue failed task to another queue
+      #
+      # Qyu::Worker.new do
+      #   failure_queue true
+      #   # or
+      #   failure_queue false
+      # end
+      #
       module FailureQueue
-        # Adds ability to workers enqueue failed task to another queue
+
+        # Configures failure queue
         #
-        # Qyu::Worker.new do
+        #   failure_queue false # default
         #   failure_queue true
-        #   # or
-        #   failure_queue false
-        # end
         #
-
+        # @param [Boolean]
         def failure_queue(fq)
           @failure_queue = fq
         end

data/lib/qyu/workers/concerns/payload_validator.rb

@@ -3,26 +3,43 @@
 module Qyu
   module Workers
     module Concerns
-      # Qyu::Concerns::PayloadValidator
+      # Qyu::Workers::Concerns::PayloadValidator
+      #
+      # Adds ability to workers to perform validations on
+      # params for the task to be processed
+      #
+      # Usage:
+      #
+      # Qyu::Worker.new do
+      #   validates :user_id,    presence: true, type: :integer, unless: :no_user
+      #   validates :name,       presence: true, type: :string
+      #   validates :hotels,     presence: true, type: :array
+      #   validates :account_id, absence: true,  if: :customer_id
+      #   validates :account_id, presence: true, unless: :customer_id
+      # end
+      #
       module PayloadValidator
-        # Adds ability to workers to perform validations on
-        # params for the task to be processed
+        # Adds a validation option to worker
         #
-        # Usage:
-        #
-        # Qyu::Worker.new do
         #   validates :user_id,    presence: true, type: :integer, unless: :no_user
         #   validates :name,       presence: true, type: :string
         #   validates :account_id, absence: true,  if: :customer_id
         #   validates :account_id, presence: true, unless: :customer_id
-        # end
         #
-
+        # @param [Symbol] payload key to validate
+        # @param [Hash] options for validation
+        # @return [Hash] registereds services
         def validates(parameter, opts = {})
           @_validations ||= {}
           @_validations[parameter.to_s] = Qyu::Utils.stringify_hash_keys(opts)
         end
 
+        # Validates payload in processing task and raises a Qyu::Errors::PayloadValidationError
+        # if any of the conditions do not hold up
+        #
+        # @param [Task] task with payload
+        # @return [nil]
+        # @raise [Qyu::Errors::PayloadValidationError]
         def validate_payload!(model)
           return unless @_validations
           payload = Qyu::Utils.stringify_hash_keys(model.payload || {})
@@ -48,6 +65,7 @@ module Qyu
 
         private
 
+        # :nodoc:
         def run_validation(option, param, value)
           # Skip if and unless conditionals (return nil)
           return if option.eql?('if')

data/lib/qyu/workers/sync.rb

@@ -2,6 +2,7 @@
 
 module Qyu
   module Workers
+    # Qyu::Workers::Sync
     class Sync < Base
       def work(queue_name)
         super do |task|

data/qyu.gemspec

@@ -21,6 +21,8 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
+  spec.metadata["yard.run"] = "yri"
+
   spec.add_runtime_dependency 'activesupport', '~> 5.1'
 
   spec.add_development_dependency 'bundler', '~> 1.16'
@@ -28,9 +30,9 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency 'rack-test'
   spec.add_development_dependency 'pry'
   spec.add_development_dependency 'pry-byebug'
-  spec.add_development_dependency 'rake', '~> 10.0'
-  spec.add_development_dependency 'rspec', '~> 3.5'
+  spec.add_development_dependency 'rake', '~> 12.0'
+  spec.add_development_dependency 'rspec', '~> 3.7'
   spec.add_development_dependency 'simplecov'
   spec.add_development_dependency 'sinatra'
-  spec.add_development_dependency 'timecop', '~> 0.8'
+  spec.add_development_dependency 'timecop', '~> 0.9'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: qyu
 version: !ruby/object:Gem::Version
-  version: 1.0.1
+  version: 1.0.2
 platform: ruby
 authors:
 - Elod Peter
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-02-23 00:00:00.000000000 Z
+date: 2018-03-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -101,28 +101,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '12.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '12.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.5'
+        version: '3.7'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.5'
+        version: '3.7'
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
@@ -157,14 +157,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: '0.9'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.8'
+        version: '0.9'
 description: Distributed task execution system for complex workflows
 email:
 - bejmuller@gmail.com
@@ -176,6 +176,7 @@ files:
 - ".gitignore"
 - ".rspec"
 - ".travis.yml"
+- CHANGELOG.md
 - CODE_OF_CONDUCT.md
 - Gemfile
 - LICENSE
@@ -254,7 +255,8 @@ files:
 - qyu.gemspec
 homepage: https://github.com/FindHotel/qyu
 licenses: []
-metadata: {}
+metadata:
+  yard.run: yri
 post_install_message: 
 rdoc_options: []
 require_paths:
@@ -271,7 +273,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.3
+rubygems_version: 2.6.8
 signing_key: 
 specification_version: 4
 summary: Distributed task execution system for complex workflows