tasque 0.0.3

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: fe02e8efd7f9659a71065ff03456285a77c3738a
+  data.tar.gz: 0148d4b75800c0249624d34a50af1902857ad804
+SHA512:
+  metadata.gz: 5446b19a1112207b663a3d91045da3c32f24d3fd4affd5d4adda441a22c412c27dc258a5f18a38f83e1409418c4cf9d8047b50c18fc7ba0f16c6cca96343b9a3
+  data.tar.gz: 6af7d2e232ef54594ee8a0f9971693bbcd22666cbcc251fa0b7b3cebc3c9bdd9b30ac15079f0b10c5b74ee713b11dab0c4ae993f29aae6fb5dd8a9948d37536d

data/.gitignore

@@ -0,0 +1,50 @@
+*.gem
+*.rbc
+/.config
+/coverage/
+/InstalledFiles
+/pkg/
+/spec/reports/
+/spec/examples.txt
+/test/tmp/
+/test/version_tmp/
+/tmp/
+
+# Used by dotenv library to load environment variables.
+# .env
+
+## Specific to RubyMotion:
+.dat*
+.repl_history
+build/
+*.bridgesupport
+build-iPhoneOS/
+build-iPhoneSimulator/
+
+## Specific to RubyMotion (use of CocoaPods):
+#
+# We recommend against adding the Pods directory to your .gitignore. However
+# you should judge for yourself, the pros and cons are mentioned at:
+# https://guides.cocoapods.org/using/using-cocoapods.html#should-i-check-the-pods-directory-into-source-control
+#
+# vendor/Pods/
+
+## Documentation cache and generated files:
+/.yardoc/
+/_yardoc/
+/doc/
+/rdoc/
+
+## Environment normalization:
+/.bundle/
+/vendor/bundle
+/lib/bundler/man/
+
+# for a library or gem, you might want to ignore these files since the code is
+# intended to run in multiple environments; otherwise, check them in:
+# Gemfile.lock
+# .ruby-version
+# .ruby-gemset
+
+# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
+.rvmrc

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in tasque.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2018 Gropher
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,64 @@
+# tasque
+
+ActiveRecord based task queue. Task processing queue with states, history and priorities. Works with your favorite database.
+
+
+## Installation
+
+Add this line to your application's `Gemfile`:
+
+    gem 'tasque'
+
+And then execute:
+
+    $ bundle
+
+Or install it manually:
+
+    $ gem install tasque
+
+Then you need to generate initializer and migration:
+
+    $ rails g tasque:install
+
+Don't forget to run migrations: 
+
+    $ rake db:migrate
+
+
+## Usage
+
+Create a simple task:
+
+    Tasque::Task.create! task: 'test', params: { text: 'This is the test!' }
+
+Or a task with priority:
+
+    Tasque::Task.create! task: 'test', priority: 9000, params: { text: 'I will be processed first!!!' }
+
+Or add a tag to find this task easily: 
+  
+    Tasque::Task.create! task: 'test', tag: 'user_123', params: { text: 'This task is property of User #123!' }
+
+Now to process the task:
+
+    # create task processor
+    processor = Tasque::Processor.new
+
+    # register a handler for test tasks
+    processor.add_handler('test') do |task|
+      puts "Got task ##{task.id}. Task says: '#{task.params['text']}'"
+      { log: 'got task, printed text' } # returning task result
+    end
+
+    # start processor and wait
+    processor.start
+    
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/lib/generators/tasque/install_generator.rb

@@ -0,0 +1,25 @@
+require 'rails/generators'
+require 'rails/generators/migration'
+
+module Tasque
+  class InstallGenerator < ::Rails::Generators::Base
+    include Rails::Generators::Migration
+
+    desc 'Create a sample Tasque initializer and migration'
+    source_root File.expand_path('../templates', __FILE__)
+
+    def self.next_migration_number(path)
+      unless @prev_migration_nr
+        @prev_migration_nr = Time.now.utc.strftime("%Y%m%d%H%M%S").to_i
+      else
+        @prev_migration_nr += 1
+      end
+      @prev_migration_nr.to_s
+    end
+
+    def create_initializer
+      template 'tasque.erb', 'config/initializers/tasque.rb'
+      migration_template "create_tasque_tasks.erb", "db/migrate/create_tasque_tasks.rb"
+    end
+  end
+end

data/lib/generators/tasque/templates/create_tasque_tasks.erb

@@ -0,0 +1,28 @@
+class CreateTasqueTasks < ActiveRecord::Migration[4.2]
+  def change
+    create_table :tasque_tasks do |t|
+      t.string :tag
+      t.string :task
+      t.text :params
+      t.text :result
+      t.string :worker
+      t.integer :priority, default: 0
+      t.integer :attempts, default: 0
+      t.integer :progress, default: 0
+      t.string :status, default: 'new'
+  
+      t.datetime :started_at
+      t.datetime :finished_at
+  
+      t.timestamps
+    end
+
+    add_index :tasque_tasks, [:status, :task]
+    add_index :tasque_tasks, :status
+    add_index :tasque_tasks, :tag     
+    add_index :tasque_tasks, :task
+    add_index :tasque_tasks, :worker
+    add_index :tasque_tasks, :priority
+    add_index :tasque_tasks, :attempts
+  end
+end

data/lib/generators/tasque/templates/tasque.erb

@@ -0,0 +1,27 @@
+TASQUE_ROOT = Rails.root
+
+Tasque.configure do |config|
+  config.logger = Rails.logger
+  
+  # Which database from database.yml we use
+  config.environment = Rails.env
+    
+  # This option need only if you want to receive tasks
+  # config.worker = 'example'
+ 
+  # How often should processor check for a new tasks
+  # config.check_interval = 10 # seconds
+ 
+  # How often should task progress be reported
+  # config.progress_interval = 5 # seconds
+  
+  # This option tells processor to fetch only tasks with priority value greater than or equal to minimum_priority
+  # config.minimum_priority = 123
+
+  # Send task state updates and progress notifications via Insque
+  # config.notify = true
+
+  # Send worker heartbeat via Insque
+  # config.heartbeat = true
+  # config.heartbeat_interval = 30 # seconds
+end

data/lib/tasque.rb

@@ -0,0 +1,10 @@
+require "active_record"
+
+require "tasque/configuration"
+require "tasque/task"
+require "tasque/task_error"
+require "tasque/processor"
+require "tasque/version"
+
+module Tasque
+end

data/lib/tasque/configuration.rb

@@ -0,0 +1,65 @@
+require 'yaml'
+
+module Tasque
+  class << self
+    attr_accessor :config
+  end
+
+  def self.configure
+    self.config ||= Configuration.new
+    yield(config) if block_given?
+    self.database_connection
+    return self.config
+  end
+  
+  def self.database_connection()
+    raise 'No configuration. Use Tasque.configure' if self.config.nil?
+    @database ||= begin
+      if !defined?(Rails) && !ActiveRecord::Base.connected?
+        ActiveRecord::Base.logger = self.config.logger
+        db = self.config.database[self.config.environment.to_s]
+        db = self.config.database if db.nil?
+        ActiveRecord::Base.establish_connection(db)
+        ActiveRecord::Base.connection
+      end
+      ActiveRecord::Base
+    end
+  end
+  
+  def self.root
+    defined?(TASQUE_ROOT) ? TASQUE_ROOT : Dir.pwd
+  end
+
+  class Configuration
+    attr_accessor :database
+    attr_accessor :database_file
+    attr_accessor :environment
+    attr_accessor :logger
+    attr_accessor :check_interval
+    attr_accessor :progress_interval
+    attr_accessor :minimum_priority
+    attr_accessor :worker
+    attr_accessor :heartbeat
+    attr_accessor :heartbeat_interval
+    attr_accessor :notify
+
+    def initialize
+      self.environment = :development
+      self.database_file = ::File.expand_path('config/database.yml', Tasque.root)
+      self.check_interval = 10 # seconds
+      self.worker = "default"
+      self.progress_interval = 5 # seconds
+      self.heartbeat = false
+      self.heartbeat_interval = 10 # seconds
+      self.notify = false
+    end
+    
+    def database_file=(path)
+      if File.exists?(path)
+        @database = YAML.load(File.read(path)) || {}
+        @database_file = path
+      end
+    end
+    
+  end
+end

data/lib/tasque/processor.rb

@@ -0,0 +1,82 @@
+require 'timers'
+
+module Tasque    
+  class Processor
+    attr_reader :timers
+    
+    def initialize
+      @timers = Timers::Group.new
+      @last_task = false
+      @current_task = nil
+      @handlers = {}
+    end
+    
+    def add_handler type, &block
+      @handlers[type.to_sym] = Proc.new do |task|
+        @current_task = task
+        task.process
+        begin
+          task.result = block.call task
+        rescue Tasque::TaskError => e
+          task.error = {
+            task_error: e.task_error
+          }
+        rescue Exception => e
+          task.error = {
+            exception: e.message,
+            backtrace: e.backtrace
+          }
+        end
+        task.error? ? task.failure : task.complete
+        @current_task = nil
+      end
+      @timers.every(check_interval) do
+        begin
+          has_task = Tasque::Task.fetch(type) do |task|
+            @handlers[type.to_sym].call(task)
+          end
+        end while has_task
+      end
+    end
+
+    def start
+      shutdown = ->(signo) {
+        if @last_task
+          unless @current_task.nil?
+            @current_task.failure
+            @current_task.reprocess
+          end
+          exit! 
+        end
+        @last_task = true
+      }
+      trap("SIGINT", shutdown)
+      trap("SIGTERM", shutdown)
+      if Tasque.config.heartbeat && defined?(Insque)
+        heartbeat_thread = Thread.new do
+          heartbeat_timers = Timers::Group.new
+          heartbeat_timers.every(Tasque.config.heartbeat_interval) do
+            message = {
+              worker: Tasque.config.worker,
+              busy: !@current_job.nil?
+            }
+            Insque.broadcast :heartbeat, message
+          end
+          loop do
+            heartbeat_timers.wait
+          end
+        end
+        heartbeat_thread.abort_on_exception = true
+      end
+      loop do
+        break if @last_task
+        @timers.wait
+      end
+    end
+    
+  private
+    def check_interval
+      Tasque.config.check_interval
+    end
+  end
+end

data/lib/tasque/task.rb

@@ -0,0 +1,139 @@
+require 'state_machine/core'
+
+module Tasque
+  class Task < ActiveRecord::Base
+    extend StateMachine::MacroMethods
+
+    MAX_ATTEMPTS=3
+    
+    self.table_name = :tasque_tasks
+    
+    serialize :params, JSON
+    serialize :result, JSON
+    
+    scope :with_task, ->(task) { where(task: task).order priority: :desc }
+    scope :minimum_priority, ->(priority) { priority.nil? ? nil : where('priority >= ?', priority) }
+    scope :to_process, -> { where status: %w(new reprocessed) }
+    scope :with_error, -> { where status: 'error' }
+    scope :to_reprocess, -> { with_error.where 'attempts < ?', MAX_ATTEMPTS }
+    scope :finished_in, ->(interval) { where('finished_at > ?', interval.ago) }
+    
+    validates :task, presence: true
+    validates :attempts, numericality: { only_integer: true, greater_than_or_equal_to: 0 }
+    validates :progress, numericality: { only_integer: true, greater_than_or_equal_to: 0, less_than_or_equal_to: 100 }
+    validates :priority, numericality: { only_integer: true }
+    
+    class << self
+      def fetch(type, &block)
+        task = nil
+        transaction do
+          minimum_priority = Tasque.config.minimum_priority
+          task = self.with_task(type).to_process.minimum_priority(minimum_priority).lock(true).first
+          if task and task.can_pickup?
+            task.pickup
+          else
+            task = nil
+          end
+        end
+        yield(task) if task
+        !!task
+      end
+    
+      def monitoring
+        {
+          queue: Tasque::Task.to_process.count,
+          errors: Tasque::Task.with_error.finished_in(1.hour).count
+        }
+      end
+      
+      def autoreprocess(reprocess_limit = nil)
+        Tasque::Task.to_reprocess.limit(reprocess_limit.to_i).each do |task|
+          task.reprocess
+        end.count
+      end
+    end
+    
+    state_machine :status, initial: :new do
+      after_transition on: :pickup do |task|
+        task.update_column :worker, Tasque.config.worker
+      end
+      
+      after_transition on: :process do |task|
+        task.update_column :started_at, Time.now
+      end
+      
+      after_transition on: :complete do |task|
+        task.update_columns progress: 100, finished_at: Time.now
+      end
+      
+      after_transition on: :failure do |task|
+        task.update_columns attempts: (task.attempts + 1), result: { error: task.error }, progress: 0
+      end
+      
+      after_transition on: :reprocess do |task|
+        task.update_columns started_at: nil, result: nil, progress: 0
+      end
+      
+      after_transition do: :notify
+      
+      
+      event :pickup do
+        transition [:new, :reprocessed] => :starting
+      end
+      
+      event :process do
+        transition :starting => :processing
+      end
+      
+      event :complete do
+        transition :processing => :complete
+      end
+      
+      event :failure do
+        transition :processing => :error
+      end
+      
+      event :reprocess do
+        transition [:processing, :complete, :error] => :reprocessed
+      end
+      
+      event :cancel do
+        transition any - [:processing] => :canceled
+      end
+      
+      
+      state :processing do
+        def progress!(val)
+          val = 0   if val < 0
+          val = 100 if val > 100
+          return if (Time.now.to_i - @last_progress_at.to_i) < Tasque.config.progress_interval || val == @last_progress_val
+          self.update_columns progress: val, updated_at: Time.now
+          @last_progress_at = Time.now
+          @last_progress_val = val
+          notify
+        end
+        
+        def error!(task_error)
+          raise Tasque::TaskError.new(self, task_error)
+        end
+      end
+      
+      state :processing, :error do
+        attr_accessor :error
+
+        def error?
+          !@error.nil?
+        end
+      end
+    end
+    
+  private
+    def notify      
+      if Tasque.config.notify && defined?(Insque)
+        Insque.broadcast :task_update, self
+      end
+    rescue Exception => e
+      logger.error "Notify error: #{e.message}"
+    end
+  end
+end

data/lib/tasque/task_error.rb

@@ -0,0 +1,11 @@
+module Tasque
+  class TaskError < Exception
+    attr_reader :task
+    attr_reader :task_error
+  
+    def initialize(task, task_error)
+      @task = task
+      @task_error = task_error
+    end
+  end
+end

data/lib/tasque/version.rb

@@ -0,0 +1,3 @@
+module Tasque
+  VERSION = "0.0.3"
+end

data/tasque.gemspec

@@ -0,0 +1,24 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'tasque/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "tasque"
+  gem.version       = Tasque::VERSION
+  gem.authors       = ["Yuri Gomozov"]
+  gem.email         = ["grophen@gmail.com"]
+  gem.description   = %q{Task processing queue with states, history and priorities. Works with your favorite database. }
+  gem.summary       = %q{ActiveRecord based task queue}
+  gem.homepage      = "https://github.com/Gropher/tasque"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+  
+  gem.add_dependency("activerecord")
+  gem.add_dependency("timers")
+  gem.add_dependency("state_machine")
+  gem.add_dependency("json")
+end

metadata

@@ -0,0 +1,114 @@
+--- !ruby/object:Gem::Specification
+name: tasque
+version: !ruby/object:Gem::Version
+  version: 0.0.3
+platform: ruby
+authors:
+- Yuri Gomozov
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2018-04-13 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: timers
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: state_machine
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 'Task processing queue with states, history and priorities. Works with
+  your favorite database. '
+email:
+- grophen@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE
+- README.md
+- lib/generators/tasque/install_generator.rb
+- lib/generators/tasque/templates/create_tasque_tasks.erb
+- lib/generators/tasque/templates/tasque.erb
+- lib/tasque.rb
+- lib/tasque/configuration.rb
+- lib/tasque/processor.rb
+- lib/tasque/task.rb
+- lib/tasque/task_error.rb
+- lib/tasque/version.rb
+- tasque.gemspec
+homepage: https://github.com/Gropher/tasque
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: ActiveRecord based task queue
+test_files: []