workerholic 0.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 61f1243274828aafe8b8c00d714b00e0832e1998
+  data.tar.gz: 7d1d1d50f9fa005e596b2500482831c749b9d8bd
+SHA512:
+  metadata.gz: 6d758f61a7cb894d6b514643e3c896cf2ea42adeeda3d66c79c99ded136b8cd9369faa89027f99695b6ebb44904f2357496ad401b1233c624fd8e2e1ebf94956
+  data.tar.gz: c9ee873f1ed1f1a402947f777b21cd706bd72418a9d40b98566e57ebd833a7e76a3abd41b0bfc6ee0b4d9af5af4e670485508f3995c2a898fc619685dccc6bd7

data/.gitignore

@@ -0,0 +1 @@
+dump.rdb

data/.rspec

@@ -0,0 +1 @@
+--require spec_helper

data/Gemfile

@@ -0,0 +1,12 @@
+source 'https://rubygems.org'
+
+gem 'redis', '~> 3.3', '>= 3.3.3'
+gem 'connection_pool', '~> 2.2', '>= 2.2.0'
+
+group :development, :test do
+  gem 'pry-byebug'
+end
+
+group :test do
+  gem 'rspec', '~> 3.6', '>= 3.6.0'
+end

data/Gemfile.lock

@@ -0,0 +1,42 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    byebug (9.0.6)
+    coderay (1.1.1)
+    connection_pool (2.2.1)
+    diff-lcs (1.3)
+    method_source (0.8.2)
+    pry (0.10.4)
+      coderay (~> 1.1.0)
+      method_source (~> 0.8.1)
+      slop (~> 3.4)
+    pry-byebug (3.4.2)
+      byebug (~> 9.0)
+      pry (~> 0.10)
+    redis (3.3.3)
+    rspec (3.6.0)
+      rspec-core (~> 3.6.0)
+      rspec-expectations (~> 3.6.0)
+      rspec-mocks (~> 3.6.0)
+    rspec-core (3.6.0)
+      rspec-support (~> 3.6.0)
+    rspec-expectations (3.6.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.6.0)
+    rspec-mocks (3.6.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.6.0)
+    rspec-support (3.6.0)
+    slop (3.6.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  connection_pool (~> 2.2, >= 2.2.0)
+  pry-byebug
+  redis (~> 3.3, >= 3.3.3)
+  rspec
+
+BUNDLED WITH
+   1.15.1

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2017 workerholic
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,2 @@
+# workerholic
+A Background Job Processing Manager

data/app_test/job_test.rb

@@ -0,0 +1,20 @@
+require_relative '../lib/workerholic'
+
+class JobTestFast
+  include Workerholic::Job
+  job_options queue_name: 'workerholic:queue:job_fast'
+
+  def perform(str, num)
+    str
+  end
+end
+
+class JobTestSlow
+  include Workerholic::Job
+  job_options queue_name: 'workerholic:queue:job_slow'
+
+  def perform(str, num)
+    str
+    sleep(0.1)
+  end
+end

data/app_test/run.rb

@@ -0,0 +1,10 @@
+require_relative 'job_test'
+
+5_000.times do |n|
+  JobTestFast.new.perform_async('NON BLOCKING', n)
+  JobTestFast.new.perform_async('NON BLOCKING', n)
+  # JobTestFast.new.perform_async('NON BLOCKING', n)
+  # JobTestFast.new.perform_async('NON BLOCKING', n)
+  # # JobTestSlow.new.perform_async('BLOCKING', n)
+  JobTestSlow.new.perform_async('BLOCKING', n)
+end

data/lib/server.rb

@@ -0,0 +1,13 @@
+$LOAD_PATH << __dir__
+
+require 'workerholic'
+
+auto_balance = ARGV.any? { |arg| arg == '--auto-balance' }
+workers_count = ARGV.find { |arg| arg.match? /^--workers=\d+$/ }
+
+if workers_count
+  workers_count = workers_count[/\d+/].to_i
+  Workerholic.workers_count = workers_count
+end
+
+Workerholic::Manager.new(auto_balance: auto_balance).start

data/lib/workerholic.rb

@@ -0,0 +1,47 @@
+$LOAD_PATH << __dir__
+
+require 'yaml'
+require 'redis'
+require 'connection_pool'
+require 'logger'
+require 'pry-byebug'
+
+require 'workerholic/manager'
+require 'workerholic/worker_balancer'
+
+require 'workerholic/job'
+require 'workerholic/job_wrapper'
+
+require 'workerholic/worker'
+require 'workerholic/job_processor'
+require 'workerholic/job_scheduler'
+require 'workerholic/job_retry'
+
+require 'workerholic/storage'
+require 'workerholic/sorted_set'
+require 'workerholic/queue'
+
+require 'workerholic/job_serializer'
+require 'workerholic/statistics'
+require 'workerholic/log_manager'
+
+require_relative '../app_test/job_test' # require the application code
+
+module Workerholic
+  # DEFAULTS = {
+  #   workers_count: 25
+  # }
+
+  def self.workers_count
+    @workers_count || 25 # DEFAULTS[:workers_count]
+  end
+
+  def self.workers_count=(num)
+    raise ArgumentError unless num.is_a?(Integer) && num < 200
+    @workers_count = num
+  end
+
+  def self.redis_pool
+    @redis ||= ConnectionPool.new(size: workers_count + 5, timeout: 5) { Redis.new }
+  end
+end

data/lib/workerholic/adapters/active_job_adapter.rb

@@ -0,0 +1,24 @@
+module ActiveJob
+  module QueueAdapters
+    class WorkerholicAdapter
+      def enqueue(job)
+        job_data = job.serialize
+
+        JobWrapper.new.perform_async(
+          class: job_data[:job_class],
+          arguments: job_data[:arguments]
+        )
+      end
+
+      class JobWrapper
+        include Workerholic::Job
+
+        def perform(job_data)
+          Base.execute job_data
+        end
+      end
+    end
+  end
+
+  autoload :WorkerholicAdapter
+end

data/lib/workerholic/job.rb

@@ -0,0 +1,49 @@
+module Workerholic
+  module Job
+    def self.included(base)
+      base.extend(ClassMethods)
+      base.job_options
+    end
+
+    module ClassMethods
+      def job_options(params={})
+        define_method(:specified_job_options) do
+          {
+            execute_at: params[:execute_at],
+            queue_name: params[:queue_name] || 'workerholic:queue:main'
+          }
+        end
+      end
+    end
+
+    def perform_async(*args)
+      serialized_job, queue_name = prepare_job_for_enqueueing(args)
+
+      Queue.new(queue_name).enqueue(serialized_job)
+    end
+
+    def perform_delayed(*args)
+      delay_in_sec = verify_delay(args[0])
+      serialized_job, queue_name = prepare_job_for_enqueueing(args)
+
+      JobScheduler.new(set_name: queue_name).schedule(serialized_job, delay_in_sec)
+    end
+
+    private
+
+    def verify_delay(delay_arg)
+      raise ArgumentError, 'Delay argument has to be of Numeric type' unless delay_arg.is_a? Numeric
+
+      delay_arg
+    end
+
+    def prepare_job_for_enqueueing(args)
+      raise ArgumentError if self.method(:perform).arity != args.size
+
+      job = JobWrapper.new(class: self.class, arguments: args)
+      job.statistics.enqueued_at = Time.now.to_f
+
+      [JobSerializer.serialize(job), specified_job_options[:queue_name]]
+    end
+  end
+end

data/lib/workerholic/job_processor.rb

@@ -0,0 +1,29 @@
+module Workerholic
+  class JobProcessor
+    def initialize(serialized_job)
+      @serialized_job = serialized_job
+      @logger = LogManager.new
+    end
+
+    def process
+      job = JobSerializer.deserialize(@serialized_job)
+
+      begin
+        job.statistics.started_at = Time.now.to_f
+        job_result = job.perform
+        job.statistics.completed_at = Time.now.to_f
+
+        # @logger.log('info', "Completed: your job from class #{job.klass} was completed on #{job.statistics.completed_at}.")
+
+        job_result
+      rescue Exception => e
+        job.statistics.errors.push([e.class, e.message])
+        JobRetry.new(job: job)
+
+        # @logger.log('error', "Failed: your job from class #{job.class} was unsuccessful. Retrying in 10 seconds.")
+      end
+
+      # Push job into some collection
+    end
+  end
+end

data/lib/workerholic/job_retry.rb

@@ -0,0 +1,32 @@
+module Workerholic
+  class JobRetry
+    attr_reader :job, :sorted_set
+
+    def initialize(options={})
+      @job = options[:job]
+      @sorted_set = options[:sorted_set] || SortedSet.new('workerholic:scheduled_jobs')
+
+      self.retry
+    end
+
+    protected
+
+    def retry
+      return if job.retry_count >= 5
+
+      increment_retry_count
+      schedule_job_for_retry
+      sorted_set.add(JobSerializer.serialize(job), job.execute_at)
+    end
+
+    private
+
+    def increment_retry_count
+      job.retry_count += 1
+    end
+
+    def schedule_job_for_retry
+      job.execute_at = Time.now.to_f + 10 * job.retry_count
+    end
+  end
+end

data/lib/workerholic/job_scheduler.rb

@@ -0,0 +1,47 @@
+module Workerholic
+  class JobScheduler
+    attr_reader :sorted_set, :queue, :scheduler_thread
+    attr_accessor :alive
+
+    def initialize(opts={})
+      @sorted_set = SortedSet.new(opts[:set_name] || 'workerholic:scheduled_jobs')
+      @queue = Queue.new(opts[:queue_name] || 'workerholic:main')
+      @alive = true
+    end
+
+    def start
+      @scheduler_thread = Thread.new do
+        enqueue_due_jobs while alive
+      end
+    end
+
+    def job_due?
+      scheduled_job = sorted_set.peek
+      return false unless scheduled_job
+
+      job_execution_time = scheduled_job.last
+      Time.now.to_f >= job_execution_time
+    end
+
+    def schedule(serialized_job, score)
+      sorted_set.add(serialized_job, score)
+    end
+
+    def enqueue_due_jobs
+      if job_due?
+        while job_due?
+          serialized_job, job_execution_time = sorted_set.peek
+          sorted_set.remove(job_execution_time)
+          queue.enqueue(serialized_job)
+        end
+      else
+        sleep(5)
+      end
+    end
+
+    def kill
+      self.alive = false
+      scheduler_thread.join if scheduler_thread
+    end
+  end
+end

data/lib/workerholic/job_serializer.rb

@@ -0,0 +1,12 @@
+module Workerholic
+  class JobSerializer
+    def self.serialize(job)
+      YAML.dump(job.to_hash)
+    end
+
+    def self.deserialize(yaml_job)
+      job_info = YAML.load(yaml_job)
+      JobWrapper.new(job_info)
+    end
+  end
+end

data/lib/workerholic/job_wrapper.rb

@@ -0,0 +1,32 @@
+module Workerholic
+  class JobWrapper
+    attr_accessor :retry_count, :execute_at
+    attr_reader :klass, :arguments, :statistics
+
+    def initialize(options={})
+      @klass = options[:class]
+      @arguments = options[:arguments]
+      @execute_at = options[:execute_at]
+      @retry_count = options[:retry_count] || 0
+      @statistics = Statistics.new(options[:statistics] || {})
+    end
+
+    def to_hash
+      {
+        class: klass,
+        arguments: arguments,
+        retry_count: retry_count,
+        execute_at: execute_at,
+        statistics: statistics.to_hash
+      }
+    end
+
+    def perform
+      klass.new.perform(*arguments)
+    end
+
+    def ==(other)
+      to_hash == other.to_hash
+    end
+  end
+end

data/lib/workerholic/log_manager.rb

@@ -0,0 +1,17 @@
+module Workerholic
+  class LogManager
+    attr_reader :logger
+
+    def initialize
+      @logger = Logger.new(STDOUT)
+    end
+
+    def log(severity, message)
+      logger.formatter = proc do |severity, datetime, progname, msg|
+        "#{severity}: #{msg}\n"
+      end
+
+      logger.send(severity, message)
+    end
+  end
+end