workerholic 0.0.8 → 0.0.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: '02909e2b39a8d03796f21f57c2914f56a4b58a7b'
-  data.tar.gz: 7696ea1dd34cf1df6c6ed21b5a627ea1ce1e4f48
+  metadata.gz: c3ace2785bcb5ca82bc4e0d0d5c738410140f3f6
+  data.tar.gz: 5829d09c00b963cff402dbe34126564b28af1386
 SHA512:
-  metadata.gz: '03138c50a4edcc225af7d17b9a55cc7cf3799ac3a8612864768629b3685bfc2d67a8a798edcc758130b2543c1e04469338fbbe4129f0fd4b1bd61f3272f49b22'
-  data.tar.gz: ff0cff28763276952f390d9b2263eb399a2c3910133f28d16d2015aea6c8f12c6dea1f0ac30b438f493e50af8c63ef30af4e1c81ad314d53b966764984a8c9e3
+  metadata.gz: 4d310181c5f824813425a6e6c3b249d27ee0baf0e27970cf4a40d6be3529237f95567b62f96a5e58904736e4d2a816c777379eb3bb4cd95593f6bc23c2b535cc
+  data.tar.gz: eabb0f28a65a6a6e46757fd58e136dfa9e628bc7ab00c544bd147cd54638bfdba5c2e046ca6ccc75593ee2f5ee42cd6acfd4bd01ab3b7a4d5bc6045e1a7542fb

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    workerholic (0.0.7)
+    workerholic (0.0.8)
       connection_pool (~> 2.2, >= 2.2.0)
       redis (~> 3.3, >= 3.3.3)
       sinatra

data/app_test/job_test.rb

@@ -1,4 +1,5 @@
 require 'workerholic'
+require 'prime'
 
 class JobTestFast
   include Workerholic::Job
@@ -15,7 +16,6 @@ class JobTestSlow
 
   def perform(str, num)
     sleep(0.5)
-    puts "#{num} - #{str}"
   end
 end
 
@@ -23,7 +23,6 @@ class ThreadKiller
   include Workerholic::Job
 
   def perform(string, n)
-    puts "#{n}. #{string}"
     Thread.main.kill
   end
 end
@@ -32,7 +31,6 @@ class LargeArg
   include Workerholic::Job
 
   def perform(arr, n)
-    puts n
   end
 end
 
@@ -40,7 +38,6 @@ class ManyArgs
   include Workerholic::Job
 
   def perform(n, *args)
-    puts "#{n}: #{args}"
   end
 end
 
@@ -50,7 +47,6 @@ class HeavyCalculation
 
   def perform(n, arr)
     arr = bubble_sort(arr)
-    puts "#{n}: #{arr[0..9]}"
   end
 
   def bubble_sort(array)
@@ -79,6 +75,5 @@ class GetPrimes
     Prime.each(max) do |prime|
       prime
     end
-    puts n
   end
 end

data/app_test/run.rb

@@ -1,43 +1,51 @@
 require_relative 'job_test'
 
-# 100000.times do |n|
-#   JobTestFast.new.perform_async('NONBLOCKING', n)
-#   # sleep(0.0015)
-# end
-
-# 10000.times do |n|
-#   JobTestSlow.new.perform_async('BLOCKING', n)
-#   sleep(0.0015)
-# end
-
-# 100000.times do |n|
-#   ThreadKiller.new.perform_async('Kill', n)
-# end
-
-# arg = Array.new(10000, 'string')
-# 10000.times do |n|
-#   LargeArg.new.perform_async(arg, n)
-# end
-
-# unsorted_array = (1..10000).to_a.shuffle
-unsorted_array = (1..1000).to_a.shuffle
-1000.times do |n|
-  HeavyCalculation.new.perform_async(n, unsorted_array)
+module TestRunner
+  def self.non_blocking(num_of_cycles)
+    num_of_cycles.times do |n|
+       JobTestFast.new.perform_async('NONBLOCKING', n)
+    end
+  end
+
+  def self.blocking(num_of_cycles)
+    num_of_cycles.times do |n|
+      JobTestSlow.new.perform_async('BLOCKING', n)
+    end
+  end
+
+  def self.thread_killer(num_of_cycles)
+    num_of_cycles.times do |n|
+      ThreadKiller.new.perform_async('Kill', n)
+    end
+  end
+
+  def self.large_arg(num_of_cycles)
+    arg = Array.new(10000, 'string')
+
+    num_of_cycles.times do |n|
+      LargeArg.new.perform_async(arg, n)
+    end
+  end
+
+  def self.sort_array(num_of_cycles, array_size)
+    unsorted_array = (0..array_size).to_a.shuffle
+
+    num_of_cycles.times do |n|
+      HeavyCalculation.new.perform_async(n, unsorted_array)
+    end
+  end
+
+  def self.many_args(num_of_cycles)
+    num_of_cycles.times do |n|
+      ManyArgs.new.perform_async(n, [1, 2, 3], { key: 'value'}, :symb, 'string', 22, false)
+    end
+  end
+
+  def self.calculate_primes(num_of_cycles)
+    num_of_cycles.times do |n|
+      GetPrimes.new.perform_async(n, 10)
+    end
+  end
 end
 
-5_000.times do |n|
-  JobTestFast.new.perform_async('NON BLOCKING', n)
-  JobTestFast.new.perform_async('NON BLOCKING', n)
-  # JobTestFast.new.perform_async('NON BLOCKING', n)
-  # JobTestFast.new.perform_async('NON BLOCKING', n)
-  # # JobTestSlow.new.perform_async('BLOCKING', n)
-  JobTestSlow.new.perform_async('BLOCKING', n)
-end
-
-# 100000.times do |n|
-#   ManyArgs.new.perform_async(n, [1, 2, 3], { key: 'value'}, :symb, 'string', 22, false)
-# end
-
-# 100000.times do |n|
-#   GetPrimes.new.perform_async(n, 1000000)
-# end
+TestRunner.non_blocking(2)

data/lib/workerholic/adapters/active_job_adapter.rb

@@ -3,11 +3,15 @@ module ActiveJob
     class WorkerholicAdapter
       def enqueue(job)
         job_data = job.serialize
+        queue_name = "workerholic:queue:#{job_data['queue_name']}"
 
-        JobWrapper.new.perform_async(
-          class: job_data[:job_class],
-          arguments: job_data[:arguments]
-        )
+        job = JobWrapper.new
+        job.instance_variable_set(:@queue_name, queue_name)
+
+        # capture job class so it can be passed to `Base.execute` in `JobWrapper#perform`
+        job.instance_variable_set(:@class, job_data['job_class'])
+
+        job.perform_async(*job_data['arguments'])
       end
 
       class JobWrapper
@@ -19,6 +23,4 @@ module ActiveJob
       end
     end
   end
-
-  autoload :WorkerholicAdapter
 end

data/lib/workerholic/cli.rb

@@ -29,7 +29,7 @@ module Workerholic
       OptionParser.new do |opts|
         opts.banner = 'Usage: workerholic [options]'
 
-        opts.on '-ab', '--auto-balance', 'auto-balance workers based on number of jobs in each queue' do
+        opts.on '-a', '--auto-balance', 'auto-balance workers based on number of jobs in each queue' do
           options[:auto_balance] = true
         end
 
@@ -55,12 +55,19 @@ module Workerholic
     def load_app
       if File.exist?('./config/environment.rb')
         require File.expand_path('./config/environment.rb')
+
         require 'workerholic/adapters/active_job_adapter'
+
+        ActiveSupport.run_load_hooks(:before_eager_load, Rails.application)
+        Rails.application.config.eager_load_namespaces.each(&:eager_load!)
       elsif options[:require]
-        if File.exist?(options[:require])
-          require File.expand_path(options[:require])
+        file_path = File.expand_path(options[:require])
+
+        if File.exist?(file_path)
+          require file_path
         else
-          log.info('The file you specified to load your application is not valid!')
+          logger.info('The file you specified to load your application is not valid!')
+
           exit
         end
       else

data/lib/workerholic/job.rb

@@ -19,7 +19,7 @@ module Workerholic
     def perform_async(*args)
       serialized_job, queue_name = prepare_job_for_enqueueing(args)
 
-      Queue.new(queue_name).enqueue(serialized_job)
+      Queue.new(@queue_name || queue_name).enqueue(serialized_job)
     end
 
     def perform_delayed(*args)
@@ -40,7 +40,8 @@ module Workerholic
     def prepare_job_for_enqueueing(args)
       raise ArgumentError if self.method(:perform).arity != args.size
 
-      job = JobWrapper.new(class: self.class, arguments: args)
+      job = JobWrapper.new(klass: @class || self.class, arguments: args, wrapper: self.class)
+
       job.statistics.enqueued_at = Time.now.to_f
 
       [JobSerializer.serialize(job), specified_job_options[:queue_name]]

data/lib/workerholic/job_processor.rb

@@ -1,29 +1,40 @@
 module Workerholic
   class JobProcessor
+    attr_reader :serialized_job
+
     def initialize(serialized_job)
       @serialized_job = serialized_job
       @logger = LogManager.new
     end
 
     def process
-      job = JobSerializer.deserialize(@serialized_job)
+      job = JobSerializer.deserialize(serialized_job)
 
       begin
         job.statistics.started_at = Time.now.to_f
         job_result = job.perform
         job.statistics.completed_at = Time.now.to_f
 
-        # @logger.info("Completed: your job from class #{job.klass} was completed on #{job.statistics.completed_at}.")
+        StatsStorage.save_job('completed_jobs', job)
 
-        job_result
+        @logger.info("Completed: your job from class #{job.klass} was completed on #{job.statistics.completed_at}.")
       rescue Exception => e
+
         job.statistics.errors.push([e.class, e.message])
-        JobRetry.new(job: job)
+        retry_job(job)
 
-        # @logger.error("Failed: your job from class #{job.class} was unsuccessful. Retrying in 10 seconds.")
+        @logger.error("Failed: your job from class #{job.class} was unsuccessful. Retrying in 10 seconds.")
       end
+      job_result
+    end
 
-      # Push job into some collection
+    private
+
+    def retry_job(job)
+      limit_reached = JobRetry.new(job: job)
+      if limit_reached
+        job.statistics.failed_on = Time.now.to_f
+      end
     end
   end
 end

data/lib/workerholic/job_retry.rb

@@ -1,6 +1,6 @@
 module Workerholic
   class JobRetry
-    attr_reader :job, :sorted_set
+    attr_reader :job, :sorted_set, :stats_storage
 
     def initialize(options={})
       @job = options[:job]
@@ -12,7 +12,10 @@ module Workerholic
     protected
 
     def retry
-      return if job.retry_count >= 5
+      if job.retry_count >= 5
+        StatsStorage.save_job('failed_jobs', job)
+        return false
+      end
 
       increment_retry_count
       schedule_job_for_retry

data/lib/workerholic/job_scheduler.rb

@@ -5,7 +5,7 @@ module Workerholic
 
     def initialize(opts={})
       @sorted_set = SortedSet.new(opts[:set_name] || 'workerholic:scheduled_jobs')
-      @queue = Queue.new(opts[:queue_name] || 'workerholic:main')
+      @queue = Queue.new(opts[:queue_name] || 'workerholic:queue:main')
       @alive = true
     end
 
@@ -35,12 +35,15 @@ module Workerholic
           queue.enqueue(serialized_job)
         end
       else
-        sleep(5)
+        sleep(2)
       end
     end
 
     def kill
       self.alive = false
+    end
+
+    def join
       scheduler_thread.join if scheduler_thread
     end
   end

data/lib/workerholic/job_serializer.rb

@@ -8,5 +8,13 @@ module Workerholic
       job_info = YAML.load(yaml_job)
       JobWrapper.new(job_info)
     end
+
+    def self.serialize_stats(stat)
+      YAML.dump(stat)
+    end
+
+    def self.deserialize_stats(yaml_stat)
+      YAML.load(yaml_stat)
+    end
   end
 end

data/lib/workerholic/job_statistics.rb

@@ -0,0 +1,38 @@
+module Workerholic
+  class JobStatistics
+
+    attr_accessor :enqueued_at,
+                  :errors,
+                  :started_at,
+                  :completed_at,
+                  :failed_on
+
+    def initialize(options={})
+      @enqueued_at = options[:enqueued_at]
+      @errors = options[:errors] || []
+      @started_at = options[:started_at]
+      @completed_at = options[:completed_at]
+    end
+
+    def to_hash
+      {
+        enqueued_at: enqueued_at,
+        errors: errors,
+        started_at: started_at,
+        completed_at: completed_at,
+        elapsed_time: elapsed_time,
+        failed_on: failed_on
+      }
+    end
+
+    def elapsed_time
+      if completed_at && started_at
+        format_elapsed_time(completed_at - started_at)
+      end
+    end
+
+    def format_elapsed_time(time)
+      '%.10f' % time
+    end
+  end
+end

data/lib/workerholic/job_wrapper.rb

@@ -1,19 +1,21 @@
 module Workerholic
   class JobWrapper
     attr_accessor :retry_count, :execute_at
-    attr_reader :klass, :arguments, :statistics
+    attr_reader :klass, :arguments, :statistics, :wrapper
 
     def initialize(options={})
-      @klass = options[:class]
+      @klass = options[:klass]
+      @wrapper = options[:wrapper]
       @arguments = options[:arguments]
       @execute_at = options[:execute_at]
       @retry_count = options[:retry_count] || 0
-      @statistics = Statistics.new(options[:statistics] || {})
+      @statistics = JobStatistics.new(options[:statistics] || {})
     end
 
     def to_hash
       {
-        class: klass,
+        klass: klass,
+        wrapper: wrapper,
         arguments: arguments,
         retry_count: retry_count,
         execute_at: execute_at,
@@ -22,7 +24,14 @@ module Workerholic
     end
 
     def perform
-      klass.new.perform(*arguments)
+      if wrapper == ActiveJob::QueueAdapters::WorkerholicAdapter::JobWrapper
+        wrapper.new.perform(
+          'job_class' => klass,
+          'arguments' => arguments
+        )
+      else
+        klass.new.perform(*arguments)
+      end
     end
 
     def ==(other)

data/lib/workerholic/manager.rb

@@ -26,8 +26,13 @@ module Workerholic
       workers.each(&:kill)
       worker_balancer.kill
       scheduler.kill
+
+      workers.each(&:join)
+      scheduler.join
     end
 
+    private
+
 =begin
     def regenerate_workers
       inactive_workers = WORKERS_COUNT - workers.size

data/lib/workerholic/statistics_api.rb

@@ -0,0 +1,77 @@
+module Workerholic
+  class StatsAPI
+    CATEGORIES = %w(completed_jobs failed_jobs)
+
+    def self.job_statistics(options={})
+      if CATEGORIES.include? options[:category]
+        job_classes = storage.keys_for_namespace(options[:category])
+
+        if options[:count_only]
+          self.parse_job_classes(job_classes)
+        else
+          self.parse_job_classes(job_classes, false)
+        end
+      else
+        logger("Invalid arguments. Please specify one of the following categories:\n'completed_jobs', 'failed_jobs'.")
+      end
+    end
+
+    def self.jobs_classes
+      classes = storage.available_keys
+
+      parsed_classes = classes.map do |klass|
+        klass.split(':').last
+      end.uniq
+
+      parsed_classes.empty? ? 'No class data is available yet.' : parsed_classes
+    end
+
+    def self.queue_names
+      fetched_queues = storage.fetch_queue_names
+      parsed_queues = fetched_queues.map do |queue|
+        queue_data = [queue.name, queue.size]
+        queues << queue_data
+      end
+
+      (parsed_queues.empty? ? 'No queues data is available yet.': parsed_queues)
+    end
+
+    class << self
+      private
+
+      def storage
+        @storage ||= Storage::RedisWrapper.new
+      end
+
+      def logger(message)
+        @log ||= LogManager.new
+      end
+    end
+
+    private
+
+    def self.parse_job_classes(job_classes, count_only = true)
+      job_classes.map do |job_class|
+        if count_only
+          self.jobs_per_class(job_class)
+        else
+          self.get_jobs_for_class(job_class)
+        end
+      end
+    end
+
+    def self.get_jobs_for_class(job_class)
+      serialized_jobs = storage.peek_namespace(job_class)
+      deserialized_stats = serialized_jobs.map do |serialized_job|
+        JobSerializer.deserialize_stats(serialized_job)
+      end
+
+      deserialized_stats << deserialized_stats.size
+    end
+
+    def self.jobs_per_class(job_class)
+      clean_class_name = job_class.split(':').last
+      [clean_class_name, storage.list_length(job_class)]
+    end
+  end
+end

data/lib/workerholic/statistics_storage.rb

@@ -0,0 +1,19 @@
+module Workerholic
+  class StatsStorage
+
+    def self.save_job(category, job)
+      serialized_job_stats = JobSerializer.serialize(job)
+
+      namespace = "workerholic:stats:#{category}:#{job.klass}"
+      storage.push(namespace, serialized_job_stats)
+    end
+
+    class << self
+      private
+
+      def storage
+        @storage ||= Storage::RedisWrapper.new
+      end
+    end
+  end
+end

data/lib/workerholic/storage.rb

@@ -42,12 +42,37 @@ module Workerholic
         execute(retry_delay) { |conn| conn.zcount(key, 0, '+inf') }
       end
 
+      def keys_count(namespace, retry_delay = 5)
+        execute(retry_delay) { |conn| conn.keys(namespace + ':*').size }
+      end
+
       def fetch_queue_names(retry_delay = 5)
         queue_name_pattern = $TESTING ? 'workerholic:testing:queue*' : 'workerholic:queue*'
 
         execute(retry_delay) { |conn| conn.scan(0, match: queue_name_pattern).last }
       end
 
+      def available_keys(retry_delay = 5)
+        execute(retry_delay) { |conn| conn.keys('workerholic:stats:*') }
+      end
+
+      def keys_for_namespace(namespace, retry_delay = 5)
+        execute(retry_delay) { |conn| conn.keys('workerholic:stats:' + namespace + ':*') }
+      end
+
+      def peek_namespace(key, retry_delay = 5)
+        execute(retry_delay) { |conn| conn.lrange(key, 0, -1) }
+      end
+
+      def peek_namespaces(keys, retry_delay = 5)
+        execute(retry_delay) do |conn|
+          keys.select do |namespace|
+            full_namespace = 'workerholic:stats:' + namespace
+            conn.keys(full_namespace + ':*').size > 0
+          end
+        end
+      end
+
       class RedisCannotRecover < Redis::CannotConnectError; end
 
       private

data/lib/workerholic/version.rb

@@ -1,3 +1,3 @@
 module Workerholic
-  VERSION = '0.0.8'
+  VERSION = '0.0.9'
 end

data/lib/workerholic/worker.rb

@@ -26,6 +26,9 @@ module Workerholic
 
     def kill
       self.alive = false
+    end
+
+    def join
       thread.join if thread
     end
 

data/lib/workerholic/worker_balancer.rb

@@ -22,7 +22,6 @@ module Workerholic
 
     def kill
       thread.kill
-      thread.join
     end
 
     private

data/lib/workerholic.rb

@@ -18,11 +18,17 @@ require 'workerholic/job_retry'
 require 'workerholic/storage'
 require 'workerholic/sorted_set'
 require 'workerholic/queue'
+require 'workerholic/statistics_storage'
 
 require 'workerholic/job_serializer'
-require 'workerholic/statistics'
+require 'workerholic/job_statistics'
 require 'workerholic/log_manager'
 
+require 'workerholic/statistics_api'
+require 'workerholic/statistics_storage'
+
+require 'workerholic/adapters/active_job_adapter' if defined?(Rails)
+
 module Workerholic
   def self.workers_count
     @workers_count || 25

data/spec/integration/enqueuing_jobs_spec.rb

@@ -9,7 +9,7 @@ describe 'enqueuing jobs to Redis' do
       serialized_job = redis.lpop(TEST_QUEUE)
       job_from_redis = Workerholic::JobSerializer.deserialize(serialized_job)
 
-      expected_job = Workerholic::JobWrapper.new(class: SimpleJobTest, arguments: ['test job'])
+      expected_job = Workerholic::JobWrapper.new(klass: SimpleJobTest, arguments: ['test job'], wrapper: SimpleJobTest)
       expected_job.statistics.enqueued_at = job_from_redis.statistics.enqueued_at
 
       expect(job_from_redis.to_hash).to eq(expected_job.to_hash)
@@ -21,9 +21,11 @@ describe 'enqueuing jobs to Redis' do
       job_from_redis = Workerholic::JobSerializer.deserialize(serialized_job)
 
       expected_job = Workerholic::JobWrapper.new(
-        class: ComplexJobTest,
-        arguments: ['test job', { a: 1, b: 2 }, [1, 2, 3]]
+        klass: ComplexJobTest,
+        arguments: ['test job', { a: 1, b: 2 }, [1, 2, 3]],
+        wrapper: ComplexJobTest
       )
+
       expected_job.statistics.enqueued_at = job_from_redis.statistics.enqueued_at
 
       expect(job_from_redis.to_hash).to eq(expected_job.to_hash)