workerholic 0.0.15 → 0.0.16

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 32b95e0d805dd75e55329784017505a16ec2f09a
-  data.tar.gz: 33186490232cb74d67502377948d138726a0471e
+  metadata.gz: dfae6532e0efa5d65df17f7809ecc5130e50d85b
+  data.tar.gz: b23e1f15e6ba5f1bc1494f98c01eefda76969bea
 SHA512:
-  metadata.gz: 3b5a55dd7464d82c4565630f715518421946146228c0bc0334e90d7964db3ec51eafd3f985535d1f8a24f2be94ff828603ace9a8102cc06a36217ec596b503e0
-  data.tar.gz: c1c000ea9636e25a563f2577e142d875f6c311837395f012f156060856d77a6173a3d2d7faacaeed2a01e2d4d6138788b8eaa66306eaf6f9a3a72f7d2b7d6b67
+  metadata.gz: 573cd2b6034209e578c3305ffdcec17a6a2da39d20f74b84e747a8a2a7e87b8f7ffc7c4158cdd5f3a39183069151af4bf5ddd7b5a96ec27ec782febdce0cedb1
+  data.tar.gz: fb8a13ed3494db31e9177ddbddcac240121b3bd24ec603aa4f521bc8b676b6df49af99e3af87791a49bdb3619a4eaa3171b75edc7203925f7ff256e8b8263b1a

data/Gemfile

@@ -8,6 +8,7 @@ gem 'sinatra'
 
 group :development, :test do
   gem 'pry-byebug'
+  gem 'sinatra-reloader'
 end
 
 group :test do

data/app_test/bm.rb

@@ -0,0 +1,45 @@
+require 'benchmark'
+
+class FibCruncher
+  def self.perform(n)
+    a, b = 0, 1
+    while b < n
+      a, b = b, a + b
+    end
+    b
+  end
+end
+
+def run_benchmark(n)
+  Benchmark.bm do |r|
+    r.report do
+      n.times do
+        FibCruncher.perform(1_000)
+      end
+    end
+
+    r.report do
+      n.times do
+        FibCruncher.perform(1_000_00)
+      end
+    end
+
+    r.report do
+      n.times do
+        FibCruncher.perform(1_000_000)
+      end
+    end
+
+    r.report do
+      n.times do
+        FibCruncher.perform(1_000_000_0)
+      end
+    end
+
+    r.report do
+      n.times do
+        FibCruncher.perform(1_000_000_000)
+      end
+    end
+  end
+end

data/app_test/job_test.rb

@@ -4,7 +4,7 @@ require 'prime'
 
 class JobTestFast
   include Workerholic::Job
-  job_options queue_name: 'workerholic:queue:job_fast'
+  job_options queue_name: 'job_fast'
 
   def perform(str, num)
     str
@@ -13,7 +13,7 @@ end
 
 class JobTestSlow
   include Workerholic::Job
-  job_options queue_name: 'workerholic:queue:job_slow'
+  job_options queue_name: 'job_slow'
 
   def perform(str, num)
     sleep(0.5)
@@ -46,8 +46,18 @@ end
 class HeavyCalculation
   include Workerholic::Job
 
-  def perform(n, arr)
-    arr = bubble_sort(arr)
+  def perform(arr_size)
+    #arr = bubble_sort(arr)
+    generate_large_array(arr_size)
+  end
+
+  def generate_large_array(arr_size)
+    result = []
+    arr_size.times do
+      result << rand(50 ** 10)
+    end
+
+    result
   end
 
   def bubble_sort(array)
@@ -87,6 +97,15 @@ class FutureJob
   end
 end
 
+class FutureJobWithQueue
+  include Workerholic::Job
+  job_options queue_name: 'specific_queue_for_delayed_jobs'
+
+  def perform(n)
+    n
+  end
+end
+
 class FailedJob
   include Workerholic::Job
 
@@ -94,3 +113,26 @@ class FailedJob
     raise Exception
   end
 end
+
+class FailedJobWithQueue
+  include Workerholic::Job
+  job_options queue_name: 'specific_queue_for_failed_jobs'
+
+  def perform(n)
+    raise Exception
+  end
+end
+
+class FibCruncher
+  include Workerholic::Job
+
+  def perform(n)
+    a, b = 0, 1
+
+    while b < n
+      a, b = b, a + b
+    end
+
+    puts b
+  end
+end

data/app_test/run.rb

@@ -27,11 +27,9 @@ module TestRunner
     end
   end
 
-  def self.sort_array(num_of_cycles, array_size)
-    unsorted_array = (0..array_size).to_a.shuffle
-
-    num_of_cycles.times do |n|
-      HeavyCalculation.new.perform_async(n, unsorted_array)
+  def self.generate_array(num_of_cycles, array_size)
+    num_of_cycles.times do
+      HeavyCalculation.new.perform_async(array_size)
     end
   end
 
@@ -49,7 +47,13 @@ module TestRunner
 
   def self.enqueue_delayed(num_of_cycles)
     num_of_cycles.times do |n|
-      FutureJob.new.perform_delayed(100, n)
+      FutureJob.new.perform_delayed(10, n)
+    end
+  end
+
+  def self.enqueue_delayed_with_queue(num_of_cycles)
+    num_of_cycles.times do |n|
+      FutureJobWithQueue.new.perform_delayed(10, n)
     end
   end
 
@@ -58,7 +62,34 @@ module TestRunner
       FailedJob.new.perform_async(n)
     end
   end
+
+  def self.failed_jobs_with_queue(num_of_cycles)
+    num_of_cycles.times do |n|
+      FailedJobWithQueue.new.perform_async(n)
+    end
+  end
+
+  def self.multiple_processes(num_of_proc, jobs)
+    pids = (1..num_of_proc).to_a.map do
+      fork do
+        jobs.each do |job|
+          TestRunner.send(:job[0], job[1..-1])
+        end
+
+        exit
+      end
+    end
+
+    pids.each { |pid| Process.wait(pid) }
+  end
+
+  def self.fibonacci_cruncher(num_of_cycles)
+    num_of_cycles.times do |n|
+      FibCruncher.new.perform_async(1_000)
+    end
+  end
 end
 
-#TestRunner.non_blocking(10)
-TestRunner.failed_jobs(1)
+TestRunner.non_blocking(200)
+TestRunner.blocking(50000)
+TestRunner.fibonacci_cruncher(100)

data/lib/workerholic/job.rb

@@ -10,7 +10,7 @@ module Workerholic
         define_method(:specified_job_options) do
           {
             execute_at: params[:execute_at],
-            queue_name: params[:queue_name] || 'workerholic:queue:main'
+            queue_name: 'workerholic:queue:' + (params[:queue_name] || 'main')
           }
         end
       end
@@ -41,7 +41,12 @@ module Workerholic
     def prepare_job_for_enqueueing(args)
       raise ArgumentError if self.method(:perform).arity != args.size
 
-      job = JobWrapper.new(klass: @class || self.class, arguments: args, wrapper: self.class)
+      job = JobWrapper.new(
+        klass: @class || self.class,
+        arguments: args,
+        wrapper: self.class,
+        queue: specified_job_options[:queue_name]
+      )
 
       job.statistics.enqueued_at = Time.now.to_f
 

data/lib/workerholic/job_processor.rb

@@ -16,6 +16,7 @@ module Workerholic
         job.statistics.completed_at = Time.now.to_f
 
         StatsStorage.save_job('completed_jobs', job)
+        StatsStorage.update_historical_stats('completed_jobs', job.klass.name)
 
         # @logger.info("Completed: your job from class #{job.klass} was completed on #{job.statistics.completed_at}.")
       rescue Exception => e
@@ -34,6 +35,7 @@ module Workerholic
       else
         job.statistics.failed_on = Time.now.to_f
         StatsStorage.save_job('failed_jobs', job)
+        StatsStorage.update_historical_stats('failed_jobs', job.klass.name)
 
         # @logger.error("Failed: your job from class #{job.class} was unsuccessful.")
       end

data/lib/workerholic/job_scheduler.rb

@@ -1,11 +1,10 @@
 module Workerholic
   class JobScheduler
-    attr_reader :sorted_set, :queue, :scheduler_thread
+    attr_reader :sorted_set, :scheduler_thread
     attr_accessor :alive
 
     def initialize(opts={})
       @sorted_set = opts[:sorted_set] || SortedSet.new
-      @queue = Queue.new(opts[:queue_name] || 'workerholic:queue:main')
       @alive = true
     end
 
@@ -31,8 +30,12 @@ module Workerholic
       if job_due?
         while job_due?
           serialized_job, job_execution_time = sorted_set.peek
-          sorted_set.remove(job_execution_time)
+          job = JobSerializer.deserialize(serialized_job)
+          queue = job.queue ? Queue.new(job.queue) : Queue.new
+
           queue.enqueue(serialized_job)
+
+          sorted_set.remove(job_execution_time)
         end
       else
         sleep(2)

data/lib/workerholic/job_wrapper.rb

@@ -1,12 +1,13 @@
 module Workerholic
   class JobWrapper
     attr_accessor :retry_count, :execute_at
-    attr_reader :klass, :arguments, :statistics, :wrapper
+    attr_reader :klass, :arguments, :statistics, :wrapper, :queue
 
     def initialize(options={})
       @klass = options[:klass]
       @wrapper = options[:wrapper]
       @arguments = options[:arguments]
+      @queue = options[:queue]
       @execute_at = options[:execute_at]
       @retry_count = options[:retry_count] || 0
       @statistics = JobStatistics.new(options[:statistics] || {})
@@ -17,6 +18,7 @@ module Workerholic
         klass: klass,
         wrapper: wrapper,
         arguments: arguments,
+        queue: queue,
         retry_count: retry_count,
         execute_at: execute_at,
         statistics: statistics.to_hash

data/lib/workerholic/starter.rb

@@ -7,12 +7,13 @@ module Workerholic
     def self.start
       apply_options
       load_app
-      track_memory_usage
+      track_memory_usage_and_expire_job_stats
       launch
     end
 
     def self.kill_memory_tracker_thread
       @thread.kill
+      StatsStorage.delete_memory_stats
     end
 
     private
@@ -67,21 +68,16 @@ module Workerholic
       exit
     end
 
-    def self.track_memory_usage
-      cleanup_old_memory_stats
-
+    def self.track_memory_usage_and_expire_job_stats
       @thread = Thread.new do
         loop do
           sleep 5
           StatsStorage.save_processes_memory_usage
+          StatsStorage.delete_expired_job_stats
         end
       end
     end
 
-    def self.cleanup_old_memory_stats
-      StatsStorage.delete_memory_stats
-    end
-
     def self.launch
       if options[:processes] && options[:processes] > 1
         begin

data/lib/workerholic/statistics_api.rb

@@ -1,39 +1,52 @@
 module Workerholic
   class StatsAPI
     CATEGORIES = %w(completed_jobs failed_jobs)
+    POLLING_INTERVAL = 10
 
     def self.job_statistics(options={})
-      if CATEGORIES.include? options[:category]
-        job_classes = storage.get_keys_for_namespace('workerholic:stats:' + options[:category] + ':*')
+      raise ArgumentError, "Please specify one of the following categories: 'completed_jobs', 'failed_jobs'" unless CATEGORIES.include? options[:category]
 
-        if options[:count_only]
-          self.parse_job_classes(job_classes)
-        else
-          self.parse_job_classes(job_classes, false)
-        end
+      job_classes = storage.get_keys_for_namespace("workerholic:stats:#{options[:category]}:*")
+
+      if options[:count_only]
+        parse_job_classes(job_classes)
       else
-        logger("Invalid arguments. Please specify one of the following categories:\n'completed_jobs', 'failed_jobs'.")
+        parse_job_classes(job_classes, false)
+      end
+    end
+
+    def self.job_statistics_history(category)
+      raise ArgumentError, "Please specify one of the following categories: 'completed_jobs', 'failed_jobs'" unless CATEGORIES.include? category
+
+      current_time = Time.now.to_i
+      all_job_stats(category).reduce([]) do |result, job|
+        completed_time = job.last.to_i
+        index = (current_time - completed_time) / POLLING_INTERVAL
+
+        result[index] = result[index] ? result[index] + 1 : 1
+
+        result
       end
     end
 
     def self.scheduled_jobs(options={})
       namespace = 'workerholic:scheduled_jobs'
       if options[:count_only]
-        storage.sorted_set_members_count(namespace)
+        storage.sorted_set_size(namespace)
       else
-        serialized_jobs = storage.sorted_set_members(namespace)
+        serialized_jobs = storage.sorted_set_all_members(namespace)
         parse_scheduled_jobs(serialized_jobs)
       end
     end
 
-    def self.jobs_classes
-      classes = storage.get_keys_for_namespace('workerholic:stats:*')
+    def self.jobs_classes(historical)
+      base_namespace = historical ? 'workerholic:stats:historical:' : 'workerholic:stats:'
 
-      parsed_classes = classes.map do |klass|
-        klass.split(':').last
-      end.uniq
+      completed_classes = storage.get_keys_for_namespace( base_namespace + 'completed_jobs:*')
+      failed_classes = storage.get_keys_for_namespace(base_namespace + 'failed_jobs:*')
+      combined_classes = completed_classes + failed_classes
 
-      parsed_classes.empty? ? 'No class data is available yet.' : parsed_classes
+      combined_classes.map { |klass| klass.split(':').last }.uniq
     end
 
     def self.queued_jobs
@@ -55,38 +68,62 @@ module Workerholic
       storage.hash_keys(namespace)
     end
 
+    def self.history_for_period(options={})
+      raise ArgumentError, 'Please provide a category namespace' unless options[:category]
+
+      if options[:klass]
+        namespace = "workerholic:stats:historical:#{options[:category]}:#{options[:klass]}"
+      else
+        namespace = "workerholic:stats:historical:#{options[:category]}"
+      end
+
+      period = options[:period] || 30
+      date_ranges = get_past_dates(period)
+
+      job_counts = storage.hash_get_multiple_elements(namespace, date_ranges)
+
+      combine_ranges(job_counts: job_counts, date_ranges: date_ranges)
+    end
+
     private
 
-    def self.storage
-      @storage ||= Storage::RedisWrapper.new
+    def self.combine_ranges(options={})
+      job_counts = options[:job_counts]
+      job_counts.map!(&:to_i)
+
+      {
+        date_ranges: options[:date_ranges],
+        job_counts: job_counts
+      }
     end
 
-    def self.logger(message)
-      @log ||= LogManager.new
+    def self.get_past_dates(days)
+      today = Time.now.utc.to_i - Time.now.utc.to_i % 86400
+
+      (0..days).map { |day| today - day * 86400 }
     end
 
     def self.parse_scheduled_jobs(jobs)
       jobs.map do |job|
         deserialized_job = JobSerializer.deserialize_stats(job)
-        self.convert_klass_to_string(deserialized_job)
+        convert_klass_to_string(deserialized_job)
       end
     end
 
     def self.parse_job_classes(job_classes, count_only = true)
       job_classes.map do |job_class|
         if count_only
-          self.jobs_per_class(job_class)
+          jobs_per_class(job_class)
         else
-          self.get_jobs_for_class(job_class)
+          get_jobs_for_class(job_class)
         end
       end
     end
 
     def self.get_jobs_for_class(job_class)
-      serialized_jobs = storage.get_all_elements_from_list(job_class)
+      serialized_jobs = storage.sorted_set_all_members(job_class)
       deserialized_stats = serialized_jobs.map do |serialized_job|
-        deserialized_job = JobSerializer.deserialize_stats(serialized_job)
-        self.convert_klass_to_string(deserialized_job)
+        JobSerializer.deserialize_stats(serialized_job)
       end
 
       deserialized_stats << deserialized_stats.size
@@ -94,13 +131,29 @@ module Workerholic
 
     def self.jobs_per_class(job_class)
       clean_class_name = job_class.split(':').last
-      [clean_class_name, storage.list_length(job_class)]
+      [clean_class_name, storage.sorted_set_size(job_class)]
     end
 
     def self.convert_klass_to_string(obj)
       obj[:klass] = obj[:klass].to_s
-      obj[:wrapper] = obj[:wrapper].to_s
+      obj[:wrapper] = nil
       obj
     end
+
+    def self.storage
+      @storage ||= Storage::RedisWrapper.new
+    end
+
+    def self.logger(message)
+      @log ||= LogManager.new
+    end
+
+    def self.all_job_stats(category)
+      current_time = Time.now.to_i
+
+      jobs_classes(false).map do |klass|
+        storage.sorted_set_range_members("workerholic:stats:#{category}:#{klass}", current_time - 100 * POLLING_INTERVAL, current_time)
+      end.flatten(1)
+    end
   end
 end

data/lib/workerholic/statistics_storage.rb

@@ -1,11 +1,21 @@
 module Workerholic
   class StatsStorage
-
     def self.save_job(category, job)
-      serialized_job_stats = JobSerializer.serialize(job)
+      job_hash = job.to_hash
+      job_hash[:klass] = job.klass.to_s
+      serialized_job_stats = JobSerializer.serialize(job_hash)
 
       namespace = "workerholic:stats:#{category}:#{job.klass}"
-      storage.push(namespace, serialized_job_stats)
+      storage.add_to_set(namespace, job.statistics.completed_at, serialized_job_stats)
+    end
+
+    def self.update_historical_stats(category, klass)
+      current_day_secs = Time.now.utc.to_i - (Time.now.utc.to_i % 86400)
+      namespace = "workerholic:stats:historical:#{category}"
+      namespace_with_class = "workerholic:stats:historical:#{category}:#{klass}"
+
+      storage.hash_increment_field(namespace, current_day_secs, 1)
+      storage.hash_increment_field(namespace_with_class, current_day_secs, 1)
     end
 
     def self.save_processes_memory_usage
@@ -19,6 +29,14 @@ module Workerholic
       storage.delete('workerholic:stats:memory:processes')
     end
 
+    def self.delete_expired_job_stats
+      max_time = Time.now.to_i - 1001
+      StatsAPI.jobs_classes(false).each do |klass|
+        storage.remove_range_from_set("workerholic:stats:completed_jobs:#{klass}", 0, max_time)
+        storage.remove_range_from_set("workerholic:stats:failed_jobs:#{klass}", 0, max_time)
+      end
+    end
+
     class << self
       private
 

data/lib/workerholic/storage.rb

@@ -1,6 +1,5 @@
 module Workerholic
   class Storage
-    # Wraps redis-rb gem methods for enqueueing/dequeuing purposes
     class RedisWrapper
       attr_reader :redis, :retries
 
@@ -21,6 +20,22 @@ module Workerholic
         execute(retry_delay) { |conn| conn.hget(key, field) }
       end
 
+      def hash_get_all(key, retry_delay = 5)
+        execute(retry_delay) { |conn| conn.hgetall(key) }
+      end
+
+      def hash_keys(namespace, retry_delay = 5)
+        execute(retry_delay) { |conn| conn.hkeys(namespace) }
+      end
+
+      def hash_get_multiple_elements(key, fields, retry_delay = 5)
+        execute(retry_delay) { |conn| conn.hmget(key, *fields) }
+      end
+
+      def hash_increment_field(key, field, increment, retry_delay = 5)
+        execute(retry_delay) { |conn| conn.hincrby(key, field, increment) }
+      end
+
       def delete(key, retry_delay = 5)
         execute(retry_delay) { |conn| conn.del(key) }
       end
@@ -51,15 +66,19 @@ module Workerholic
       end
 
       def sorted_set_size(key, retry_delay = 5)
-        execute(retry_delay) { |conn| conn.zcount(key, 0, '+inf') }
+        execute(retry_delay) { |conn| conn.zcard(key) }
       end
 
-      def sorted_set_members(key, retry_delay = 5)
+      def sorted_set_all_members(key, retry_delay = 5)
         execute(retry_delay) { |conn| conn.zrange(key, 0, -1) }
       end
 
-      def sorted_set_members_count(key, retry_delay = 5)
-        execute(retry_delay) { |conn| conn.zcard(key) }
+      def sorted_set_range_members(key, minscore, maxscore, retry_delay = 5)
+        execute(retry_delay) { |conn| conn.zrangebyscore(key, minscore, maxscore, with_scores: true) }
+      end
+
+      def remove_range_from_set(key, minscore, maxscore, retry_delay = 5)
+        execute(retry_delay) { |conn| conn.zremrangebyscore(key, minscore, maxscore) }
       end
 
       def keys_count(namespace, retry_delay = 5)
@@ -67,9 +86,9 @@ module Workerholic
       end
 
       def fetch_queue_names(retry_delay = 5)
-        queue_name_pattern = $TESTING ? 'workerholic:testing:queue*' : 'workerholic:queue*'
+        queue_name_pattern = 'workerholic:queue:*'
 
-        execute(retry_delay) { |conn| conn.scan(0, match: queue_name_pattern).last }
+        execute(retry_delay) { |conn| conn.keys(queue_name_pattern) }
       end
 
       def get_keys_for_namespace(namespace, retry_delay = 5)
@@ -80,18 +99,6 @@ module Workerholic
         execute(retry_delay) { |conn| conn.lrange(key, 0, -1) }
       end
 
-      def hash_get(key, field, retry_delay = 5)
-        execute(retry_delay) { |conn| conn.hget(key, field) }
-      end
-
-      def hash_get_all(key, retry_delay = 5)
-        execute(retry_delay) { |conn| conn.hgetall(key) }
-      end
-
-      def hash_keys(namespace, retry_delay = 5)
-        execute(retry_delay) { |conn| conn.hkeys(namespace) }
-      end
-
       class RedisCannotRecover < Redis::CannotConnectError; end
 
       private

data/lib/workerholic/version.rb

@@ -1,3 +1,3 @@
 module Workerholic
-  VERSION = '0.0.15'
+  VERSION = '0.0.16'
 end

data/lib/workerholic.rb

@@ -32,6 +32,7 @@ require 'workerholic/adapters/active_job_adapter' if defined?(Rails)
 
 module Workerholic
   PIDS = [Process.pid]
+  REDIS_URL = ENV['REDIS_URL'] || 'redis://localhost:' + ($TESTING ? '1234' : '6379')
 
   def self.workers_count
     @workers_count || 25
@@ -43,6 +44,8 @@ module Workerholic
   end
 
   def self.redis_pool
-    @redis ||= ConnectionPool.new(size: workers_count + 5, timeout: 5) { Redis.new }
+    @redis ||= ConnectionPool.new(size: workers_count + 5, timeout: 5) do
+      Redis.new(url: REDIS_URL)
+    end
   end
 end

data/spec/helpers/helper_methods.rb

@@ -1,7 +1,9 @@
-TEST_QUEUE = 'workerholic:testing:queue:test_queue'
-ANOTHER_TEST_QUEUE = 'workerholic:testing:queue:another_test_queue'
-BALANCER_TEST_QUEUE = 'workerholic:testing:queue:balancer_test_queue'
-ANOTHER_BALANCER_TEST_QUEUE = 'workerholic:testing:queue:another_balancer_test_queue'
+WORKERHOLIC_QUEUE_NAMESPACE = 'workerholic:queue:'
+
+TEST_QUEUE = 'test_queue'
+ANOTHER_TEST_QUEUE = 'another_test_queue'
+BALANCER_TEST_QUEUE = 'balancer_test_queue'
+ANOTHER_BALANCER_TEST_QUEUE = 'another_balancer_test_queue'
 TEST_SCHEDULED_SORTED_SET = 'workerholic:testing:scheduled_jobs'
 HASH_TEST = 'workerholic:testing:hash_test'