workerholic 0.0.14 → 0.0.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bd7e07d27a135ac70338a061db747e0fe209c58c
-  data.tar.gz: 7df8a6e7fc49e27e44b151be044abe3d7a383a09
+  metadata.gz: 32b95e0d805dd75e55329784017505a16ec2f09a
+  data.tar.gz: 33186490232cb74d67502377948d138726a0471e
 SHA512:
-  metadata.gz: fe19b8b022fff53d1ad9a97148a29d9ae8d40bda40f3490cb49a9cc6bd05308d2176dc1f09ea3f68f01dfdd8867e3e77d525fbe8138b261c4ce4f418222c4af5
-  data.tar.gz: '087d1047d3d7bd1c36732b5d4b3f503f4d9a64a80644f2fd5ef73e0841a2a9cbf7326e4edef28be3e5716e0649b0148308401c33f351679ce0b202a09ab898c2'
+  metadata.gz: 3b5a55dd7464d82c4565630f715518421946146228c0bc0334e90d7964db3ec51eafd3f985535d1f8a24f2be94ff828603ace9a8102cc06a36217ec596b503e0
+  data.tar.gz: c1c000ea9636e25a563f2577e142d875f6c311837395f012f156060856d77a6173a3d2d7faacaeed2a01e2d4d6138788b8eaa66306eaf6f9a3a72f7d2b7d6b67

data/.gitignore

@@ -1,2 +1,3 @@
+Gemfile.lock
 dump.rdb
 pkg/*.gem

data/app_test/job_test.rb

@@ -1,3 +1,4 @@
+$LOAD_PATH.unshift(__dir__ + '/../lib')
 require 'workerholic'
 require 'prime'
 
@@ -77,3 +78,19 @@ class GetPrimes
     end
   end
 end
+
+class FutureJob
+  include Workerholic::Job
+
+  def perform(n)
+    n
+  end
+end
+
+class FailedJob
+  include Workerholic::Job
+
+  def perform(n)
+    raise Exception
+  end
+end

data/app_test/run.rb

@@ -46,6 +46,19 @@ module TestRunner
       GetPrimes.new.perform_async(n, 10)
     end
   end
+
+  def self.enqueue_delayed(num_of_cycles)
+    num_of_cycles.times do |n|
+      FutureJob.new.perform_delayed(100, n)
+    end
+  end
+
+  def self.failed_jobs(num_of_cycles)
+    num_of_cycles.times do |n|
+      FailedJob.new.perform_async(n)
+    end
+  end
 end
 
-TestRunner.non_blocking(2)
+#TestRunner.non_blocking(10)
+TestRunner.failed_jobs(1)

data/lib/workerholic.rb

@@ -4,6 +4,7 @@ require 'connection_pool'
 require 'logger'
 require 'pry-byebug'
 
+require 'workerholic/starter'
 require 'workerholic/manager'
 require 'workerholic/worker_balancer'
 
@@ -30,6 +31,8 @@ require 'workerholic/statistics_storage'
 require 'workerholic/adapters/active_job_adapter' if defined?(Rails)
 
 module Workerholic
+  PIDS = [Process.pid]
+
   def self.workers_count
     @workers_count || 25
   end

data/lib/workerholic/cli.rb

@@ -16,13 +16,13 @@ module Workerholic
 
     def run
       parse_options
-      set_options
 
-      load_app
-
-      Manager.new(auto_balance: options[:auto_balance]).start
+      Starter.options = options
+      Starter.start
     end
 
+    private
+
     def parse_options
       @options = {}
 
@@ -34,7 +34,14 @@ module Workerholic
         end
 
         opts.on '-w', '--workers INT', 'number of concurrent workers' do |count|
-          options[:workers] = count.to_i
+          count = count.to_i
+
+          if count < 1
+            logger.error('Invalid number of workers. Please specify a valid number of workers.')
+            exit
+          else
+            options[:workers] = count.to_i
+          end
         end
 
         opts.on '-r', '--require PATH', 'file to be required to load your application' do |file|
@@ -45,37 +52,18 @@ module Workerholic
           logger.info(opts)
           exit
         end
-      end.parse!
-    end
-
-    def set_options
-      Workerholic.workers_count = options[:workers] if options[:workers]
-    end
 
-    def load_app
-      if File.exist?('./config/environment.rb')
-        require File.expand_path('./config/environment.rb')
+        opts.on '-p', '--processes INT', 'number of processes to start in parallel' do |count|
+          count = count.to_i
 
-        require 'workerholic/adapters/active_job_adapter'
-
-        ActiveSupport.run_load_hooks(:before_eager_load, Rails.application)
-        Rails.application.config.eager_load_namespaces.each(&:eager_load!)
-      elsif options[:require]
-        file_path = File.expand_path(options[:require])
-
-        if File.exist?(file_path)
-          require file_path
-        else
-          logger.info('The file you specified to load your application is not valid!')
-
-          exit
+          if count < 1
+            logger.error('Invalid number of processes. Please specify a valid number of processes.')
+            exit
+          else
+            options[:processes] = count.to_i
+          end
         end
-      else
-        logger.info('If you are using a Rails app, make sure to navigate to your root directory before starting Workerholic!')
-        logger.info('If you are not using a Rails app, you can load your app by using the option --require and specifying the file needing to be required in order to load your application.')
-
-        exit
-      end
+      end.parse!
     end
   end
 end

data/lib/workerholic/job.rb

@@ -23,18 +23,19 @@ module Workerholic
     end
 
     def perform_delayed(*args)
-      delay_in_sec = verify_delay(args[0])
-      serialized_job, queue_name = prepare_job_for_enqueueing(args)
+      execution_time = Time.now.to_f + verify_delay(args)
+      serialized_job = prepare_job_for_enqueueing(args).first
 
-      JobScheduler.new(set_name: queue_name).schedule(serialized_job, delay_in_sec)
+      sorted_set = SortedSet.new
+      sorted_set.add(serialized_job, execution_time)
     end
 
     private
 
-    def verify_delay(delay_arg)
-      raise ArgumentError, 'Delay argument has to be of Numeric type' unless delay_arg.is_a? Numeric
+    def verify_delay(args)
+      raise ArgumentError, 'Delay argument has to be of Numeric type' unless args[0].is_a? Numeric
 
-      delay_arg
+      args.shift
     end
 
     def prepare_job_for_enqueueing(args)

data/lib/workerholic/job_processor.rb

@@ -17,12 +17,11 @@ module Workerholic
 
         StatsStorage.save_job('completed_jobs', job)
 
-        @logger.info("Completed: your job from class #{job.klass} was completed on #{job.statistics.completed_at}.")
+        # @logger.info("Completed: your job from class #{job.klass} was completed on #{job.statistics.completed_at}.")
       rescue Exception => e
         job.statistics.errors.push([e.class, e.message])
         retry_job(job)
 
-        @logger.error("Failed: your job from class #{job.class} was unsuccessful. Retrying in 10 seconds.")
       end
       job_result
     end
@@ -30,9 +29,13 @@ module Workerholic
     private
 
     def retry_job(job)
-      limit_reached = JobRetry.new(job: job)
-      if limit_reached
+      if JobRetry.new(job: job).retry
+        # @logger.error("Failed: your job from class #{job.class} was unsuccessful. Retrying in 10 secs...")
+      else
         job.statistics.failed_on = Time.now.to_f
+        StatsStorage.save_job('failed_jobs', job)
+
+        # @logger.error("Failed: your job from class #{job.class} was unsuccessful.")
       end
     end
   end

data/lib/workerholic/job_retry.rb

@@ -5,17 +5,10 @@ module Workerholic
     def initialize(options={})
       @job = options[:job]
       @sorted_set = options[:sorted_set] || SortedSet.new('workerholic:scheduled_jobs')
-
-      self.retry
     end
 
-    protected
-
     def retry
-      if job.retry_count >= 5
-        StatsStorage.save_job('failed_jobs', job)
-        return false
-      end
+      return if job.retry_count >= 5
 
       increment_retry_count
       schedule_job_for_retry

data/lib/workerholic/job_scheduler.rb

@@ -4,7 +4,7 @@ module Workerholic
     attr_accessor :alive
 
     def initialize(opts={})
-      @sorted_set = SortedSet.new(opts[:set_name] || 'workerholic:scheduled_jobs')
+      @sorted_set = opts[:sorted_set] || SortedSet.new
       @queue = Queue.new(opts[:queue_name] || 'workerholic:queue:main')
       @alive = true
     end

data/lib/workerholic/manager.rb

@@ -1,7 +1,7 @@
 module Workerholic
   # Handles polling from Redis and hands job to worker
   class Manager
-    attr_reader :workers, :scheduler, :worker_balancer
+    attr_reader :workers, :scheduler, :worker_balancer, :logger
 
     def initialize(opts = {})
       @workers = []
@@ -9,16 +9,20 @@ module Workerholic
 
       @scheduler = JobScheduler.new
       @worker_balancer = WorkerBalancer.new(workers: workers, auto_balance: opts[:auto_balance])
+
+      @logger = LogManager.new
     end
 
     def start
       worker_balancer.start
       workers.each(&:work)
       scheduler.start
+
       sleep
     rescue SystemExit, Interrupt
-      puts "\nWorkerholic is now shutting down. We are letting the workers finish their current jobs..."
+      logger.info("Workerholic's process #{Process.pid} is gracefully shutting down, letting workers finish their current jobs...")
       shutdown
+
       exit
     end
 
@@ -26,20 +30,10 @@ module Workerholic
       workers.each(&:kill)
       worker_balancer.kill
       scheduler.kill
+      Starter.kill_memory_tracker_thread
 
       workers.each(&:join)
       scheduler.join
     end
-
-    private
-
-=begin
-    def regenerate_workers
-      inactive_workers = WORKERS_COUNT - workers.size
-      if inactive_workers > 0
-        inactive_workers.times { @workers << Worker.new }
-      end
-    end
-=end
   end
 end

data/lib/workerholic/starter.rb

@@ -0,0 +1,106 @@
+module Workerholic
+  class Starter
+    def self.options=(opts={})
+      @options = opts
+    end
+
+    def self.start
+      apply_options
+      load_app
+      track_memory_usage
+      launch
+    end
+
+    def self.kill_memory_tracker_thread
+      @thread.kill
+    end
+
+    private
+
+    def self.options
+      @options
+    end
+
+    def self.logger
+      @logger ||= LogManager.new
+    end
+
+    def self.apply_options
+      Workerholic.workers_count = options[:workers] if options[:workers]
+    end
+
+    def self.load_app
+      if File.exist?('./config/environment.rb')
+        load_rails
+      elsif options[:require]
+        load_specified_file
+      else
+        display_app_load_info
+      end
+    end
+
+    def self.load_rails
+      require File.expand_path('./config/environment.rb')
+
+      require 'workerholic/adapters/active_job_adapter'
+
+      ActiveSupport.run_load_hooks(:before_eager_load, Rails.application)
+      Rails.application.config.eager_load_namespaces.each(&:eager_load!)
+    end
+
+    def self.load_specified_file
+      file_path = File.expand_path(options[:require])
+
+      if File.exist?(file_path)
+        require file_path
+      else
+        logger.info('The file you specified to load your application is not valid!')
+
+        exit
+      end
+    end
+
+    def self.display_app_load_info
+      logger.info('If you are using a Rails app, make sure to navigate to your root directory before starting Workerholic!')
+      logger.info('If you are not using a Rails app, you can load your app by using the option --require and specifying the file needing to be required in order to load your application.')
+
+      exit
+    end
+
+    def self.track_memory_usage
+      cleanup_old_memory_stats
+
+      @thread = Thread.new do
+        loop do
+          sleep 5
+          StatsStorage.save_processes_memory_usage
+        end
+      end
+    end
+
+    def self.cleanup_old_memory_stats
+      StatsStorage.delete_memory_stats
+    end
+
+    def self.launch
+      if options[:processes] && options[:processes] > 1
+        begin
+          fork_processes
+          sleep
+        rescue SystemExit, Interrupt
+          exit
+        end
+      else
+        Manager.new(auto_balance: options[:auto_balance]).start
+      end
+    end
+
+    def self.fork_processes
+      options[:processes].times do
+        PIDS << fork { Manager.new(auto_balance: options[:auto_balance]).start }
+      end
+
+      PIDS.freeze
+    end
+  end
+end

data/lib/workerholic/statistics_api.rb

@@ -4,7 +4,7 @@ module Workerholic
 
     def self.job_statistics(options={})
       if CATEGORIES.include? options[:category]
-        job_classes = storage.keys_for_namespace(options[:category])
+        job_classes = storage.get_keys_for_namespace('workerholic:stats:' + options[:category] + ':*')
 
         if options[:count_only]
           self.parse_job_classes(job_classes)
@@ -16,8 +16,18 @@ module Workerholic
       end
     end
 
+    def self.scheduled_jobs(options={})
+      namespace = 'workerholic:scheduled_jobs'
+      if options[:count_only]
+        storage.sorted_set_members_count(namespace)
+      else
+        serialized_jobs = storage.sorted_set_members(namespace)
+        parse_scheduled_jobs(serialized_jobs)
+      end
+    end
+
     def self.jobs_classes
-      classes = storage.available_keys
+      classes = storage.get_keys_for_namespace('workerholic:stats:*')
 
       parsed_classes = classes.map do |klass|
         klass.split(':').last
@@ -29,14 +39,22 @@ module Workerholic
     def self.queued_jobs
       fetched_queues = storage.fetch_queue_names
       parsed_queues = fetched_queues.map do |queue|
-        clean_queue_name = queue.split(':').last
-        [clean_queue_name, storage.list_length(queue)]
+        [queue, storage.list_length(queue)]
       end
 
-      # (parsed_queues.empty? ? 'No queues data is available yet.': parsed_queues)
       parsed_queues
     end
 
+    def self.process_stats
+      namespace = 'workerholic:stats:memory:processes'
+      storage.hash_get_all(namespace)
+    end
+
+    def self.active_proccesses
+      namespace = 'workerholic:stats:memory:processes'
+      storage.hash_keys(namespace)
+    end
+
     private
 
     def self.storage
@@ -47,6 +65,13 @@ module Workerholic
       @log ||= LogManager.new
     end
 
+    def self.parse_scheduled_jobs(jobs)
+      jobs.map do |job|
+        deserialized_job = JobSerializer.deserialize_stats(job)
+        self.convert_klass_to_string(deserialized_job)
+      end
+    end
+
     def self.parse_job_classes(job_classes, count_only = true)
       job_classes.map do |job_class|
         if count_only
@@ -58,9 +83,10 @@ module Workerholic
     end
 
     def self.get_jobs_for_class(job_class)
-      serialized_jobs = storage.peek_namespace(job_class)
+      serialized_jobs = storage.get_all_elements_from_list(job_class)
       deserialized_stats = serialized_jobs.map do |serialized_job|
-        JobSerializer.deserialize_stats(serialized_job)
+        deserialized_job = JobSerializer.deserialize_stats(serialized_job)
+        self.convert_klass_to_string(deserialized_job)
       end
 
       deserialized_stats << deserialized_stats.size
@@ -70,5 +96,11 @@ module Workerholic
       clean_class_name = job_class.split(':').last
       [clean_class_name, storage.list_length(job_class)]
     end
+
+    def self.convert_klass_to_string(obj)
+      obj[:klass] = obj[:klass].to_s
+      obj[:wrapper] = obj[:wrapper].to_s
+      obj
+    end
   end
 end