workerholic 0.0.14 → 0.0.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: bd7e07d27a135ac70338a061db747e0fe209c58c
4
- data.tar.gz: 7df8a6e7fc49e27e44b151be044abe3d7a383a09
3
+ metadata.gz: 32b95e0d805dd75e55329784017505a16ec2f09a
4
+ data.tar.gz: 33186490232cb74d67502377948d138726a0471e
5
5
  SHA512:
6
- metadata.gz: fe19b8b022fff53d1ad9a97148a29d9ae8d40bda40f3490cb49a9cc6bd05308d2176dc1f09ea3f68f01dfdd8867e3e77d525fbe8138b261c4ce4f418222c4af5
7
- data.tar.gz: '087d1047d3d7bd1c36732b5d4b3f503f4d9a64a80644f2fd5ef73e0841a2a9cbf7326e4edef28be3e5716e0649b0148308401c33f351679ce0b202a09ab898c2'
6
+ metadata.gz: 3b5a55dd7464d82c4565630f715518421946146228c0bc0334e90d7964db3ec51eafd3f985535d1f8a24f2be94ff828603ace9a8102cc06a36217ec596b503e0
7
+ data.tar.gz: c1c000ea9636e25a563f2577e142d875f6c311837395f012f156060856d77a6173a3d2d7faacaeed2a01e2d4d6138788b8eaa66306eaf6f9a3a72f7d2b7d6b67
data/.gitignore CHANGED
@@ -1,2 +1,3 @@
1
+ Gemfile.lock
1
2
  dump.rdb
2
3
  pkg/*.gem
data/app_test/job_test.rb CHANGED
@@ -1,3 +1,4 @@
1
+ $LOAD_PATH.unshift(__dir__ + '/../lib')
1
2
  require 'workerholic'
2
3
  require 'prime'
3
4
 
@@ -77,3 +78,19 @@ class GetPrimes
77
78
  end
78
79
  end
79
80
  end
81
+
82
+ class FutureJob
83
+ include Workerholic::Job
84
+
85
+ def perform(n)
86
+ n
87
+ end
88
+ end
89
+
90
+ class FailedJob
91
+ include Workerholic::Job
92
+
93
+ def perform(n)
94
+ raise Exception
95
+ end
96
+ end
data/app_test/run.rb CHANGED
@@ -46,6 +46,19 @@ module TestRunner
46
46
  GetPrimes.new.perform_async(n, 10)
47
47
  end
48
48
  end
49
+
50
+ def self.enqueue_delayed(num_of_cycles)
51
+ num_of_cycles.times do |n|
52
+ FutureJob.new.perform_delayed(100, n)
53
+ end
54
+ end
55
+
56
+ def self.failed_jobs(num_of_cycles)
57
+ num_of_cycles.times do |n|
58
+ FailedJob.new.perform_async(n)
59
+ end
60
+ end
49
61
  end
50
62
 
51
- TestRunner.non_blocking(2)
63
+ #TestRunner.non_blocking(10)
64
+ TestRunner.failed_jobs(1)
data/lib/workerholic.rb CHANGED
@@ -4,6 +4,7 @@ require 'connection_pool'
4
4
  require 'logger'
5
5
  require 'pry-byebug'
6
6
 
7
+ require 'workerholic/starter'
7
8
  require 'workerholic/manager'
8
9
  require 'workerholic/worker_balancer'
9
10
 
@@ -30,6 +31,8 @@ require 'workerholic/statistics_storage'
30
31
  require 'workerholic/adapters/active_job_adapter' if defined?(Rails)
31
32
 
32
33
  module Workerholic
34
+ PIDS = [Process.pid]
35
+
33
36
  def self.workers_count
34
37
  @workers_count || 25
35
38
  end
@@ -16,13 +16,13 @@ module Workerholic
16
16
 
17
17
  def run
18
18
  parse_options
19
- set_options
20
19
 
21
- load_app
22
-
23
- Manager.new(auto_balance: options[:auto_balance]).start
20
+ Starter.options = options
21
+ Starter.start
24
22
  end
25
23
 
24
+ private
25
+
26
26
  def parse_options
27
27
  @options = {}
28
28
 
@@ -34,7 +34,14 @@ module Workerholic
34
34
  end
35
35
 
36
36
  opts.on '-w', '--workers INT', 'number of concurrent workers' do |count|
37
- options[:workers] = count.to_i
37
+ count = count.to_i
38
+
39
+ if count < 1
40
+ logger.error('Invalid number of workers. Please specify a valid number of workers.')
41
+ exit
42
+ else
43
+ options[:workers] = count.to_i
44
+ end
38
45
  end
39
46
 
40
47
  opts.on '-r', '--require PATH', 'file to be required to load your application' do |file|
@@ -45,37 +52,18 @@ module Workerholic
45
52
  logger.info(opts)
46
53
  exit
47
54
  end
48
- end.parse!
49
- end
50
-
51
- def set_options
52
- Workerholic.workers_count = options[:workers] if options[:workers]
53
- end
54
55
 
55
- def load_app
56
- if File.exist?('./config/environment.rb')
57
- require File.expand_path('./config/environment.rb')
56
+ opts.on '-p', '--processes INT', 'number of processes to start in parallel' do |count|
57
+ count = count.to_i
58
58
 
59
- require 'workerholic/adapters/active_job_adapter'
60
-
61
- ActiveSupport.run_load_hooks(:before_eager_load, Rails.application)
62
- Rails.application.config.eager_load_namespaces.each(&:eager_load!)
63
- elsif options[:require]
64
- file_path = File.expand_path(options[:require])
65
-
66
- if File.exist?(file_path)
67
- require file_path
68
- else
69
- logger.info('The file you specified to load your application is not valid!')
70
-
71
- exit
59
+ if count < 1
60
+ logger.error('Invalid number of processes. Please specify a valid number of processes.')
61
+ exit
62
+ else
63
+ options[:processes] = count.to_i
64
+ end
72
65
  end
73
- else
74
- logger.info('If you are using a Rails app, make sure to navigate to your root directory before starting Workerholic!')
75
- logger.info('If you are not using a Rails app, you can load your app by using the option --require and specifying the file needing to be required in order to load your application.')
76
-
77
- exit
78
- end
66
+ end.parse!
79
67
  end
80
68
  end
81
69
  end
@@ -23,18 +23,19 @@ module Workerholic
23
23
  end
24
24
 
25
25
  def perform_delayed(*args)
26
- delay_in_sec = verify_delay(args[0])
27
- serialized_job, queue_name = prepare_job_for_enqueueing(args)
26
+ execution_time = Time.now.to_f + verify_delay(args)
27
+ serialized_job = prepare_job_for_enqueueing(args).first
28
28
 
29
- JobScheduler.new(set_name: queue_name).schedule(serialized_job, delay_in_sec)
29
+ sorted_set = SortedSet.new
30
+ sorted_set.add(serialized_job, execution_time)
30
31
  end
31
32
 
32
33
  private
33
34
 
34
- def verify_delay(delay_arg)
35
- raise ArgumentError, 'Delay argument has to be of Numeric type' unless delay_arg.is_a? Numeric
35
+ def verify_delay(args)
36
+ raise ArgumentError, 'Delay argument has to be of Numeric type' unless args[0].is_a? Numeric
36
37
 
37
- delay_arg
38
+ args.shift
38
39
  end
39
40
 
40
41
  def prepare_job_for_enqueueing(args)
@@ -17,12 +17,11 @@ module Workerholic
17
17
 
18
18
  StatsStorage.save_job('completed_jobs', job)
19
19
 
20
- @logger.info("Completed: your job from class #{job.klass} was completed on #{job.statistics.completed_at}.")
20
+ # @logger.info("Completed: your job from class #{job.klass} was completed on #{job.statistics.completed_at}.")
21
21
  rescue Exception => e
22
22
  job.statistics.errors.push([e.class, e.message])
23
23
  retry_job(job)
24
24
 
25
- @logger.error("Failed: your job from class #{job.class} was unsuccessful. Retrying in 10 seconds.")
26
25
  end
27
26
  job_result
28
27
  end
@@ -30,9 +29,13 @@ module Workerholic
30
29
  private
31
30
 
32
31
  def retry_job(job)
33
- limit_reached = JobRetry.new(job: job)
34
- if limit_reached
32
+ if JobRetry.new(job: job).retry
33
+ # @logger.error("Failed: your job from class #{job.class} was unsuccessful. Retrying in 10 secs...")
34
+ else
35
35
  job.statistics.failed_on = Time.now.to_f
36
+ StatsStorage.save_job('failed_jobs', job)
37
+
38
+ # @logger.error("Failed: your job from class #{job.class} was unsuccessful.")
36
39
  end
37
40
  end
38
41
  end
@@ -5,17 +5,10 @@ module Workerholic
5
5
  def initialize(options={})
6
6
  @job = options[:job]
7
7
  @sorted_set = options[:sorted_set] || SortedSet.new('workerholic:scheduled_jobs')
8
-
9
- self.retry
10
8
  end
11
9
 
12
- protected
13
-
14
10
  def retry
15
- if job.retry_count >= 5
16
- StatsStorage.save_job('failed_jobs', job)
17
- return false
18
- end
11
+ return if job.retry_count >= 5
19
12
 
20
13
  increment_retry_count
21
14
  schedule_job_for_retry
@@ -4,7 +4,7 @@ module Workerholic
4
4
  attr_accessor :alive
5
5
 
6
6
  def initialize(opts={})
7
- @sorted_set = SortedSet.new(opts[:set_name] || 'workerholic:scheduled_jobs')
7
+ @sorted_set = opts[:sorted_set] || SortedSet.new
8
8
  @queue = Queue.new(opts[:queue_name] || 'workerholic:queue:main')
9
9
  @alive = true
10
10
  end
@@ -1,7 +1,7 @@
1
1
  module Workerholic
2
2
  # Handles polling from Redis and hands job to worker
3
3
  class Manager
4
- attr_reader :workers, :scheduler, :worker_balancer
4
+ attr_reader :workers, :scheduler, :worker_balancer, :logger
5
5
 
6
6
  def initialize(opts = {})
7
7
  @workers = []
@@ -9,16 +9,20 @@ module Workerholic
9
9
 
10
10
  @scheduler = JobScheduler.new
11
11
  @worker_balancer = WorkerBalancer.new(workers: workers, auto_balance: opts[:auto_balance])
12
+
13
+ @logger = LogManager.new
12
14
  end
13
15
 
14
16
  def start
15
17
  worker_balancer.start
16
18
  workers.each(&:work)
17
19
  scheduler.start
20
+
18
21
  sleep
19
22
  rescue SystemExit, Interrupt
20
- puts "\nWorkerholic is now shutting down. We are letting the workers finish their current jobs..."
23
+ logger.info("Workerholic's process #{Process.pid} is gracefully shutting down, letting workers finish their current jobs...")
21
24
  shutdown
25
+
22
26
  exit
23
27
  end
24
28
 
@@ -26,20 +30,10 @@ module Workerholic
26
30
  workers.each(&:kill)
27
31
  worker_balancer.kill
28
32
  scheduler.kill
33
+ Starter.kill_memory_tracker_thread
29
34
 
30
35
  workers.each(&:join)
31
36
  scheduler.join
32
37
  end
33
-
34
- private
35
-
36
- =begin
37
- def regenerate_workers
38
- inactive_workers = WORKERS_COUNT - workers.size
39
- if inactive_workers > 0
40
- inactive_workers.times { @workers << Worker.new }
41
- end
42
- end
43
- =end
44
38
  end
45
39
  end
@@ -0,0 +1,106 @@
1
+ module Workerholic
2
+ class Starter
3
+ def self.options=(opts={})
4
+ @options = opts
5
+ end
6
+
7
+ def self.start
8
+ apply_options
9
+ load_app
10
+ track_memory_usage
11
+ launch
12
+ end
13
+
14
+ def self.kill_memory_tracker_thread
15
+ @thread.kill
16
+ end
17
+
18
+ private
19
+
20
+ def self.options
21
+ @options
22
+ end
23
+
24
+ def self.logger
25
+ @logger ||= LogManager.new
26
+ end
27
+
28
+ def self.apply_options
29
+ Workerholic.workers_count = options[:workers] if options[:workers]
30
+ end
31
+
32
+ def self.load_app
33
+ if File.exist?('./config/environment.rb')
34
+ load_rails
35
+ elsif options[:require]
36
+ load_specified_file
37
+ else
38
+ display_app_load_info
39
+ end
40
+ end
41
+
42
+ def self.load_rails
43
+ require File.expand_path('./config/environment.rb')
44
+
45
+ require 'workerholic/adapters/active_job_adapter'
46
+
47
+ ActiveSupport.run_load_hooks(:before_eager_load, Rails.application)
48
+ Rails.application.config.eager_load_namespaces.each(&:eager_load!)
49
+ end
50
+
51
+ def self.load_specified_file
52
+ file_path = File.expand_path(options[:require])
53
+
54
+ if File.exist?(file_path)
55
+ require file_path
56
+ else
57
+ logger.info('The file you specified to load your application is not valid!')
58
+
59
+ exit
60
+ end
61
+ end
62
+
63
+ def self.display_app_load_info
64
+ logger.info('If you are using a Rails app, make sure to navigate to your root directory before starting Workerholic!')
65
+ logger.info('If you are not using a Rails app, you can load your app by using the option --require and specifying the file needing to be required in order to load your application.')
66
+
67
+ exit
68
+ end
69
+
70
+ def self.track_memory_usage
71
+ cleanup_old_memory_stats
72
+
73
+ @thread = Thread.new do
74
+ loop do
75
+ sleep 5
76
+ StatsStorage.save_processes_memory_usage
77
+ end
78
+ end
79
+ end
80
+
81
+ def self.cleanup_old_memory_stats
82
+ StatsStorage.delete_memory_stats
83
+ end
84
+
85
+ def self.launch
86
+ if options[:processes] && options[:processes] > 1
87
+ begin
88
+ fork_processes
89
+ sleep
90
+ rescue SystemExit, Interrupt
91
+ exit
92
+ end
93
+ else
94
+ Manager.new(auto_balance: options[:auto_balance]).start
95
+ end
96
+ end
97
+
98
+ def self.fork_processes
99
+ options[:processes].times do
100
+ PIDS << fork { Manager.new(auto_balance: options[:auto_balance]).start }
101
+ end
102
+
103
+ PIDS.freeze
104
+ end
105
+ end
106
+ end
@@ -4,7 +4,7 @@ module Workerholic
4
4
 
5
5
  def self.job_statistics(options={})
6
6
  if CATEGORIES.include? options[:category]
7
- job_classes = storage.keys_for_namespace(options[:category])
7
+ job_classes = storage.get_keys_for_namespace('workerholic:stats:' + options[:category] + ':*')
8
8
 
9
9
  if options[:count_only]
10
10
  self.parse_job_classes(job_classes)
@@ -16,8 +16,18 @@ module Workerholic
16
16
  end
17
17
  end
18
18
 
19
+ def self.scheduled_jobs(options={})
20
+ namespace = 'workerholic:scheduled_jobs'
21
+ if options[:count_only]
22
+ storage.sorted_set_members_count(namespace)
23
+ else
24
+ serialized_jobs = storage.sorted_set_members(namespace)
25
+ parse_scheduled_jobs(serialized_jobs)
26
+ end
27
+ end
28
+
19
29
  def self.jobs_classes
20
- classes = storage.available_keys
30
+ classes = storage.get_keys_for_namespace('workerholic:stats:*')
21
31
 
22
32
  parsed_classes = classes.map do |klass|
23
33
  klass.split(':').last
@@ -29,14 +39,22 @@ module Workerholic
29
39
  def self.queued_jobs
30
40
  fetched_queues = storage.fetch_queue_names
31
41
  parsed_queues = fetched_queues.map do |queue|
32
- clean_queue_name = queue.split(':').last
33
- [clean_queue_name, storage.list_length(queue)]
42
+ [queue, storage.list_length(queue)]
34
43
  end
35
44
 
36
- # (parsed_queues.empty? ? 'No queues data is available yet.': parsed_queues)
37
45
  parsed_queues
38
46
  end
39
47
 
48
+ def self.process_stats
49
+ namespace = 'workerholic:stats:memory:processes'
50
+ storage.hash_get_all(namespace)
51
+ end
52
+
53
+ def self.active_proccesses
54
+ namespace = 'workerholic:stats:memory:processes'
55
+ storage.hash_keys(namespace)
56
+ end
57
+
40
58
  private
41
59
 
42
60
  def self.storage
@@ -47,6 +65,13 @@ module Workerholic
47
65
  @log ||= LogManager.new
48
66
  end
49
67
 
68
+ def self.parse_scheduled_jobs(jobs)
69
+ jobs.map do |job|
70
+ deserialized_job = JobSerializer.deserialize_stats(job)
71
+ self.convert_klass_to_string(deserialized_job)
72
+ end
73
+ end
74
+
50
75
  def self.parse_job_classes(job_classes, count_only = true)
51
76
  job_classes.map do |job_class|
52
77
  if count_only
@@ -58,9 +83,10 @@ module Workerholic
58
83
  end
59
84
 
60
85
  def self.get_jobs_for_class(job_class)
61
- serialized_jobs = storage.peek_namespace(job_class)
86
+ serialized_jobs = storage.get_all_elements_from_list(job_class)
62
87
  deserialized_stats = serialized_jobs.map do |serialized_job|
63
- JobSerializer.deserialize_stats(serialized_job)
88
+ deserialized_job = JobSerializer.deserialize_stats(serialized_job)
89
+ self.convert_klass_to_string(deserialized_job)
64
90
  end
65
91
 
66
92
  deserialized_stats << deserialized_stats.size
@@ -70,5 +96,11 @@ module Workerholic
70
96
  clean_class_name = job_class.split(':').last
71
97
  [clean_class_name, storage.list_length(job_class)]
72
98
  end
99
+
100
+ def self.convert_klass_to_string(obj)
101
+ obj[:klass] = obj[:klass].to_s
102
+ obj[:wrapper] = obj[:wrapper].to_s
103
+ obj
104
+ end
73
105
  end
74
106
  end