workerholic 0.0.23 → 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 86a4ad33857310f6ae88067255e2ec98f0290e31
4
- data.tar.gz: 0047fd7373d52d09e8d93a7f297ef9e9143657f9
3
+ metadata.gz: 6e37a5ab3f970c981fbabbad367f73e8fed83ad0
4
+ data.tar.gz: c2836310d4031983846dd8778dba6de4de4e624a
5
5
  SHA512:
6
- metadata.gz: 2eccb9b6cccbec30b2c2c67232b6137fd8028b84a90dedc224698580bd3f4723785d617f00a9c7b9af338c611ba60ef1ef97dfbf1f4a2bd554a298c5e6bed026
7
- data.tar.gz: d738cd59fefb6ef6abaff08ed10d07a5397a80c18db3b093aefb1fda4cb61f67f26084def316c955ad67c83161777308406215de653a0c3b973cf637f778b26b
6
+ metadata.gz: 5998191e60aa3a283434c85a5e7bf5e5a22b60ad27e8dbdbe459eef17418760613fa00a4a688087322ebc870cc1adedb07753a591ebba653f095f75f760121ba
7
+ data.tar.gz: 5f46779c52593659ace61aa4d4eb7a184549283bf3f2d0057cb31424dee5c43b110fde84f33225f7ca157028c159f8b05b95b7ea8ad21d1f25e85c5fe9598335
data/README.md CHANGED
@@ -168,6 +168,24 @@ Use the following option if you want to have workers provisioned based on the lo
168
168
 
169
169
  This will ensure that each queue will be provisioned with a number of workers based on its relative load compared to the aggregated load for all job queues.
170
170
 
171
+ #### Optimize by specifying IO-blocking jobs
172
+
173
+ IO blocking jobs include, any type of jobs that will require the machine to spend some time on IO, such as performing requests over the wire, opening a file, `sleep`ing, etc.
174
+
175
+ You can specify that your job is IO blocking by adding `-io` at the end of the queue name you specified in your job class, like the following:
176
+
177
+ ```ruby
178
+ class MyJob < ApplicationJob
179
+ queue_as: 'my_queue-io'
180
+
181
+ def perform(args)
182
+ # job logic goes here
183
+ end
184
+ end
185
+ ```
186
+
187
+ In this case, the auto-balancing algorithm will assume that all other queues contain CPU blocking jobs and will assign only 1 worker to each of these queues, saving the rest of the workers for queues containing IO blocking jobs.
188
+
171
189
  ## Integration
172
190
  ### ActiveJob
173
191
 
@@ -12,7 +12,9 @@ module Workerholic
12
12
 
13
13
  increment_retry_count
14
14
  schedule_job_for_retry
15
- sorted_set.add(JobSerializer.serialize(job), job.execute_at)
15
+ Workerholic.manager
16
+ .scheduler
17
+ .schedule(JobSerializer.serialize(job), job.execute_at)
16
18
  end
17
19
 
18
20
  private
@@ -7,7 +7,7 @@ module Workerholic
7
7
  @workers = []
8
8
  Workerholic.workers_count.times { @workers << Worker.new }
9
9
 
10
- @scheduler = JobScheduler.new
10
+ @scheduler = JobScheduler.new(sorted_set: opts[:sorted_set])
11
11
  @worker_balancer = WorkerBalancer.new(workers: workers, auto_balance: opts[:auto_balance])
12
12
 
13
13
  @logger = LogManager.new
@@ -80,21 +80,18 @@ module Workerholic
80
80
  end
81
81
 
82
82
  def self.launch
83
- if options[:processes] && options[:processes] > 1
84
- begin
85
- fork_processes
86
- sleep
87
- rescue SystemExit, Interrupt
88
- exit
89
- end
90
- else
91
- Manager.new(auto_balance: options[:auto_balance]).start
92
- end
83
+ fork_processes if options[:processes] && options[:processes] > 1
84
+
85
+ Workerholic.manager = Manager.new(auto_balance: options[:auto_balance])
86
+ Workerholic.manager.start
93
87
  end
94
88
 
95
89
  def self.fork_processes
96
- options[:processes].times do
97
- PIDS << fork { Manager.new(auto_balance: options[:auto_balance]).start }
90
+ (options[:processes] - 1).times do
91
+ PIDS << fork do
92
+ Workerholic.manager = Manager.new(auto_balance: options[:auto_balance])
93
+ Workerholic.manager.start
94
+ end
98
95
  end
99
96
 
100
97
  PIDS.freeze
@@ -2,7 +2,6 @@ module Workerholic
2
2
  class StatsStorage
3
3
  def self.save_job(category, job)
4
4
  job_hash = job.to_hash
5
- job_hash[:klass] = job.klass.to_s
6
5
  serialized_job_stats = JobSerializer.serialize(job_hash)
7
6
 
8
7
  namespace = "workerholic:stats:#{category}:#{job.klass}"
@@ -1,3 +1,3 @@
1
1
  module Workerholic
2
- VERSION = '0.0.23'
2
+ VERSION = '0.1.0'
3
3
  end
@@ -1,4 +1,5 @@
1
1
  # $LOAD_PATH << __dir__ + '/../..'
2
+
2
3
  require 'sinatra/base'
3
4
 
4
5
  # require 'sinatra/reloader'
@@ -77,7 +78,6 @@ class WorkerholicWeb < Sinatra::Base
77
78
  failed_jobs: Workerholic::StatsAPI.job_statistics( {category: 'failed_jobs', count_only: true} ),
78
79
  queued_jobs: Workerholic::StatsAPI.queued_jobs,
79
80
  scheduled_jobs: Workerholic::StatsAPI.scheduled_jobs( { count_only: true }),
80
- workers_count: Workerholic.workers_count,
81
81
  memory_usage: Workerholic::StatsAPI.process_stats,
82
82
  completed_jobs_per_second: Workerholic::StatsAPI.job_statistics_history('completed_jobs'),
83
83
  })
@@ -4,7 +4,7 @@ var App = {
4
4
  jobsCompletedHistory: [],
5
5
  jobsCompletedPerSecondHistory: [],
6
6
  totalMemoryHistory: [],
7
- maxTime: 240,
7
+ maxTime: 1000,
8
8
  pollingInterval: 10,
9
9
  chartFont: 'arial',
10
10
  freshDataCount: function() {
@@ -38,7 +38,6 @@ var App = {
38
38
  context: this,
39
39
  success: function(data) {
40
40
  var deserializedData = JSON.parse(data);
41
- var workersCount = deserializedData.workers_count;
42
41
  var scheduledJobsCount = deserializedData.scheduled_jobs;
43
42
 
44
43
  var completedJobs = deserializedData.completed_jobs.reduce(function(sum, subArray) {
@@ -82,7 +81,6 @@ var App = {
82
81
  $('.queue_count').text(queuedJobs.length);
83
82
  $('.queued_jobs_count').text(queuedJobsCount);
84
83
  $('.scheduled_jobs').text(scheduledJobsCount);
85
- $('.workers_count').text(workersCount);
86
84
  $('.memory_usage').text(totalMemoryUsage / 1000 + ' MB');
87
85
  }
88
86
  });
@@ -457,8 +455,7 @@ var App = {
457
455
  this.tab = $(location).attr('href').match(/(?:(?!\?).)*/)[0].split('/').pop();
458
456
  var $active = $('a[href=' + this.tab + ']');
459
457
 
460
- $active.css('background', '#a2a2a2');
461
- $active.css('color', '#fff');
458
+ $active.addClass('is-active');
462
459
  },
463
460
  pollData: function(tab) {
464
461
  if (tab === 'overview') {
@@ -5,43 +5,44 @@ body {
5
5
  background: #f2f2f2;
6
6
  }
7
7
 
8
- header {
8
+ .nav {
9
9
  font-family: 'Dosis', sans-serif;
10
- font-size: 128px;
11
- color: #fff;
12
- text-align: center;
13
- background: linear-gradient(to bottom, #3A0E40, #E739FF);
14
10
  }
15
11
 
16
- section {
17
- width: 1200px;
18
- padding-bottom: 100px;
19
- margin: 0 auto;
12
+ .nav-item {
13
+ font-size: 20px;
14
+ color: #073763 !important;
20
15
  }
21
16
 
22
- footer p {
23
- display: inline-block;
24
- width: 100%;
25
- text-align: center;
17
+ .nav-item a:not(.button).is-tab:hover,
18
+ a.nav-item:not(.button).is-tab:hover {
19
+ border-bottom: 1px solid #073763;
26
20
  }
27
21
 
28
- nav {
29
- background: #fff;
22
+ .nav-item a:not(.button).is-tab.is-active,
23
+ a.nav-item:not(.button).is-tab.is-active {
24
+ border-bottom: 3px solid #073763;
30
25
  }
31
26
 
32
- nav a {
27
+ #logo {
28
+ line-height: 1;
29
+ }
30
+
31
+ #logo img {
32
+ width: 200px;
33
+ max-height: 100px;
34
+ }
35
+
36
+ footer p {
33
37
  display: inline-block;
34
38
  width: 100%;
35
- padding: 10px;
36
39
  text-align: center;
37
- border-radius: 5px;
38
- color: #000;
39
- background: #cecece;
40
40
  }
41
41
 
42
- nav a:hover {
43
- color: #fff;
44
- background: #a2a2a2;
42
+ section {
43
+ width: 1200px;
44
+ padding-bottom: 100px;
45
+ margin: 0 auto;
45
46
  }
46
47
 
47
48
  .table {
@@ -3,7 +3,7 @@
3
3
  <title>Workerholic Overview</title>
4
4
  <link href="https://fonts.googleapis.com/css?family=Dosis" rel="stylesheet">
5
5
  <link href="https://cdnjs.cloudflare.com/ajax/libs/bulma/0.4.4/css/bulma.css" rel="stylesheet">
6
- <link href='stylesheets/application.css' rel='stylesheet'>
6
+ <link href="stylesheets/application.css" rel="stylesheet">
7
7
  <script src="https://canvasjs.com/assets/script/canvasjs.min.js"></script>
8
8
  <script src="https://ajax.googleapis.com/ajax/libs/jquery/3.2.1/jquery.min.js"></script>
9
9
  <script src="javascripts/application.js"></script>
@@ -11,20 +11,23 @@
11
11
 
12
12
  <body>
13
13
  <header>
14
- Workerholic
14
+ <nav class="nav has-shadow">
15
+ <div class="container">
16
+ <div class="nav-center">
17
+ <a class="nav-item" id="logo">
18
+ <img src="/images/workerholic_logo.png" alt="Workerholic logo">
19
+ </a>
20
+ </div>
21
+ <div class="nav-center">
22
+ <a class="nav-item is-tab is-hidden-mobile" href="overview">Overview</a>
23
+ <a class="nav-item is-tab is-hidden-mobile" href="details">Details</a>
24
+ <a class="nav-item is-tab is-hidden-mobile" href="queues">Queues</a>
25
+ <a class="nav-item is-tab is-hidden-mobile" href="history">History</a>
26
+ </div>
27
+ </div>
28
+ </nav>
15
29
  </header>
16
30
 
17
- <nav>
18
- <ul class='columns'>
19
- <li class='column'><a href='overview'>Overview</a></li>
20
- <li class='column'><a href='details'>Job Details</a></li>
21
- <li class='column'><a href='queues'>Queues</a></li>
22
- <li class='column'><a href='history'>History</a></li>
23
- <!-- <li class='column'><a href='failed'>Failed</a></li>
24
- <li class='column'><a href='scheduled'>Scheduled</a></li> -->
25
- </ul>
26
- </nav>
27
-
28
31
  <section>
29
32
  <%= yield %>
30
33
  </section>
@@ -26,10 +26,6 @@
26
26
  <td>Number of Queues</td>
27
27
  <td class='queue_count count'></td>
28
28
  </tr>
29
- <tr>
30
- <td>Number of Workers</td>
31
- <td class='workers_count count'></td>
32
- </tr>
33
29
  <tr id='memory_usage'>
34
30
  <td>Total Memory Usage</td>
35
31
  <td class='memory_usage count'></td>
@@ -16,8 +16,6 @@ module Workerholic
16
16
  serialized_job = poll
17
17
  JobProcessor.new(serialized_job).process if serialized_job
18
18
  end
19
-
20
- #puts "DONE!"
21
19
  end
22
20
  rescue ThreadError => e
23
21
  @logger.info(e.message)
@@ -24,78 +24,64 @@ module Workerholic
24
24
  thread.kill
25
25
  end
26
26
 
27
- private
27
+ private unless $TESTING
28
28
 
29
- def auto_balance_workers
29
+ def evenly_balance_workers
30
30
  @thread = Thread.new do
31
31
  while alive
32
- self.queues = fetch_queues
33
-
34
- total_workers_count = assign_one_worker_per_queue
32
+ evenly_balanced_workers_distribution
33
+ output_balancer_stats
35
34
 
36
- remaining_workers_count = workers.size - (total_workers_count + 1)
37
- average_job_count_per_worker = total_jobs / remaining_workers_count.to_f
35
+ sleep 1
36
+ end
37
+ end
38
+ end
38
39
 
39
- queues.each do |queue|
40
- workers_count = queue.size / average_job_count_per_worker
40
+ def evenly_balanced_workers_distribution
41
+ self.queues = fetch_queues
41
42
 
42
- if workers_count % 1 == 0.5
43
- workers_count = workers_count.floor
44
- else
45
- workers_count = workers_count.round
46
- end
43
+ total_workers_count = assign_one_worker_per_queue
47
44
 
48
- assign_workers_to_queue(queue, workers_count, total_workers_count)
45
+ remaining_workers_count = workers.size - total_workers_count
49
46
 
50
- total_workers_count += workers_count
51
- end
47
+ queues.each do |queue|
48
+ workers_count = remaining_workers_count / queues.size
49
+ workers_count = round(workers_count)
52
50
 
53
- distribute_unassigned_worker(total_workers_count)
54
- output_balancer_stats
51
+ assign_workers_to_queue(queue, workers_count, total_workers_count)
55
52
 
56
- sleep 2
57
- end
53
+ total_workers_count += workers_count
58
54
  end
55
+
56
+ distribute_unassigned_worker(total_workers_count)
59
57
  end
60
58
 
61
- def evenly_balance_workers
59
+ def auto_balance_workers
62
60
  @thread = Thread.new do
63
61
  while alive
64
- self.queues = fetch_queues
65
-
66
- total_workers_count = assign_one_worker_per_queue
67
-
68
- remaining_workers_count = workers.size - (total_workers_count + 1)
69
-
70
- queues.each do |queue|
71
- workers_count = remaining_workers_count / queues.size
72
- assign_workers_to_queue(queue, workers_count, total_workers_count)
73
- total_workers_count += workers_count
74
- end
75
-
76
- distribute_unassigned_worker(total_workers_count)
62
+ auto_balanced_workers_distribution
77
63
  output_balancer_stats
78
64
 
79
- sleep 2
65
+ sleep 1
80
66
  end
81
67
  end
82
68
  end
83
69
 
84
- def distribute_unassigned_worker(total_workers_count)
85
- workers[workers.size - 1].queue = queues.sample if workers.size - total_workers_count == 1
86
- end
70
+ def auto_balanced_workers_distribution
71
+ self.queues = fetch_queues
87
72
 
88
- def output_balancer_stats
89
- queues_with_size = queues.map { |q| { name: q.name, size: q.size } }
73
+ total_workers_count = assign_one_worker_per_queue
90
74
 
91
- queues_with_size.each do |q|
92
- output = <<~LOG
93
- Queue #{q[:name]}:
94
- => #{q[:size]} jobs
95
- => #{current_workers_count_per_queue[q[:name]]} workers
96
- LOG
97
- @logger.info(output)
98
- end
75
+ remaining_workers_count = workers.size - total_workers_count
76
+ average_jobs_count_per_worker = total_jobs / remaining_workers_count.to_f
77
+
78
+ total_workers_count = provision_queues(io_queues, average_jobs_count_per_worker, total_workers_count)
79
+
80
+ distribute_unassigned_worker(total_workers_count)
81
+ end
82
+
83
+ def fetch_queues
84
+ storage.fetch_queue_names.map { |queue_name| Queue.new(queue_name) }
99
85
  end
100
86
 
101
87
  def assign_one_worker_per_queue
@@ -108,12 +94,31 @@ module Workerholic
108
94
  index
109
95
  end
110
96
 
111
- def fetch_queues
112
- storage.fetch_queue_names.map { |queue_name| Queue.new(queue_name) }
97
+ def total_jobs
98
+ io_queues.map(&:size).reduce(:+) || 0
113
99
  end
114
100
 
115
- def total_jobs
116
- @queues.map(&:size).reduce(:+) || 0
101
+ def io_queues
102
+ io_qs = queues.select { |q| q.name.match(/.*-io$/) }
103
+
104
+ if io_qs.empty?
105
+ queues
106
+ else
107
+ io_qs
108
+ end
109
+ end
110
+
111
+ def provision_queues(qs, average_jobs_count_per_worker, total_workers_count)
112
+ qs.each do |q|
113
+ workers_count = q.size / average_jobs_count_per_worker
114
+ workers_count = round(workers_count)
115
+
116
+ assign_workers_to_queue(q, workers_count, total_workers_count)
117
+
118
+ total_workers_count += workers_count
119
+ end
120
+
121
+ total_workers_count
117
122
  end
118
123
 
119
124
  def assign_workers_to_queue(queue, workers_count, total_workers_count)
@@ -122,6 +127,34 @@ module Workerholic
122
127
  end
123
128
  end
124
129
 
130
+ def round(n)
131
+ return n.floor if n % 1 == 0.5
132
+
133
+ n.round
134
+ end
135
+
136
+ def distribute_unassigned_worker(total_workers_count)
137
+ workers[workers.size - 1].queue = io_queues.find { |q| q.size == io_queues.map(&:size).max } if workers.size - total_workers_count == 1
138
+ end
139
+
140
+ def output_balancer_stats
141
+ queues_with_size = queues.map { |q| { name: q.name, size: q.size } }
142
+
143
+ queues_with_size.each do |q|
144
+ output = <<~LOG
145
+ Queue #{q[:name]}:
146
+ => #{q[:size]} jobs
147
+ => #{current_workers_count_per_queue[q[:name]]} workers
148
+ LOG
149
+ @logger.info(output)
150
+ end
151
+
152
+ if queues_with_size.empty?
153
+ @logger.info("DONE")
154
+ raise Interrupt
155
+ end
156
+ end
157
+
125
158
  def current_workers_count_per_queue
126
159
  workers.reduce({}) do |result, worker|
127
160
  if worker.queue
data/lib/workerholic.rb CHANGED
@@ -43,7 +43,7 @@ module Workerholic
43
43
  end
44
44
 
45
45
  def self.redis_connections_count
46
- @redis_connections_count || (workers_count + 5)
46
+ @redis_connections_count || (workers_count + 3)
47
47
  end
48
48
 
49
49
  def self.redis_connections_count=(num)
@@ -56,4 +56,12 @@ module Workerholic
56
56
  Redis.new(url: REDIS_URL)
57
57
  end
58
58
  end
59
+
60
+ def self.manager=(mgr)
61
+ @manager = mgr
62
+ end
63
+
64
+ def self.manager
65
+ @manager
66
+ end
59
67
  end
@@ -2,8 +2,12 @@ WORKERHOLIC_QUEUE_NAMESPACE = 'workerholic:queue:'
2
2
 
3
3
  TEST_QUEUE = 'test_queue'
4
4
  ANOTHER_TEST_QUEUE = 'another_test_queue'
5
- BALANCER_TEST_QUEUE = 'balancer_test_queue'
6
- ANOTHER_BALANCER_TEST_QUEUE = 'another_balancer_test_queue'
5
+
6
+ FIRST_BALANCER_TEST_QUEUE = 'first_balancer_test_queue'
7
+ SECOND_BALANCER_TEST_QUEUE = 'second_balancer_test_queue'
8
+ THIRD_BALANCER_TEST_QUEUE = 'third_balancer_test_queue'
9
+ FOURTH_BALANCER_TEST_QUEUE = 'fourth_balancer_test_queue'
10
+
7
11
  TEST_SCHEDULED_SORTED_SET = 'workerholic:testing:scheduled_jobs'
8
12
  HASH_TEST = 'workerholic:testing:hash_test'
9
13
 
@@ -18,7 +18,7 @@ end
18
18
 
19
19
  class FirstJobBalancerTest
20
20
  include Workerholic::Job
21
- job_options queue_name: BALANCER_TEST_QUEUE
21
+ job_options queue_name: FIRST_BALANCER_TEST_QUEUE
22
22
 
23
23
  def perform(str, n)
24
24
  str
@@ -27,7 +27,25 @@ end
27
27
 
28
28
  class SecondJobBalancerTest
29
29
  include Workerholic::Job
30
- job_options queue_name: ANOTHER_BALANCER_TEST_QUEUE
30
+ job_options queue_name: SECOND_BALANCER_TEST_QUEUE
31
+
32
+ def perform(str, n)
33
+ str
34
+ end
35
+ end
36
+
37
+ class ThirdJobBalancerTest
38
+ include Workerholic::Job
39
+ job_options queue_name: THIRD_BALANCER_TEST_QUEUE
40
+
41
+ def perform(str, n)
42
+ str
43
+ end
44
+ end
45
+
46
+ class FourthJobBalancerTest
47
+ include Workerholic::Job
48
+ job_options queue_name: FOURTH_BALANCER_TEST_QUEUE
31
49
 
32
50
  def perform(str, n)
33
51
  str
@@ -8,6 +8,9 @@ end
8
8
 
9
9
  describe Workerholic::JobRetry do
10
10
  let(:redis) { Redis.new(url: Workerholic::REDIS_URL) }
11
+ before do
12
+ Workerholic.manager = Workerholic::Manager.new(sorted_set: Workerholic::SortedSet.new(TEST_SCHEDULED_SORTED_SET))
13
+ end
11
14
 
12
15
  it 'increments retry count' do
13
16
  job = Workerholic::JobWrapper.new(class: JobWithError, arguments: [])
@@ -19,6 +19,7 @@ describe Workerholic::StatsStorage do
19
19
  serialized_stats = storage.sorted_set_all_members(namespace).first
20
20
 
21
21
  expect(storage.sorted_set_size(namespace)).to eq 1
22
+ expect(serialized_stats).to eq(JSON.dump(job.to_hash))
22
23
  end
23
24
 
24
25
  it 'saves process memory usage' do
@@ -1,26 +1,248 @@
1
1
  require_relative 'spec_helper'
2
2
 
3
3
  TESTED_QUEUES = [
4
- WORKERHOLIC_QUEUE_NAMESPACE + BALANCER_TEST_QUEUE,
5
- WORKERHOLIC_QUEUE_NAMESPACE + ANOTHER_BALANCER_TEST_QUEUE
4
+ WORKERHOLIC_QUEUE_NAMESPACE + FIRST_BALANCER_TEST_QUEUE,
5
+ WORKERHOLIC_QUEUE_NAMESPACE + SECOND_BALANCER_TEST_QUEUE,
6
+ WORKERHOLIC_QUEUE_NAMESPACE + THIRD_BALANCER_TEST_QUEUE,
7
+ WORKERHOLIC_QUEUE_NAMESPACE + FOURTH_BALANCER_TEST_QUEUE
6
8
  ]
7
9
 
8
10
  describe Workerholic::WorkerBalancer do
9
11
  let(:storage) { Workerholic::Storage::RedisWrapper.new }
10
12
  let(:redis) { Redis.new(url: Workerholic::REDIS_URL) }
13
+ let(:workers) do
14
+ Array.new(25).map { Workerholic::Worker.new }
15
+ end
11
16
 
12
- before do
17
+ it 'fetches queues' do
13
18
  100.times do |n|
14
19
  FirstJobBalancerTest.new.perform_async('first string', n)
15
20
  SecondJobBalancerTest.new.perform_async('second string', n)
21
+ ThirdJobBalancerTest.new.perform_async('third string', n)
22
+ FourthJobBalancerTest.new.perform_async('fourth string', n)
16
23
  end
17
- end
18
24
 
19
- it 'fetches queues' do
20
25
  allow(Workerholic::WorkerBalancer.new).to receive(:fetch_queues).and_return(TESTED_QUEUES)
21
26
 
22
27
  balancer = Workerholic::WorkerBalancer.new(workers: [])
23
28
 
24
29
  expect(balancer.queues.map(&:name)).to match_array(TESTED_QUEUES)
25
30
  end
31
+
32
+ context 'auto-balancing workers distribution' do
33
+ it 'does not throw an error when there are no queues' do
34
+ wb = Workerholic::WorkerBalancer.new(workers: workers, auto: true)
35
+
36
+ wb.auto_balanced_workers_distribution
37
+
38
+ expect(wb.current_workers_count_per_queue).to eq({})
39
+ end
40
+
41
+ it 'correctly distributes workers for 1 queue' do
42
+ 100.times do |n|
43
+ FirstJobBalancerTest.new.perform_async('first string', n)
44
+ end
45
+ wb = Workerholic::WorkerBalancer.new(workers: workers, auto: true)
46
+
47
+ wb.auto_balanced_workers_distribution
48
+
49
+ expect(wb.current_workers_count_per_queue).to eq({ TESTED_QUEUES[0] => 25 })
50
+ end
51
+
52
+ it 'correctly distributes workers for 2 queues with equivalent loads' do
53
+ 100.times do |n|
54
+ FirstJobBalancerTest.new.perform_async('first string', n)
55
+ SecondJobBalancerTest.new.perform_async('second string', n)
56
+ end
57
+ wb = Workerholic::WorkerBalancer.new(workers: workers, auto: true)
58
+
59
+ wb.auto_balanced_workers_distribution
60
+
61
+ expect(wb.current_workers_count_per_queue).to eq(
62
+ {
63
+ TESTED_QUEUES[0] => 12,
64
+ TESTED_QUEUES[1] => 13
65
+ }
66
+ )
67
+ end
68
+
69
+ it 'correctly distributes workers for 2 queues with uneven loads' do
70
+ 100.times do |n|
71
+ FirstJobBalancerTest.new.perform_async('first string', n)
72
+ end
73
+
74
+ 200.times do |n|
75
+ SecondJobBalancerTest.new.perform_async('second string', n)
76
+ end
77
+
78
+ wb = Workerholic::WorkerBalancer.new(workers: workers, auto: true)
79
+
80
+ wb.auto_balanced_workers_distribution
81
+
82
+ expect(wb.current_workers_count_per_queue).to eq(
83
+ {
84
+ TESTED_QUEUES[0] => 9,
85
+ TESTED_QUEUES[1] => 16
86
+ }
87
+ )
88
+ end
89
+
90
+ it 'correctly distributes workers for 3 queues with uneven loads' do
91
+ 95.times do |n|
92
+ FirstJobBalancerTest.new.perform_async('first string', n)
93
+ SecondJobBalancerTest.new.perform_async('second string', n)
94
+ end
95
+
96
+ 110.times do |n|
97
+ ThirdJobBalancerTest.new.perform_async('third string', n)
98
+ end
99
+
100
+ wb = Workerholic::WorkerBalancer.new(workers: workers, auto: true)
101
+
102
+ wb.auto_balanced_workers_distribution
103
+
104
+ expect(wb.current_workers_count_per_queue).to eq(
105
+ {
106
+ TESTED_QUEUES[0] => 8,
107
+ TESTED_QUEUES[1] => 8,
108
+ TESTED_QUEUES[2] => 9
109
+ }
110
+ )
111
+ end
112
+
113
+ it 'correctly distributes workers for 4 queues with uneven loads' do
114
+ 100.times do |n|
115
+ FirstJobBalancerTest.new.perform_async('first string', n)
116
+ SecondJobBalancerTest.new.perform_async('second string', n)
117
+ end
118
+
119
+ 200.times do |n|
120
+ ThirdJobBalancerTest.new.perform_async('third string', n)
121
+ FourthJobBalancerTest.new.perform_async('fourth string', n)
122
+ end
123
+ FourthJobBalancerTest.new.perform_async('fourth string', 201)
124
+
125
+ wb = Workerholic::WorkerBalancer.new(workers: workers, auto: true)
126
+
127
+ wb.auto_balanced_workers_distribution
128
+
129
+ expect(wb.current_workers_count_per_queue).to eq(
130
+ {
131
+ TESTED_QUEUES[0] => 4,
132
+ TESTED_QUEUES[1] => 4,
133
+ TESTED_QUEUES[2] => 8,
134
+ TESTED_QUEUES[3] => 9
135
+ }
136
+ )
137
+ end
138
+ end
139
+
140
+ context 'evenly-balancing workers distribution' do
141
+ it 'does not throw an error when there are no queues' do
142
+ wb = Workerholic::WorkerBalancer.new(workers: workers, auto: true)
143
+
144
+ wb.evenly_balanced_workers_distribution
145
+
146
+ expect(wb.current_workers_count_per_queue).to eq({})
147
+ end
148
+
149
+ it 'correctly distributes workers for 1 queue' do
150
+ 100.times do |n|
151
+ FirstJobBalancerTest.new.perform_async('first string', n)
152
+ end
153
+ wb = Workerholic::WorkerBalancer.new(workers: workers, auto: true)
154
+
155
+ wb.evenly_balanced_workers_distribution
156
+
157
+ expect(wb.current_workers_count_per_queue).to eq({ TESTED_QUEUES[0] => 25 })
158
+ end
159
+
160
+ it 'correctly distributes workers for 2 queues with equivalent loads' do
161
+ 100.times do |n|
162
+ FirstJobBalancerTest.new.perform_async('first string', n)
163
+ SecondJobBalancerTest.new.perform_async('second string', n)
164
+ end
165
+ wb = Workerholic::WorkerBalancer.new(workers: workers, auto: true)
166
+
167
+ wb.evenly_balanced_workers_distribution
168
+
169
+ expect(wb.current_workers_count_per_queue).to eq(
170
+ {
171
+ TESTED_QUEUES[0] => 12,
172
+ TESTED_QUEUES[1] => 13
173
+ }
174
+ )
175
+ end
176
+
177
+ it 'correctly distributes workers for 2 queues with uneven loads' do
178
+ 100.times do |n|
179
+ FirstJobBalancerTest.new.perform_async('first string', n)
180
+ end
181
+
182
+ 200.times do |n|
183
+ SecondJobBalancerTest.new.perform_async('second string', n)
184
+ end
185
+
186
+ wb = Workerholic::WorkerBalancer.new(workers: workers, auto: true)
187
+
188
+ wb.evenly_balanced_workers_distribution
189
+
190
+ expect(wb.current_workers_count_per_queue).to eq(
191
+ {
192
+ TESTED_QUEUES[0] => 12,
193
+ TESTED_QUEUES[1] => 13
194
+ }
195
+ )
196
+ end
197
+
198
+ it 'correctly distributes workers for 3 queues with uneven loads' do
199
+ 95.times do |n|
200
+ FirstJobBalancerTest.new.perform_async('first string', n)
201
+ SecondJobBalancerTest.new.perform_async('second string', n)
202
+ end
203
+
204
+ 110.times do |n|
205
+ ThirdJobBalancerTest.new.perform_async('third string', n)
206
+ end
207
+
208
+ wb = Workerholic::WorkerBalancer.new(workers: workers, auto: true)
209
+
210
+ wb.evenly_balanced_workers_distribution
211
+
212
+ expect(wb.current_workers_count_per_queue).to eq(
213
+ {
214
+ TESTED_QUEUES[0] => 8,
215
+ TESTED_QUEUES[1] => 8,
216
+ TESTED_QUEUES[2] => 9
217
+ }
218
+ )
219
+ end
220
+
221
+ it 'correctly distributes workers for 4 queues with uneven loads' do
222
+ 100.times do |n|
223
+ FirstJobBalancerTest.new.perform_async('first string', n)
224
+ SecondJobBalancerTest.new.perform_async('second string', n)
225
+ end
226
+
227
+ 200.times do |n|
228
+ ThirdJobBalancerTest.new.perform_async('third string', n)
229
+ FourthJobBalancerTest.new.perform_async('fourth string', n)
230
+ end
231
+ FourthJobBalancerTest.new.perform_async('fourth string', 201)
232
+
233
+ wb = Workerholic::WorkerBalancer.new(workers: workers, auto: true)
234
+
235
+ wb.evenly_balanced_workers_distribution
236
+
237
+ expect(wb.current_workers_count_per_queue).to eq(
238
+ {
239
+ TESTED_QUEUES[0] => 6,
240
+ TESTED_QUEUES[1] => 6,
241
+ TESTED_QUEUES[2] => 6,
242
+ TESTED_QUEUES[3] => 7
243
+ }
244
+ )
245
+ end
246
+ end
26
247
  end
248
+
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: workerholic
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.23
4
+ version: 0.1.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Antoine Leclercq
@@ -10,7 +10,7 @@ authors:
10
10
  autorequire:
11
11
  bindir: bin
12
12
  cert_chain: []
13
- date: 2017-08-11 00:00:00.000000000 Z
13
+ date: 2017-08-22 00:00:00.000000000 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
16
  name: redis
@@ -139,6 +139,7 @@ files:
139
139
  - lib/workerholic/storage.rb
140
140
  - lib/workerholic/version.rb
141
141
  - lib/workerholic/web/application.rb
142
+ - lib/workerholic/web/public/images/workerholic_logo.png
142
143
  - lib/workerholic/web/public/javascripts/application.js
143
144
  - lib/workerholic/web/public/stylesheets/application.css
144
145
  - lib/workerholic/web/public/stylesheets/whitespace-reset.css
@@ -152,7 +153,6 @@ files:
152
153
  - lib/workerholic/web/views/workers.erb
153
154
  - lib/workerholic/worker.rb
154
155
  - lib/workerholic/worker_balancer.rb
155
- - logos.png
156
156
  - spec/helpers/helper_methods.rb
157
157
  - spec/helpers/job_tests.rb
158
158
  - spec/integration/dequeuing_and_job_processing_spec.rb
data/logos.png DELETED
Binary file