workerholic 0.0.2 → 0.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 61f1243274828aafe8b8c00d714b00e0832e1998
4
- data.tar.gz: 7d1d1d50f9fa005e596b2500482831c749b9d8bd
3
+ metadata.gz: 3dbbd1a9c47965c3c80a7325581badfc7dc96b96
4
+ data.tar.gz: 37df7707f5d84b23bc934bd0bc16885d8dfa8d99
5
5
  SHA512:
6
- metadata.gz: 6d758f61a7cb894d6b514643e3c896cf2ea42adeeda3d66c79c99ded136b8cd9369faa89027f99695b6ebb44904f2357496ad401b1233c624fd8e2e1ebf94956
7
- data.tar.gz: c9ee873f1ed1f1a402947f777b21cd706bd72418a9d40b98566e57ebd833a7e76a3abd41b0bfc6ee0b4d9af5af4e670485508f3995c2a898fc619685dccc6bd7
6
+ metadata.gz: 28a2e6a4335623487fa2b3913022c0a9675de2220fd16bd55490273876cfbaff8f1d8bd007ea354cac0b976245704a2a30888c7704b84a2102db761e07a1c964
7
+ data.tar.gz: 417af745f8894b013567f19b635441cf23eef8050b46f91c1bf2e366a4e006b726fb91c0206cd45a20d28e745871eae560b04d9ac0b591af597723d0ab9d87d2
data/Gemfile CHANGED
@@ -1,7 +1,10 @@
1
1
  source 'https://rubygems.org'
2
2
 
3
+ gemspec
4
+
3
5
  gem 'redis', '~> 3.3', '>= 3.3.3'
4
6
  gem 'connection_pool', '~> 2.2', '>= 2.2.0'
7
+ gem 'sinatra'
5
8
 
6
9
  group :development, :test do
7
10
  gem 'pry-byebug'
data/Gemfile.lock CHANGED
@@ -1,3 +1,11 @@
1
+ PATH
2
+ remote: .
3
+ specs:
4
+ workerholic (0.0.2)
5
+ connection_pool (~> 2.2, >= 2.2.0)
6
+ redis (~> 3.3, >= 3.3.3)
7
+ sinatra
8
+
1
9
  GEM
2
10
  remote: https://rubygems.org/
3
11
  specs:
@@ -6,6 +14,7 @@ GEM
6
14
  connection_pool (2.2.1)
7
15
  diff-lcs (1.3)
8
16
  method_source (0.8.2)
17
+ mustermann (1.0.0)
9
18
  pry (0.10.4)
10
19
  coderay (~> 1.1.0)
11
20
  method_source (~> 0.8.1)
@@ -13,6 +22,9 @@ GEM
13
22
  pry-byebug (3.4.2)
14
23
  byebug (~> 9.0)
15
24
  pry (~> 0.10)
25
+ rack (2.0.3)
26
+ rack-protection (2.0.0)
27
+ rack
16
28
  redis (3.3.3)
17
29
  rspec (3.6.0)
18
30
  rspec-core (~> 3.6.0)
@@ -27,7 +39,13 @@ GEM
27
39
  diff-lcs (>= 1.2.0, < 2.0)
28
40
  rspec-support (~> 3.6.0)
29
41
  rspec-support (3.6.0)
42
+ sinatra (2.0.0)
43
+ mustermann (~> 1.0)
44
+ rack (~> 2.0)
45
+ rack-protection (= 2.0.0)
46
+ tilt (~> 2.0)
30
47
  slop (3.6.0)
48
+ tilt (2.0.7)
31
49
 
32
50
  PLATFORMS
33
51
  ruby
@@ -36,7 +54,9 @@ DEPENDENCIES
36
54
  connection_pool (~> 2.2, >= 2.2.0)
37
55
  pry-byebug
38
56
  redis (~> 3.3, >= 3.3.3)
39
- rspec
57
+ rspec (~> 3.6, >= 3.6.0)
58
+ sinatra
59
+ workerholic!
40
60
 
41
61
  BUNDLED WITH
42
- 1.15.1
62
+ 1.15.3
data/Rakefile ADDED
@@ -0,0 +1 @@
1
+ require 'bundler/gem_tasks'
data/app_test/job_test.rb CHANGED
@@ -1,4 +1,4 @@
1
- require_relative '../lib/workerholic'
1
+ require 'workerholic'
2
2
 
3
3
  class JobTestFast
4
4
  include Workerholic::Job
@@ -14,7 +14,71 @@ class JobTestSlow
14
14
  job_options queue_name: 'workerholic:queue:job_slow'
15
15
 
16
16
  def perform(str, num)
17
- str
18
- sleep(0.1)
17
+ sleep(0.5)
18
+ puts "#{num} - #{str}"
19
+ end
20
+ end
21
+
22
+ class ThreadKiller
23
+ include Workerholic::Job
24
+
25
+ def perform(string, n)
26
+ puts "#{n}. #{string}"
27
+ Thread.main.kill
28
+ end
29
+ end
30
+
31
+ class LargeArg
32
+ include Workerholic::Job
33
+
34
+ def perform(arr, n)
35
+ puts n
36
+ end
37
+ end
38
+
39
+ class ManyArgs
40
+ include Workerholic::Job
41
+
42
+ def perform(n, *args)
43
+ puts "#{n}: #{args}"
44
+ end
45
+ end
46
+
47
+ # A CPU-blocking operation
48
+ class HeavyCalculation
49
+ include Workerholic::Job
50
+
51
+ def perform(n, arr)
52
+ arr = bubble_sort(arr)
53
+ puts "#{n}: #{arr[0..9]}"
54
+ end
55
+
56
+ def bubble_sort(array)
57
+ return array if array.size <= 1
58
+
59
+ unsorted = true
60
+
61
+ while unsorted do
62
+ unsorted = false
63
+ 0.upto(array.size-2) do |i|
64
+ if array[i] > array[i+1]
65
+ array[i], array[i+1] = array[i+1], array[i]
66
+ unsorted = true
67
+ end
68
+ end
69
+ end
70
+
71
+ array
72
+ end
73
+ end
74
+
75
+ class GetPrimes
76
+ include Workerholic::Job
77
+
78
+ def perform(n, max)
79
+ Prime.each(max) do |prime|
80
+ prime
81
+ end
82
+ puts n
19
83
  end
20
84
  end
data/app_test/run.rb CHANGED
@@ -1,5 +1,30 @@
1
1
  require_relative 'job_test'
2
2
 
3
+ # 100000.times do |n|
4
+ # JobTestFast.new.perform_async('NONBLOCKING', n)
5
+ # # sleep(0.0015)
6
+ # end
7
+
8
+ # 10000.times do |n|
9
+ # JobTestSlow.new.perform_async('BLOCKING', n)
10
+ # sleep(0.0015)
11
+ # end
12
+
13
+ # 100000.times do |n|
14
+ # ThreadKiller.new.perform_async('Kill', n)
15
+ # end
16
+
17
+ # arg = Array.new(10000, 'string')
18
+ # 10000.times do |n|
19
+ # LargeArg.new.perform_async(arg, n)
20
+ # end
21
+
22
+ # unsorted_array = (1..10000).to_a.shuffle
23
+ unsorted_array = (1..1000).to_a.shuffle
24
+ 1000.times do |n|
25
+ HeavyCalculation.new.perform_async(n, unsorted_array)
26
+ end
27
+
3
28
  5_000.times do |n|
4
29
  JobTestFast.new.perform_async('NON BLOCKING', n)
5
30
  JobTestFast.new.perform_async('NON BLOCKING', n)
@@ -8,3 +33,11 @@ require_relative 'job_test'
8
33
  # # JobTestSlow.new.perform_async('BLOCKING', n)
9
34
  JobTestSlow.new.perform_async('BLOCKING', n)
10
35
  end
36
+
37
+ # 100000.times do |n|
38
+ # ManyArgs.new.perform_async(n, [1, 2, 3], { key: 'value'}, :symb, 'string', 22, false)
39
+ # end
40
+
41
+ # 100000.times do |n|
42
+ # GetPrimes.new.perform_async(n, 1000000)
43
+ # end
data/bin/workerholic ADDED
@@ -0,0 +1,5 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require_relative '../lib/workerholic/cli'
4
+
5
+ Workerholic::CLI.run
@@ -0,0 +1,21 @@
1
+ $LOAD_PATH << __dir__ + '/..'
2
+
3
+ require 'workerholic'
4
+
5
+ module Workerholic
6
+ class CLI
7
+ def self.run
8
+ auto_balance = ARGV.any? { |arg| arg == '--auto-balance' }
9
+ workers_count = ARGV.find { |arg| arg.match? /^--workers=\d+$/ }
10
+
11
+ if workers_count
12
+ workers_count = workers_count[/\d+/].to_i
13
+ Workerholic.workers_count = workers_count
14
+ end
15
+
16
+ Manager.new.start
17
+ end
18
+ end
19
+ end
20
+
21
+
@@ -7,6 +7,8 @@ module Workerholic
7
7
  end
8
8
 
9
9
  def log(severity, message)
10
+ return if $TESTING
11
+
10
12
  logger.formatter = proc do |severity, datetime, progname, msg|
11
13
  "#{severity}: #{msg}\n"
12
14
  end
@@ -1,7 +1,7 @@
1
1
  module Workerholic
2
2
  # Handles background job enqueueing/dequeuing functionality
3
3
  class Queue
4
- attr_reader :storage, :name
4
+ attr_reader :storage, :name, :logger
5
5
 
6
6
  def initialize(name = 'workerholic:queue:main')
7
7
  @storage = Storage::RedisWrapper.new
@@ -11,7 +11,7 @@ module Workerholic
11
11
 
12
12
  def enqueue(serialized_job)
13
13
  storage.push(name, serialized_job)
14
- @logger.log('info', "Your job was placed in the #{name} queue on #{Time.now}.")
14
+ logger.log('info', "Your job was placed in the #{name} queue on #{Time.now}.")
15
15
  end
16
16
 
17
17
  def dequeue
@@ -20,7 +20,7 @@ module Workerholic
20
20
  end
21
21
 
22
22
  def empty?
23
- storage.list_length(name).zero?
23
+ storage.list_length(name) == 0
24
24
  end
25
25
 
26
26
  def size
@@ -20,7 +20,7 @@ module Workerholic
20
20
  end
21
21
 
22
22
  def empty?
23
- storage.set_empty?(name) == 0
23
+ storage.sorted_set_size(name) == 0
24
24
  end
25
25
  end
26
26
  end
@@ -13,55 +13,56 @@ module Workerholic
13
13
  end
14
14
  end
15
15
 
16
- def list_length(key)
17
- execute { |conn| conn.llen(key) }
16
+ def list_length(key, retry_delay = 5)
17
+ execute(retry_delay) { |conn| conn.llen(key) }
18
18
  end
19
19
 
20
- def push(key, value)
21
- execute { |conn| conn.rpush(key, value) }
20
+ def push(key, value, retry_delay = 5)
21
+ execute(retry_delay) { |conn| conn.rpush(key, value) }
22
22
  end
23
23
 
24
24
  # blocking pop from Redis queue
25
- def pop(key, timeout = 1)
26
- execute { |conn| conn.blpop(key, timeout) }
25
+ def pop(key, timeout = 1, retry_delay = 5)
26
+ execute(retry_delay) { |conn| conn.blpop(key, timeout) }
27
27
  end
28
28
 
29
- def add_to_set(key, score, value)
30
- execute { |conn| conn.zadd(key, score, value) }
29
+ def add_to_set(key, score, value, retry_delay = 5)
30
+ execute(retry_delay) { |conn| conn.zadd(key, score, value) }
31
31
  end
32
32
 
33
- def peek(key)
34
- execute { |conn| conn.zrange(key, 0, 0, with_scores: true).first }
33
+ def peek(key, retry_delay = 5)
34
+ execute(retry_delay) { |conn| conn.zrange(key, 0, 0, with_scores: true).first }
35
35
  end
36
36
 
37
- def remove_from_set(key, score)
38
- execute { |conn| conn.zremrangebyscore(key, score, score) }
37
+ def remove_from_set(key, score, retry_delay = 5)
38
+ execute(retry_delay) { |conn| conn.zremrangebyscore(key, score, score) }
39
39
  end
40
40
 
41
- def set_empty?(key)
42
- execute { |conn| conn.zcount(key, 0, '+inf') }
41
+ def sorted_set_size(key, retry_delay = 5)
42
+ execute(retry_delay) { |conn| conn.zcount(key, 0, '+inf') }
43
43
  end
44
44
 
45
- def fetch_queue_names
46
- execute { |conn| conn.scan(0, match: 'workerholic:queue*').last }
45
+ def fetch_queue_names(retry_delay = 5)
46
+ queue_name_pattern = $TESTING ? 'workerholic:testing:queue*' : 'workerholic:queue*'
47
+
48
+ execute(retry_delay) { |conn| conn.scan(0, match: queue_name_pattern).last }
47
49
  end
48
50
 
49
51
  class RedisCannotRecover < Redis::CannotConnectError; end
50
52
 
51
53
  private
52
54
 
53
- def execute
55
+ def execute(retry_delay = 5)
54
56
  begin
55
57
  result = redis.with { |conn| yield conn }
56
58
  reset_retries
57
59
  rescue Redis::CannotConnectError
58
- # LogManager might want to output our retries to the user
59
60
  @retries += 1
60
61
  if retries_exhausted?
61
62
  raise RedisCannotRecover, 'Redis reconnect retries exhausted. Main Workerholic thread will be terminated now.'
62
63
  end
63
64
 
64
- sleep(5)
65
+ sleep retry_delay
65
66
  retry
66
67
  end
67
68
 
@@ -0,0 +1,3 @@
1
+ module Workerholic
2
+ VERSION = '0.0.3'
3
+ end
data/lib/workerholic.rb CHANGED
@@ -1,5 +1,3 @@
1
- $LOAD_PATH << __dir__
2
-
3
1
  require 'yaml'
4
2
  require 'redis'
5
3
  require 'connection_pool'
Binary file
@@ -1,5 +1,8 @@
1
- TEST_QUEUE = 'workerholic:queue:_test_queue'
2
- TEST_SCHEDULED_SORTED_SET = 'workerholic:test:scheduled_jobs'
1
+ TEST_QUEUE = 'workerholic:testing:queue:test_queue'
2
+ ANOTHER_TEST_QUEUE = 'workerholic:testing:queue:another_test_queue'
3
+ BALANCER_TEST_QUEUE = 'workerholic:testing:queue:balancer_test_queue'
4
+ ANOTHER_BALANCER_TEST_QUEUE = 'workerholic:testing:queue:another_balancer_test_queue'
5
+ TEST_SCHEDULED_SORTED_SET = 'workerholic:testing:scheduled_jobs'
3
6
 
4
7
  def expect_during(duration_in_secs, target)
5
8
  timeout = Time.now.to_f + duration_in_secs
@@ -15,3 +15,22 @@ class ComplexJobTest
15
15
  [arg1, arg2, arg3]
16
16
  end
17
17
  end
18
+
19
+ class FirstJobBalancerTest
20
+ include Workerholic::Job
21
+ job_options queue_name: BALANCER_TEST_QUEUE
22
+
23
+ def perform(str, num)
24
+ str
25
+ end
26
+ end
27
+
28
+ class SecondJobBalancerTest
29
+ include Workerholic::Job
30
+ job_options queue_name: ANOTHER_BALANCER_TEST_QUEUE
31
+
32
+ def perform(str, num)
33
+ str
34
+ end
35
+ end
36
+
@@ -2,7 +2,6 @@ require_relative '../spec_helper'
2
2
 
3
3
  describe 'dequeuing and processesing of jobs' do
4
4
  let(:redis) { Redis.new }
5
- before { redis.del(TEST_QUEUE) }
6
5
 
7
6
  xit 'successfully dequeues and process a simple job' do
8
7
  serialized_job = Workerholic::JobSerializer.serialize(
@@ -13,7 +12,10 @@ describe 'dequeuing and processesing of jobs' do
13
12
  manager = Workerholic::Manager.new
14
13
 
15
14
  Thread.new { manager.start }
16
- expect_during(1, false) { redis.exists(TEST_QUEUE) }
15
+ sleep(0.05)
16
+
17
+ expect(redis.llen(TEST_QUEUE)).to eq(0)
18
+ Thread.list.reject { |t| t == Thread.main }.each(&:kill)
17
19
  end
18
20
 
19
21
  it 'successfully dequeues and process a complex job'
@@ -2,7 +2,6 @@ require_relative '../spec_helper'
2
2
 
3
3
  describe 'enqueuing jobs to Redis' do
4
4
  let(:redis) { Redis.new }
5
- before { redis.del(TEST_QUEUE) }
6
5
 
7
6
  context 'successfully creates a job and enqueues it in Redis' do
8
7
  it 'enqueues a simple job in redis' do
@@ -33,17 +33,17 @@ describe Workerholic::JobProcessor do
33
33
  expect(job_processor.process).to eq(complex_job_result)
34
34
  end
35
35
 
36
- # it 'raises a custom error when processing a job with error' do
37
- # serialized_job = Workerholic::JobSerializer.serialize({
38
- # class: SimpleJobTestWithError,
39
- # arguments: [],
40
- # statistics: Workerholic::Statistics.new.to_hash
41
- # })
36
+ it 'does not raise an error when processing a job with error' do
37
+ serialized_job = Workerholic::JobSerializer.serialize({
38
+ class: SimpleJobTestWithError,
39
+ arguments: [],
40
+ statistics: Workerholic::Statistics.new.to_hash
41
+ })
42
42
 
43
- # job_processor = Workerholic::JobProcessor.new(serialized_job)
43
+ job_processor = Workerholic::JobProcessor.new(serialized_job)
44
44
 
45
- # expect { job_processor.process }.to raise_error(Workerholic::JobProcessingError)
46
- # end
45
+ expect { job_processor.process }.not_to raise_error
46
+ end
47
47
 
48
48
  it 'retries job when job processing fails' do
49
49
  job = {
@@ -54,7 +54,6 @@ describe Workerholic::JobProcessor do
54
54
  serialized_job = Workerholic::JobSerializer.serialize(job)
55
55
 
56
56
  allow(Workerholic::JobRetry).to receive(:new)
57
-
58
57
  expect(Workerholic::JobRetry).to receive(:new)
59
58
 
60
59
  Workerholic::JobProcessor.new(serialized_job).process
@@ -9,8 +9,6 @@ end
9
9
  describe Workerholic::JobRetry do
10
10
  let(:redis) { Redis.new }
11
11
 
12
- before { redis.del(TEST_SCHEDULED_SORTED_SET) }
13
-
14
12
  it 'increments retry count' do
15
13
  job = Workerholic::JobWrapper.new(class: JobWithError, arguments: [])
16
14
 
@@ -13,8 +13,6 @@ describe Workerholic::JobScheduler do
13
13
  let(:scheduler) { Workerholic::JobScheduler.new(set_name: TEST_SCHEDULED_SORTED_SET, queue_name: TEST_QUEUE) }
14
14
  let(:redis) { Redis.new }
15
15
 
16
- before { redis.del(TEST_SCHEDULED_SORTED_SET) }
17
-
18
16
  context 'with non-empty set' do
19
17
  let(:serialized_job) do
20
18
  job = Workerholic::JobWrapper.new(
data/spec/manager_spec.rb CHANGED
@@ -5,7 +5,7 @@ describe Workerholic::Manager do
5
5
  manager = Workerholic::Manager.new
6
6
 
7
7
  manager.workers.each { |worker| expect(worker).to be_a(Workerholic::Worker) }
8
- expect(manager.workers.size).to eq(Workerholic::Manager::WORKERS_COUNT)
8
+ expect(manager.workers.size).to eq(Workerholic.workers_count)
9
9
  end
10
10
 
11
11
  it 'creates a job scheduler' do
@@ -17,10 +17,12 @@ describe Workerholic::Manager do
17
17
  it 'starts up the workers and the scheduler' do
18
18
  manager = Workerholic::Manager.new
19
19
 
20
+ allow_any_instance_of(Workerholic::Worker).to receive(:work) { nil }
21
+
20
22
  expect(manager.workers.first).to receive(:work)
21
23
  expect(manager.scheduler).to receive(:start)
22
- Thread.new { manager.start }
23
24
 
24
- sleep(0.1)
25
+ t = Thread.new { manager.start }
26
+ sleep(0.01)
25
27
  end
26
28
  end
data/spec/queue_spec.rb CHANGED
@@ -1,7 +1,8 @@
1
1
  require_relative 'spec_helper'
2
2
 
3
3
  describe Workerholic::Queue do
4
- let(:queue) { Workerholic::Queue.new('test') }
4
+ let(:redis) { Redis.new }
5
+ let(:queue) { Workerholic::Queue.new(TEST_QUEUE) }
5
6
  let(:job) { 'test job' }
6
7
 
7
8
  it 'enqueues a job' do
@@ -13,4 +14,16 @@ describe Workerholic::Queue do
13
14
  expect(queue.storage).to receive(:pop).with(queue.name).and_return([queue.name,job])
14
15
  queue.dequeue
15
16
  end
17
+
18
+ it 'checks if queue is empty' do
19
+ expect(queue.empty?).to eq(true)
20
+ end
21
+
22
+ it 'returns size of queue' do
23
+ redis.rpush(TEST_QUEUE, job)
24
+ redis.rpush(TEST_QUEUE, job)
25
+ redis.rpush(TEST_QUEUE, job)
26
+
27
+ expect(queue.size).to eq(3)
28
+ end
16
29
  end
@@ -5,21 +5,34 @@ describe Workerholic::SortedSet do
5
5
  let(:redis) { Redis.new }
6
6
  let(:sorted_set) { Workerholic::SortedSet.new(TEST_SCHEDULED_SORTED_SET) }
7
7
 
8
- after { redis.del(TEST_SCHEDULED_SORTED_SET) }
9
-
10
8
  it 'adds a serialized job to the sorted set' do
11
9
  serialized_job = Workerholic::JobSerializer.serialize(job)
12
10
  score = Time.now.to_f
13
- expect(sorted_set.add(score, serialized_job)).to eq(true)
11
+
12
+ expect(sorted_set.storage).to receive(:add_to_set).and_return(true)
13
+
14
+ sorted_set.add(serialized_job, score)
14
15
  end
15
16
 
16
17
  it 'removes due job from the sorted set' do
17
18
  serialized_job = Workerholic::JobSerializer.serialize(job)
18
19
  score = Time.now.to_f
20
+ redis.zadd(TEST_SCHEDULED_SORTED_SET, score, serialized_job)
19
21
 
20
- sorted_set.add(score, serialized_job)
21
22
  sorted_set.remove(score)
22
23
 
23
24
  expect(redis.zcount(TEST_SCHEDULED_SORTED_SET, 0, '+inf')).to eq(0)
24
25
  end
26
+
27
+ it 'returns first element from sorted set' do
28
+ expect(sorted_set.storage).to receive(:peek)
29
+
30
+ sorted_set.peek
31
+ end
32
+
33
+ it 'checks if set is empty' do
34
+ expect(sorted_set.storage).to receive(:sorted_set_size)
35
+
36
+ sorted_set.empty?
37
+ end
25
38
  end
data/spec/spec_helper.rb CHANGED
@@ -1,5 +1,7 @@
1
1
  $LOAD_PATH << __dir__ + '/../lib/'
2
2
 
3
+ $TESTING = true
4
+
3
5
  require 'workerholic'
4
6
 
5
7
  require_relative 'helpers/helper_methods'
@@ -15,4 +17,8 @@ RSpec.configure do |config|
15
17
  end
16
18
 
17
19
  config.shared_context_metadata_behavior = :apply_to_host_groups
20
+
21
+ config.before do
22
+ Redis.new.del(TEST_QUEUE, ANOTHER_TEST_QUEUE, BALANCER_TEST_QUEUE, ANOTHER_BALANCER_TEST_QUEUE, TEST_SCHEDULED_SORTED_SET)
23
+ end
18
24
  end
data/spec/storage_spec.rb CHANGED
@@ -2,16 +2,95 @@ require_relative 'spec_helper'
2
2
 
3
3
  describe Workerholic::Storage do
4
4
  let(:storage) { Workerholic::Storage::RedisWrapper.new }
5
+ let(:redis) { Redis.new }
5
6
  let(:queue_name) { TEST_QUEUE }
6
7
  let(:job) { 'test job' }
7
8
 
8
- it 'adds a job to the test queue' do
9
- expect(storage.redis).to receive(:rpush).with(queue_name, job)
10
- storage.push(queue_name, job)
9
+ context 'with Redis running' do
10
+ it 'adds a job to the test queue' do
11
+ storage.push(queue_name, job)
12
+
13
+ expect(redis.llen(queue_name)).to eq(1)
14
+ end
15
+
16
+ it 'pops a job from the test queue' do
17
+ storage.push(queue_name, job)
18
+ storage.pop(queue_name)
19
+
20
+ expect(storage.list_length(queue_name)).to eq(0)
21
+ end
22
+
23
+ it 'gets the size for a specific queue' do
24
+ storage.push(queue_name, job)
25
+ storage.push(queue_name, job)
26
+
27
+ expect(storage.list_length(queue_name)).to eq(2)
28
+ end
29
+
30
+ it 'adds job to sorted set' do
31
+ score = Time.now.to_f
32
+ storage.add_to_set(TEST_SCHEDULED_SORTED_SET, score, job)
33
+
34
+ expect(redis.zrange(TEST_SCHEDULED_SORTED_SET, 0, 0, with_scores: true).first).to eq([job, score])
35
+ end
36
+
37
+ it 'returns first element in sorted set' do
38
+ score = Time.now.to_f
39
+ redis.zadd(TEST_SCHEDULED_SORTED_SET, score, job)
40
+
41
+ expect(storage.peek(TEST_SCHEDULED_SORTED_SET)).to eq([job, score])
42
+ end
43
+
44
+ it 'removes specified element from set' do
45
+ score1 = Time.now.to_f
46
+ score2 = score1 + 10
47
+ job2 = 'second test job'
48
+ redis.zadd(TEST_SCHEDULED_SORTED_SET, score1, job)
49
+ redis.zadd(TEST_SCHEDULED_SORTED_SET, score2, job2)
50
+
51
+ expect(storage.remove_from_set(TEST_SCHEDULED_SORTED_SET, score1)).to eq(1)
52
+ expect(redis.zcount(TEST_SCHEDULED_SORTED_SET, 0, '+inf')).to eq(1)
53
+ end
54
+
55
+ it 'checks if the sorted set is empty' do
56
+ score1 = Time.now.to_f
57
+ score2 = score1 + 10
58
+ job2 = 'second test job'
59
+ redis.zadd(TEST_SCHEDULED_SORTED_SET, score1, job)
60
+ redis.zadd(TEST_SCHEDULED_SORTED_SET, score2, job2)
61
+
62
+ expect(storage.sorted_set_size(TEST_SCHEDULED_SORTED_SET)).to eq(2)
63
+ end
64
+
65
+ it 'returns the workerholic queue names that are in redis' do
66
+ storage.push(queue_name, job)
67
+ storage.push(ANOTHER_TEST_QUEUE, job)
68
+
69
+ expect(storage.fetch_queue_names).to match_array([queue_name, ANOTHER_TEST_QUEUE])
70
+ end
11
71
  end
12
72
 
13
- it 'pops a job from the test queue' do
14
- expect(storage.redis).to receive(:blpop).with(queue_name, 1)
15
- storage.pop(queue_name)
73
+ context 'with Redis not running' do
74
+ it 'calls Redis command inside a block wrapper' do
75
+ expect(storage).to receive(:execute)
76
+
77
+ storage.list_length(queue_name)
78
+ end
79
+
80
+ it 'increments the retries variable on inaccessible Redis instance' do
81
+ expect(storage.redis).to receive(:with).at_least(1).and_raise(Redis::CannotConnectError)
82
+
83
+ begin
84
+ storage.push(queue_name, job, 0.01)
85
+ rescue Redis::CannotConnectError
86
+ expect(storage.instance_variable_get(:@retries)).to eq(5)
87
+ end
88
+ end
89
+
90
+ it 'raises error if the number of retries has been exceeded' do
91
+ expect(storage.redis).to receive(:with).at_least(1).and_raise(Redis::CannotConnectError)
92
+
93
+ expect { storage.push(queue_name, job, 0.01) }.to raise_error(Workerholic::Storage::RedisWrapper::RedisCannotRecover)
94
+ end
16
95
  end
17
96
  end
@@ -0,0 +1,23 @@
1
+ require_relative 'spec_helper'
2
+
3
+ TESTED_QUEUES = [BALANCER_TEST_QUEUE, ANOTHER_BALANCER_TEST_QUEUE]
4
+
5
+ describe Workerholic::WorkerBalancer do
6
+ let(:storage) { Workerholic::Storage::RedisWrapper.new }
7
+ let(:redis) { Redis.new }
8
+
9
+ before do
10
+ 100.times do |n|
11
+ FirstJobBalancerTest.new.perform_async('first string', n)
12
+ SecondJobBalancerTest.new.perform_async('second string', n)
13
+ end
14
+ end
15
+
16
+ it 'fetches queues' do
17
+ allow(Workerholic::WorkerBalancer.new).to receive(:fetch_queues).and_return(TESTED_QUEUES)
18
+
19
+ manager = Workerholic::WorkerBalancer.new(workers: [])
20
+
21
+ expect(manager.queues.map(&:name)).to match_array(TESTED_QUEUES)
22
+ end
23
+ end
data/spec/worker_spec.rb CHANGED
@@ -26,10 +26,7 @@ describe Workerholic::Worker do
26
26
  }
27
27
  end
28
28
 
29
- before do
30
- redis.del(TEST_SCHEDULED_SORTED_SET)
31
- WorkerJobTest.reset
32
- end
29
+ before { WorkerJobTest.reset }
33
30
 
34
31
  context '#work' do
35
32
  it 'polls a job from a thread' do
@@ -0,0 +1,30 @@
1
+ require 'sinatra'
2
+ require 'sinatra/reloader'
3
+
4
+ get '/' do
5
+ erb :index
6
+ end
7
+
8
+ get '/details' do
9
+ erb :details
10
+ end
11
+
12
+ get '/queues' do
13
+ #placeholder
14
+ erb :index
15
+ end
16
+
17
+ get '/workers' do
18
+ #placeholder
19
+ erb :workers
20
+ end
21
+
22
+ get '/failed' do
23
+ #placeholder
24
+ erb :index
25
+ end
26
+
27
+ get '/scheduled' do
28
+ #placeholder
29
+ erb :scheduled
30
+ end
@@ -0,0 +1,79 @@
1
+ @import 'whitespace-reset.css';
2
+
3
+ body {
4
+ font-family: Arial, sans-serif;
5
+ background: #FFD252;
6
+ }
7
+
8
+ header {
9
+ font-family: 'Dosis', sans-serif;
10
+ font-size: 128px;
11
+ color: #fff;
12
+ text-align: center;
13
+ background: linear-gradient(to bottom, #3A0E40, #E739FF);
14
+ }
15
+
16
+ footer {
17
+ position: fixed;
18
+ bottom: 0;
19
+ left: 0;
20
+ width: 100%;
21
+ }
22
+
23
+ footer p {
24
+ display: inline-block;
25
+ width: 100%;
26
+ text-align: center;
27
+ }
28
+
29
+ nav {
30
+ background: #fff;
31
+ }
32
+
33
+ nav li {
34
+ display: inline-block;
35
+ font-size: 20px;
36
+ padding: 10px 0 10px 0;
37
+ margin-left: 20px;
38
+ color: #000;
39
+ }
40
+
41
+ nav a {
42
+ display: inline-block;
43
+ width: 100px;
44
+ padding: 20px;
45
+ text-align: center;
46
+ text-decoration: none;
47
+ border-radius: 5px;
48
+ border: 3px solid #E739FF;
49
+ color: #fff;
50
+ background: #A016B2;
51
+ }
52
+
53
+ #statistics {
54
+ font-family: 'Monaco', serif;
55
+ width: 1000px;
56
+ padding: 25px;
57
+ margin: 50px auto;
58
+ border: 2px dotted #000;
59
+ border-radius: 10px;
60
+ background: #fff;
61
+ }
62
+
63
+ #statistics h2 {
64
+ font-size: 32px;
65
+ }
66
+
67
+ #statistics p {
68
+ font-size: 24px;
69
+ margin-top: 20px;
70
+ }
71
+
72
+ #completed {
73
+ color: green;
74
+ }
75
+
76
+ #failed {
77
+ color: red;
78
+ font-weight: bold;
79
+ }
@@ -0,0 +1,22 @@
1
+ /*
2
+ ----------------------------------------
3
+ Tantek Celik's Whitepsace Reset
4
+ Author: Tantek Celik, Shane Riley
5
+ Version: (CC) 2010 Some Rights Reserved - http://creativecommons.org/licenses/by/2.0
6
+ Description: Resets default styling of common browsers to a common base
7
+ ----------------------------------------
8
+ */
9
+
10
+ ul,ol { list-style: none; }
11
+ h1,h2,h3,h4,h5,h6,pre,code { font-size: 1em; }
12
+ ul,ol,li,h1,h2,h3,h4,h5,h6,pre,form,body,html,p,blockquote,fieldset,input,dl,dt,dd, figure, figcaption {
13
+ margin: 0;
14
+ padding: 0; }
15
+ a img,:link img,:visited img, fieldset { border: none; }
16
+ address { font-style: normal; }
17
+ header, section, article, nav, footer, hgroup, details, summary, figure, main { display: block; }
18
+ mark {
19
+ color: inherit;
20
+ background: transparent; }
21
+ abbr { border: none; }
22
+ summary::-webkit-details-marker { display: none; }
@@ -0,0 +1,32 @@
1
+ <html>
2
+ <head>
3
+ <title>Workerholic Overview</title>
4
+ <link href="https://fonts.googleapis.com/css?family=Dosis" rel="stylesheet">
5
+ <link href='stylesheets/application.css' rel='stylesheet'>
6
+ </head>
7
+
8
+ <body>
9
+ <header>
10
+ Workerholic
11
+ </header>
12
+ <nav>
13
+ <ul>
14
+ <li><a href='/'>Overview</a></li>
15
+ <li><a href='/details'>Job Details</a></li>
16
+ <li><a href='/queues'>Queues</a></li>
17
+ <li><a href='/queues'>Workers</a></li>
18
+ <li><a href='/what'>Failed</a></li>
19
+ <li><a href='/what'>Scheduled</a></li>
20
+ </ul>
21
+ </nav>
22
+
23
+ <div id="statistics">
24
+ <h2>Your current active queues:</h2>
25
+ <p>JobTestFast: 1,234</p>
26
+ <p>JobtestSlow: 2,144,822</p>
27
+ <p>GetPrimes: 102,891</p>
28
+ <p>Total: 2,248,947</p>
29
+ </div>
30
+
31
+ </body>
32
+ </html>
@@ -0,0 +1,35 @@
1
+ <html>
2
+ <head>
3
+ <title>Workerholic Overview</title>
4
+ <link href="https://fonts.googleapis.com/css?family=Dosis" rel="stylesheet">
5
+ <link href='stylesheets/application.css' rel='stylesheet'>
6
+ </head>
7
+
8
+ <body>
9
+ <header>
10
+ Workerholic
11
+ </header>
12
+ <nav>
13
+ <ul>
14
+ <li><a href='/'>Overview</a></li>
15
+ <li><a href='/details'>Job Details</a></li>
16
+ <li><a href='/placehold'>Queues</a></li>
17
+ <li><a href='/workers'>Workers</a></li>
18
+ <li><a href='/what'>Failed</a></li>
19
+ <li><a href='/what'>Scheduled</a></li>
20
+ </ul>
21
+ </nav>
22
+
23
+ <div id="statistics">
24
+ <h2>Your current jobs overview</h2>
25
+ <p id="completed">Finished Jobs: 0</p>
26
+ <p>Jobs in queue: 100,000,000</p>
27
+ <p id="failed">Jobs failed: 10,000,000</p>
28
+ </div>
29
+
30
+ <footer>
31
+ <p>&copy; 2017 Workerholic. All rights reserved.</p>
32
+ </footer>
33
+
34
+ </body>
35
+ </html>
@@ -0,0 +1,45 @@
1
+ <html>
2
+ <head>
3
+ <title>Workerholic Overview</title>
4
+ <link href="https://fonts.googleapis.com/css?family=Dosis" rel="stylesheet">
5
+ <link href='stylesheets/application.css' rel='stylesheet'>
6
+ </head>
7
+
8
+ <body>
9
+ <header>
10
+ Workerholic
11
+ </header>
12
+ <nav>
13
+ <ul>
14
+ <li><a href='/'>Overview</a></li>
15
+ <li><a href='/details'>Job Details</a></li>
16
+ <li><a href='/placehold'>Queues</a></li>
17
+ <li><a href='/workers'>Workers</a></li>
18
+ <li><a href='/what'>Failed</a></li>
19
+ <li><a href='/what'>Scheduled</a></li>
20
+ </ul>
21
+ </nav>
22
+
23
+ <div id="workers">
24
+ <table>
25
+ <tr>
26
+ <th>Worker</th>
27
+ <th>Queue</th>
28
+ <th>Class</th>
29
+ <th>Arguments</th>
30
+ </tr>
31
+ <tr>
32
+ <td>Worker 1</td>
33
+ <td>workerholic:queue:main</td>
34
+ <td>GetPrimes</td>
35
+ <td>[n, 1000000]</td>
36
+ </tr>
37
+ </table>
38
+ </div>
39
+
40
+ <footer>
41
+ <p>&copy; 2017 Workerholic. All rights reserved.</p>
42
+ </footer>
43
+
44
+ </body>
45
+ </html>
data/workerholic.gemspec CHANGED
@@ -13,13 +13,14 @@ Gem::Specification.new do |gem|
13
13
 
14
14
  gem.files = `git ls-files`.split("\n")
15
15
  gem.test_files = `git ls-files -- spec/*`.split("\n")
16
- gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
16
+ gem.executables = ['workerholic']
17
17
  gem.require_paths = ['lib']
18
18
 
19
19
  gem.required_ruby_version = '>= 2.2.2'
20
20
 
21
21
  gem.add_dependency 'redis', '~> 3.3', '>= 3.3.3'
22
22
  gem.add_dependency 'connection_pool', '~> 2.2', '>= 2.2.0'
23
+ gem.add_dependency 'sinatra'
23
24
 
24
25
  gem.add_development_dependency 'pry-byebug'
25
26
  gem.add_development_dependency 'rspec', '~> 3.6', '>= 3.6.0'
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: workerholic
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.2
4
+ version: 0.0.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Antoine Leclercq
@@ -10,7 +10,7 @@ authors:
10
10
  autorequire:
11
11
  bindir: bin
12
12
  cert_chain: []
13
- date: 2017-07-26 00:00:00.000000000 Z
13
+ date: 2017-07-27 00:00:00.000000000 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
16
  name: redis
@@ -52,6 +52,20 @@ dependencies:
52
52
  - - ">="
53
53
  - !ruby/object:Gem::Version
54
54
  version: 2.2.0
55
+ - !ruby/object:Gem::Dependency
56
+ name: sinatra
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - ">="
60
+ - !ruby/object:Gem::Version
61
+ version: '0'
62
+ type: :runtime
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - ">="
67
+ - !ruby/object:Gem::Version
68
+ version: '0'
55
69
  - !ruby/object:Gem::Dependency
56
70
  name: pry-byebug
57
71
  requirement: !ruby/object:Gem::Requirement
@@ -91,7 +105,8 @@ email:
91
105
  - antoine.leclercq.49@gmail.com
92
106
  - lmwinr@gmail.com
93
107
  - tim-lee92@outlook.com
94
- executables: []
108
+ executables:
109
+ - workerholic
95
110
  extensions: []
96
111
  extra_rdoc_files: []
97
112
  files:
@@ -101,11 +116,13 @@ files:
101
116
  - Gemfile.lock
102
117
  - LICENSE
103
118
  - README.md
119
+ - Rakefile
104
120
  - app_test/job_test.rb
105
121
  - app_test/run.rb
106
- - lib/server.rb
122
+ - bin/workerholic
107
123
  - lib/workerholic.rb
108
124
  - lib/workerholic/adapters/active_job_adapter.rb
125
+ - lib/workerholic/cli.rb
109
126
  - lib/workerholic/job.rb
110
127
  - lib/workerholic/job_processor.rb
111
128
  - lib/workerholic/job_retry.rb
@@ -118,8 +135,10 @@ files:
118
135
  - lib/workerholic/sorted_set.rb
119
136
  - lib/workerholic/statistics.rb
120
137
  - lib/workerholic/storage.rb
138
+ - lib/workerholic/version.rb
121
139
  - lib/workerholic/worker.rb
122
140
  - lib/workerholic/worker_balancer.rb
141
+ - pkg/workerholic-0.0.2.gem
123
142
  - spec/helpers/helper_methods.rb
124
143
  - spec/helpers/job_tests.rb
125
144
  - spec/integration/dequeuing_and_job_processing_spec.rb
@@ -131,11 +150,18 @@ files:
131
150
  - spec/job_wrapper_spec.rb
132
151
  - spec/manager_spec.rb
133
152
  - spec/queue_spec.rb
134
- - spec/sorted_set.rb
153
+ - spec/sorted_set_spec.rb
135
154
  - spec/spec_helper.rb
136
155
  - spec/statistics_spec.rb
137
156
  - spec/storage_spec.rb
157
+ - spec/worker_balancer_spec.rb
138
158
  - spec/worker_spec.rb
159
+ - web/application.rb
160
+ - web/public/stylesheets/application.css
161
+ - web/public/stylesheets/whitespace-reset.css
162
+ - web/views/details.erb
163
+ - web/views/index.erb
164
+ - web/views/workers.erb
139
165
  - workerholic.gemspec
140
166
  homepage: https://github.com/workerholic/workerholic
141
167
  licenses:
@@ -173,8 +199,9 @@ test_files:
173
199
  - spec/job_wrapper_spec.rb
174
200
  - spec/manager_spec.rb
175
201
  - spec/queue_spec.rb
176
- - spec/sorted_set.rb
202
+ - spec/sorted_set_spec.rb
177
203
  - spec/spec_helper.rb
178
204
  - spec/statistics_spec.rb
179
205
  - spec/storage_spec.rb
206
+ - spec/worker_balancer_spec.rb
180
207
  - spec/worker_spec.rb
data/lib/server.rb DELETED
@@ -1,13 +0,0 @@
1
- $LOAD_PATH << __dir__
2
-
3
- require 'workerholic'
4
-
5
- auto_balance = ARGV.any? { |arg| arg == '--auto-balance' }
6
- workers_count = ARGV.find { |arg| arg.match? /^--workers=\d+$/ }
7
-
8
- if workers_count
9
- workers_count = workers_count[/\d+/].to_i
10
- Workerholic.workers_count = workers_count
11
- end
12
-
13
- Workerholic::Manager.new(auto_balance: auto_balance).start