workerholic 0.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (44) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +1 -0
  3. data/.rspec +1 -0
  4. data/Gemfile +12 -0
  5. data/Gemfile.lock +42 -0
  6. data/LICENSE +21 -0
  7. data/README.md +2 -0
  8. data/app_test/job_test.rb +20 -0
  9. data/app_test/run.rb +10 -0
  10. data/lib/server.rb +13 -0
  11. data/lib/workerholic.rb +47 -0
  12. data/lib/workerholic/adapters/active_job_adapter.rb +24 -0
  13. data/lib/workerholic/job.rb +49 -0
  14. data/lib/workerholic/job_processor.rb +29 -0
  15. data/lib/workerholic/job_retry.rb +32 -0
  16. data/lib/workerholic/job_scheduler.rb +47 -0
  17. data/lib/workerholic/job_serializer.rb +12 -0
  18. data/lib/workerholic/job_wrapper.rb +32 -0
  19. data/lib/workerholic/log_manager.rb +17 -0
  20. data/lib/workerholic/manager.rb +40 -0
  21. data/lib/workerholic/queue.rb +30 -0
  22. data/lib/workerholic/sorted_set.rb +26 -0
  23. data/lib/workerholic/statistics.rb +21 -0
  24. data/lib/workerholic/storage.rb +80 -0
  25. data/lib/workerholic/worker.rb +43 -0
  26. data/lib/workerholic/worker_balancer.rb +128 -0
  27. data/spec/helpers/helper_methods.rb +15 -0
  28. data/spec/helpers/job_tests.rb +17 -0
  29. data/spec/integration/dequeuing_and_job_processing_spec.rb +24 -0
  30. data/spec/integration/enqueuing_jobs_spec.rb +53 -0
  31. data/spec/job_processor_spec.rb +62 -0
  32. data/spec/job_retry_spec.rb +59 -0
  33. data/spec/job_scheduler_spec.rb +66 -0
  34. data/spec/job_serializer_spec.rb +28 -0
  35. data/spec/job_wrapper_spec.rb +27 -0
  36. data/spec/manager_spec.rb +26 -0
  37. data/spec/queue_spec.rb +16 -0
  38. data/spec/sorted_set.rb +25 -0
  39. data/spec/spec_helper.rb +18 -0
  40. data/spec/statistics_spec.rb +25 -0
  41. data/spec/storage_spec.rb +17 -0
  42. data/spec/worker_spec.rb +59 -0
  43. data/workerholic.gemspec +26 -0
  44. metadata +180 -0
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 61f1243274828aafe8b8c00d714b00e0832e1998
4
+ data.tar.gz: 7d1d1d50f9fa005e596b2500482831c749b9d8bd
5
+ SHA512:
6
+ metadata.gz: 6d758f61a7cb894d6b514643e3c896cf2ea42adeeda3d66c79c99ded136b8cd9369faa89027f99695b6ebb44904f2357496ad401b1233c624fd8e2e1ebf94956
7
+ data.tar.gz: c9ee873f1ed1f1a402947f777b21cd706bd72418a9d40b98566e57ebd833a7e76a3abd41b0bfc6ee0b4d9af5af4e670485508f3995c2a898fc619685dccc6bd7
data/.gitignore ADDED
@@ -0,0 +1 @@
1
+ dump.rdb
data/.rspec ADDED
@@ -0,0 +1 @@
1
+ --require spec_helper
data/Gemfile ADDED
@@ -0,0 +1,12 @@
1
+ source 'https://rubygems.org'
2
+
3
+ gem 'redis', '~> 3.3', '>= 3.3.3'
4
+ gem 'connection_pool', '~> 2.2', '>= 2.2.0'
5
+
6
+ group :development, :test do
7
+ gem 'pry-byebug'
8
+ end
9
+
10
+ group :test do
11
+ gem 'rspec', '~> 3.6', '>= 3.6.0'
12
+ end
data/Gemfile.lock ADDED
@@ -0,0 +1,42 @@
1
+ GEM
2
+ remote: https://rubygems.org/
3
+ specs:
4
+ byebug (9.0.6)
5
+ coderay (1.1.1)
6
+ connection_pool (2.2.1)
7
+ diff-lcs (1.3)
8
+ method_source (0.8.2)
9
+ pry (0.10.4)
10
+ coderay (~> 1.1.0)
11
+ method_source (~> 0.8.1)
12
+ slop (~> 3.4)
13
+ pry-byebug (3.4.2)
14
+ byebug (~> 9.0)
15
+ pry (~> 0.10)
16
+ redis (3.3.3)
17
+ rspec (3.6.0)
18
+ rspec-core (~> 3.6.0)
19
+ rspec-expectations (~> 3.6.0)
20
+ rspec-mocks (~> 3.6.0)
21
+ rspec-core (3.6.0)
22
+ rspec-support (~> 3.6.0)
23
+ rspec-expectations (3.6.0)
24
+ diff-lcs (>= 1.2.0, < 2.0)
25
+ rspec-support (~> 3.6.0)
26
+ rspec-mocks (3.6.0)
27
+ diff-lcs (>= 1.2.0, < 2.0)
28
+ rspec-support (~> 3.6.0)
29
+ rspec-support (3.6.0)
30
+ slop (3.6.0)
31
+
32
+ PLATFORMS
33
+ ruby
34
+
35
+ DEPENDENCIES
36
+ connection_pool (~> 2.2, >= 2.2.0)
37
+ pry-byebug
38
+ redis (~> 3.3, >= 3.3.3)
39
+ rspec
40
+
41
+ BUNDLED WITH
42
+ 1.15.1
data/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2017 workerholic
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,2 @@
1
+ # workerholic
2
+ A Background Job Processing Manager
@@ -0,0 +1,20 @@
1
+ require_relative '../lib/workerholic'
2
+
3
+ class JobTestFast
4
+ include Workerholic::Job
5
+ job_options queue_name: 'workerholic:queue:job_fast'
6
+
7
+ def perform(str, num)
8
+ str
9
+ end
10
+ end
11
+
12
+ class JobTestSlow
13
+ include Workerholic::Job
14
+ job_options queue_name: 'workerholic:queue:job_slow'
15
+
16
+ def perform(str, num)
17
+ str
18
+ sleep(0.1)
19
+ end
20
+ end
data/app_test/run.rb ADDED
@@ -0,0 +1,10 @@
1
+ require_relative 'job_test'
2
+
3
+ 5_000.times do |n|
4
+ JobTestFast.new.perform_async('NON BLOCKING', n)
5
+ JobTestFast.new.perform_async('NON BLOCKING', n)
6
+ # JobTestFast.new.perform_async('NON BLOCKING', n)
7
+ # JobTestFast.new.perform_async('NON BLOCKING', n)
8
+ # # JobTestSlow.new.perform_async('BLOCKING', n)
9
+ JobTestSlow.new.perform_async('BLOCKING', n)
10
+ end
data/lib/server.rb ADDED
@@ -0,0 +1,13 @@
1
+ $LOAD_PATH << __dir__
2
+
3
+ require 'workerholic'
4
+
5
+ auto_balance = ARGV.any? { |arg| arg == '--auto-balance' }
6
+ workers_count = ARGV.find { |arg| arg.match? /^--workers=\d+$/ }
7
+
8
+ if workers_count
9
+ workers_count = workers_count[/\d+/].to_i
10
+ Workerholic.workers_count = workers_count
11
+ end
12
+
13
+ Workerholic::Manager.new(auto_balance: auto_balance).start
@@ -0,0 +1,47 @@
1
+ $LOAD_PATH << __dir__
2
+
3
+ require 'yaml'
4
+ require 'redis'
5
+ require 'connection_pool'
6
+ require 'logger'
7
+ require 'pry-byebug'
8
+
9
+ require 'workerholic/manager'
10
+ require 'workerholic/worker_balancer'
11
+
12
+ require 'workerholic/job'
13
+ require 'workerholic/job_wrapper'
14
+
15
+ require 'workerholic/worker'
16
+ require 'workerholic/job_processor'
17
+ require 'workerholic/job_scheduler'
18
+ require 'workerholic/job_retry'
19
+
20
+ require 'workerholic/storage'
21
+ require 'workerholic/sorted_set'
22
+ require 'workerholic/queue'
23
+
24
+ require 'workerholic/job_serializer'
25
+ require 'workerholic/statistics'
26
+ require 'workerholic/log_manager'
27
+
28
+ require_relative '../app_test/job_test' # require the application code
29
+
30
+ module Workerholic
31
+ # DEFAULTS = {
32
+ # workers_count: 25
33
+ # }
34
+
35
+ def self.workers_count
36
+ @workers_count || 25 # DEFAULTS[:workers_count]
37
+ end
38
+
39
+ def self.workers_count=(num)
40
+ raise ArgumentError unless num.is_a?(Integer) && num < 200
41
+ @workers_count = num
42
+ end
43
+
44
+ def self.redis_pool
45
+ @redis ||= ConnectionPool.new(size: workers_count + 5, timeout: 5) { Redis.new }
46
+ end
47
+ end
@@ -0,0 +1,24 @@
1
+ module ActiveJob
2
+ module QueueAdapters
3
+ class WorkerholicAdapter
4
+ def enqueue(job)
5
+ job_data = job.serialize
6
+
7
+ JobWrapper.new.perform_async(
8
+ class: job_data[:job_class],
9
+ arguments: job_data[:arguments]
10
+ )
11
+ end
12
+
13
+ class JobWrapper
14
+ include Workerholic::Job
15
+
16
+ def perform(job_data)
17
+ Base.execute job_data
18
+ end
19
+ end
20
+ end
21
+ end
22
+
23
+ autoload :WorkerholicAdapter
24
+ end
@@ -0,0 +1,49 @@
1
+ module Workerholic
2
+ module Job
3
+ def self.included(base)
4
+ base.extend(ClassMethods)
5
+ base.job_options
6
+ end
7
+
8
+ module ClassMethods
9
+ def job_options(params={})
10
+ define_method(:specified_job_options) do
11
+ {
12
+ execute_at: params[:execute_at],
13
+ queue_name: params[:queue_name] || 'workerholic:queue:main'
14
+ }
15
+ end
16
+ end
17
+ end
18
+
19
+ def perform_async(*args)
20
+ serialized_job, queue_name = prepare_job_for_enqueueing(args)
21
+
22
+ Queue.new(queue_name).enqueue(serialized_job)
23
+ end
24
+
25
+ def perform_delayed(*args)
26
+ delay_in_sec = verify_delay(args[0])
27
+ serialized_job, queue_name = prepare_job_for_enqueueing(args)
28
+
29
+ JobScheduler.new(set_name: queue_name).schedule(serialized_job, delay_in_sec)
30
+ end
31
+
32
+ private
33
+
34
+ def verify_delay(delay_arg)
35
+ raise ArgumentError, 'Delay argument has to be of Numeric type' unless delay_arg.is_a? Numeric
36
+
37
+ delay_arg
38
+ end
39
+
40
+ def prepare_job_for_enqueueing(args)
41
+ raise ArgumentError if self.method(:perform).arity != args.size
42
+
43
+ job = JobWrapper.new(class: self.class, arguments: args)
44
+ job.statistics.enqueued_at = Time.now.to_f
45
+
46
+ [JobSerializer.serialize(job), specified_job_options[:queue_name]]
47
+ end
48
+ end
49
+ end
@@ -0,0 +1,29 @@
1
+ module Workerholic
2
+ class JobProcessor
3
+ def initialize(serialized_job)
4
+ @serialized_job = serialized_job
5
+ @logger = LogManager.new
6
+ end
7
+
8
+ def process
9
+ job = JobSerializer.deserialize(@serialized_job)
10
+
11
+ begin
12
+ job.statistics.started_at = Time.now.to_f
13
+ job_result = job.perform
14
+ job.statistics.completed_at = Time.now.to_f
15
+
16
+ # @logger.log('info', "Completed: your job from class #{job.klass} was completed on #{job.statistics.completed_at}.")
17
+
18
+ job_result
19
+ rescue Exception => e
20
+ job.statistics.errors.push([e.class, e.message])
21
+ JobRetry.new(job: job)
22
+
23
+ # @logger.log('error', "Failed: your job from class #{job.class} was unsuccessful. Retrying in 10 seconds.")
24
+ end
25
+
26
+ # Push job into some collection
27
+ end
28
+ end
29
+ end
@@ -0,0 +1,32 @@
1
+ module Workerholic
2
+ class JobRetry
3
+ attr_reader :job, :sorted_set
4
+
5
+ def initialize(options={})
6
+ @job = options[:job]
7
+ @sorted_set = options[:sorted_set] || SortedSet.new('workerholic:scheduled_jobs')
8
+
9
+ self.retry
10
+ end
11
+
12
+ protected
13
+
14
+ def retry
15
+ return if job.retry_count >= 5
16
+
17
+ increment_retry_count
18
+ schedule_job_for_retry
19
+ sorted_set.add(JobSerializer.serialize(job), job.execute_at)
20
+ end
21
+
22
+ private
23
+
24
+ def increment_retry_count
25
+ job.retry_count += 1
26
+ end
27
+
28
+ def schedule_job_for_retry
29
+ job.execute_at = Time.now.to_f + 10 * job.retry_count
30
+ end
31
+ end
32
+ end
@@ -0,0 +1,47 @@
1
+ module Workerholic
2
+ class JobScheduler
3
+ attr_reader :sorted_set, :queue, :scheduler_thread
4
+ attr_accessor :alive
5
+
6
+ def initialize(opts={})
7
+ @sorted_set = SortedSet.new(opts[:set_name] || 'workerholic:scheduled_jobs')
8
+ @queue = Queue.new(opts[:queue_name] || 'workerholic:main')
9
+ @alive = true
10
+ end
11
+
12
+ def start
13
+ @scheduler_thread = Thread.new do
14
+ enqueue_due_jobs while alive
15
+ end
16
+ end
17
+
18
+ def job_due?
19
+ scheduled_job = sorted_set.peek
20
+ return false unless scheduled_job
21
+
22
+ job_execution_time = scheduled_job.last
23
+ Time.now.to_f >= job_execution_time
24
+ end
25
+
26
+ def schedule(serialized_job, score)
27
+ sorted_set.add(serialized_job, score)
28
+ end
29
+
30
+ def enqueue_due_jobs
31
+ if job_due?
32
+ while job_due?
33
+ serialized_job, job_execution_time = sorted_set.peek
34
+ sorted_set.remove(job_execution_time)
35
+ queue.enqueue(serialized_job)
36
+ end
37
+ else
38
+ sleep(5)
39
+ end
40
+ end
41
+
42
+ def kill
43
+ self.alive = false
44
+ scheduler_thread.join if scheduler_thread
45
+ end
46
+ end
47
+ end
@@ -0,0 +1,12 @@
1
+ module Workerholic
2
+ class JobSerializer
3
+ def self.serialize(job)
4
+ YAML.dump(job.to_hash)
5
+ end
6
+
7
+ def self.deserialize(yaml_job)
8
+ job_info = YAML.load(yaml_job)
9
+ JobWrapper.new(job_info)
10
+ end
11
+ end
12
+ end
@@ -0,0 +1,32 @@
1
+ module Workerholic
2
+ class JobWrapper
3
+ attr_accessor :retry_count, :execute_at
4
+ attr_reader :klass, :arguments, :statistics
5
+
6
+ def initialize(options={})
7
+ @klass = options[:class]
8
+ @arguments = options[:arguments]
9
+ @execute_at = options[:execute_at]
10
+ @retry_count = options[:retry_count] || 0
11
+ @statistics = Statistics.new(options[:statistics] || {})
12
+ end
13
+
14
+ def to_hash
15
+ {
16
+ class: klass,
17
+ arguments: arguments,
18
+ retry_count: retry_count,
19
+ execute_at: execute_at,
20
+ statistics: statistics.to_hash
21
+ }
22
+ end
23
+
24
+ def perform
25
+ klass.new.perform(*arguments)
26
+ end
27
+
28
+ def ==(other)
29
+ to_hash == other.to_hash
30
+ end
31
+ end
32
+ end
@@ -0,0 +1,17 @@
1
+ module Workerholic
2
+ class LogManager
3
+ attr_reader :logger
4
+
5
+ def initialize
6
+ @logger = Logger.new(STDOUT)
7
+ end
8
+
9
+ def log(severity, message)
10
+ logger.formatter = proc do |severity, datetime, progname, msg|
11
+ "#{severity}: #{msg}\n"
12
+ end
13
+
14
+ logger.send(severity, message)
15
+ end
16
+ end
17
+ end