toiler 0.1.5 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. checksums.yaml +4 -4
  2. data/.ruby-version +1 -0
  3. data/Gemfile +7 -4
  4. data/Gemfile.lock +26 -57
  5. data/README.md +15 -0
  6. data/lib/toiler.rb +15 -44
  7. data/lib/toiler/actor/fetcher.rb +89 -0
  8. data/lib/toiler/actor/processor.rb +106 -0
  9. data/lib/toiler/actor/supervisor.rb +45 -0
  10. data/lib/toiler/actor/utils/actor_logging.rb +28 -0
  11. data/lib/toiler/aws/message.rb +64 -0
  12. data/lib/toiler/aws/queue.rb +61 -0
  13. data/lib/toiler/cli.rb +67 -94
  14. data/lib/toiler/utils/argument_parser.rb +50 -0
  15. data/lib/toiler/utils/environment_loader.rb +104 -0
  16. data/lib/toiler/utils/logging.rb +37 -0
  17. data/lib/toiler/version.rb +2 -1
  18. data/lib/toiler/worker.rb +35 -15
  19. data/toiler.gemspec +4 -4
  20. metadata +32 -32
  21. data/celluloid-task-pooledfiber/.gitignore +0 -9
  22. data/celluloid-task-pooledfiber/.rspec +0 -2
  23. data/celluloid-task-pooledfiber/.travis.yml +0 -5
  24. data/celluloid-task-pooledfiber/Gemfile +0 -11
  25. data/celluloid-task-pooledfiber/LICENSE.txt +0 -21
  26. data/celluloid-task-pooledfiber/README.md +0 -37
  27. data/celluloid-task-pooledfiber/Rakefile +0 -8
  28. data/celluloid-task-pooledfiber/celluloid-task-pooled-fiber.gemspec +0 -18
  29. data/celluloid-task-pooledfiber/lib/celluloid/task/pooled_fiber.rb +0 -26
  30. data/celluloid-task-pooledfiber/lib/celluloid/util/fiber_pool.rb +0 -95
  31. data/celluloid-task-pooledfiber/spec/celluloid/tasks/pooled_fiber_spec.rb +0 -5
  32. data/celluloid-task-pooledfiber/spec/spec_helper.rb +0 -60
  33. data/celluloid-task-pooledfiber/spec/support/shared_examples_for_task.rb +0 -49
  34. data/lib/toiler/core_ext.rb +0 -47
  35. data/lib/toiler/environment_loader.rb +0 -82
  36. data/lib/toiler/fetcher.rb +0 -56
  37. data/lib/toiler/logging.rb +0 -42
  38. data/lib/toiler/manager.rb +0 -71
  39. data/lib/toiler/message.rb +0 -60
  40. data/lib/toiler/processor.rb +0 -86
  41. data/lib/toiler/queue.rb +0 -53
  42. data/lib/toiler/scheduler.rb +0 -16
  43. data/lib/toiler/supervisor.rb +0 -66
@@ -1,42 +0,0 @@
1
- require 'time'
2
- require 'logger'
3
-
4
- module Toiler
5
- module Logging
6
- class Pretty < Logger::Formatter
7
- # Provide a call() method that returns the formatted message.
8
- def call(severity, time, _program_name, message)
9
- "#{time.utc.iso8601} #{Process.pid} TID-#{Thread.current.object_id.to_s(36)}#{context} #{severity}: #{message}\n"
10
- end
11
-
12
- def context
13
- c = Thread.current[:toiler_context]
14
- c ? " #{c}" : ''
15
- end
16
- end
17
-
18
- module_function
19
-
20
- def with_context(msg)
21
- Thread.current[:toiler_context] = msg
22
- yield
23
- ensure
24
- Thread.current[:toiler_context] = nil
25
- end
26
-
27
- def initialize_logger(log_target = STDOUT)
28
- @logger = Logger.new(log_target)
29
- @logger.level = Logger::INFO
30
- @logger.formatter = Pretty.new
31
- @logger
32
- end
33
-
34
- def logger
35
- @logger || initialize_logger
36
- end
37
-
38
- def logger=(log)
39
- @logger = (log ? log : Logger.new('/dev/null'))
40
- end
41
- end
42
- end
@@ -1,71 +0,0 @@
1
- require 'toiler/fetcher'
2
- require 'toiler/processor'
3
-
4
- module Toiler
5
- class Manager
6
- include Celluloid
7
- include Celluloid::Internals::Logger
8
-
9
- finalizer :shutdown
10
-
11
- def initialize
12
- async.init
13
- end
14
-
15
- def init
16
- debug 'Initializing manager...'
17
- init_workers
18
- awake_fetchers
19
- debug 'Finished initializing manager...'
20
- end
21
-
22
- def awake_fetchers
23
- queues.each do |q, _klass|
24
- fetcher = Toiler.fetcher(q)
25
- fetcher.processor_finished if fetcher && fetcher.alive? && free_processors(q) > 0
26
- end
27
- end
28
-
29
- def queues
30
- Toiler.worker_class_registry
31
- end
32
-
33
- def shutdown
34
- debug 'Manager shutting down...'
35
- instance_variables.each { |iv| remove_instance_variable iv }
36
- end
37
-
38
- def processor_finished(queue)
39
- fetcher = Toiler.fetcher(queue)
40
- fetcher.processor_finished if fetcher && fetcher.alive?
41
- end
42
-
43
- def init_workers
44
- queues.each do |q, klass|
45
- Toiler.worker_registry[q] = klass.new
46
- end
47
- end
48
-
49
- def free_processors(queue)
50
- pool = Toiler.processor_pool(queue)
51
- pool && pool.alive? ? pool.idle_size : 0
52
- end
53
-
54
- def assign_messages(queue, messages)
55
- debug "Manager assigning #{messages.count} for queue #{queue}"
56
- processor_pool = Toiler.processor_pool(queue)
57
- if batch? queue
58
- processor_pool.async.process(queue, messages)
59
- else
60
- messages.each do |m|
61
- processor_pool.async.process(queue, m)
62
- end
63
- end
64
- debug "Manager finished assigning #{messages.count} for queue #{queue}"
65
- end
66
-
67
- def batch?(queue)
68
- queues[queue].batch?
69
- end
70
- end
71
- end
@@ -1,60 +0,0 @@
1
- module Toiler
2
- class Message
3
- attr_accessor :client, :queue_url, :data
4
-
5
- def initialize(client, queue_url, data)
6
- @client = client
7
- @queue_url = queue_url
8
- @data = data
9
- end
10
-
11
- def delete
12
- client.delete_message(
13
- queue_url: queue_url,
14
- receipt_handle: data.receipt_handle
15
- )
16
- end
17
-
18
- def change_visibility(options)
19
- client.change_message_visibility(
20
- options.merge(queue_url: queue_url, receipt_handle: data.receipt_handle)
21
- )
22
- end
23
-
24
- def visibility_timeout=(timeout)
25
- client.change_message_visibility(
26
- queue_url: queue_url,
27
- receipt_handle: data.receipt_handle,
28
- visibility_timeout: timeout
29
- )
30
- end
31
-
32
- def message_id
33
- data.message_id
34
- end
35
-
36
- def receipt_handle
37
- data.receipt_handle
38
- end
39
-
40
- def md5_of_body
41
- data.md5_of_body
42
- end
43
-
44
- def body
45
- data.body
46
- end
47
-
48
- def attributes
49
- data.attributes
50
- end
51
-
52
- def md5_of_message_attributes
53
- data.md5_of_message_attributes
54
- end
55
-
56
- def message_attributes
57
- data.message_attributes
58
- end
59
- end
60
- end
@@ -1,86 +0,0 @@
1
- require 'json'
2
- require 'toiler/scheduler'
3
-
4
- module Toiler
5
- class Processor
6
- include Celluloid
7
- include Celluloid::Internals::Logger
8
-
9
- attr_accessor :queue
10
-
11
- finalizer :shutdown
12
-
13
- def initialize(queue)
14
- debug "Initializing Processor for queue #{queue}"
15
- @queue = queue
16
- processor_finished
17
- debug "Finished initializing Processor for queue #{queue}"
18
- end
19
-
20
- def shutdown
21
- debug "Processor for queue #{queue} shutting down..."
22
- ::ActiveRecord::Base.clear_active_connections! if defined? ActiveRecord
23
- instance_variables.each { |iv| remove_instance_variable iv }
24
- end
25
-
26
- def process(queue, sqs_msg)
27
- debug "Processor #{queue} begins processing..."
28
- worker = Toiler.worker_registry[queue]
29
- timer = auto_visibility_timeout(queue, sqs_msg, worker.class)
30
-
31
- body = get_body(worker.class, sqs_msg)
32
- worker.perform(sqs_msg, body)
33
- sqs_msg.delete if worker.class.auto_delete?
34
- rescue StandardError => e
35
- error "Processor #{queue} faild processing msg: #{e.message}\n#{e.backtrace.join("\n")}"
36
- ensure
37
- timer.cancel if timer
38
- ::ActiveRecord::Base.clear_active_connections! if defined? ActiveRecord
39
- processor_finished
40
- debug "Processor #{queue} finishes processing..."
41
- end
42
-
43
- private
44
-
45
- def processor_finished
46
- Toiler.manager.async.processor_finished queue
47
- end
48
-
49
- def auto_visibility_timeout(queue, sqs_msg, worker_class)
50
- return unless worker_class.auto_visibility_timeout?
51
- queue_visibility_timeout = Toiler.fetcher(queue).queue.visibility_timeout
52
- block = lambda do |msg, visibility_timeout, q|
53
- debug "Processor #{q} updating visibility_timeout..."
54
- msg.visibility_timeout = visibility_timeout
55
- end
56
-
57
- Toiler.scheduler(queue).custom_every(queue_visibility_timeout - 5, sqs_msg, queue_visibility_timeout, queue, block)
58
- end
59
-
60
- def get_body(worker_class, sqs_msg)
61
- if sqs_msg.is_a? Array
62
- sqs_msg.map { |m| parse_body(worker_class, m) }
63
- else
64
- parse_body(worker_class, sqs_msg)
65
- end
66
- end
67
-
68
- def parse_body(worker_class, sqs_msg)
69
- body_parser = worker_class.get_toiler_options[:parser]
70
-
71
- case body_parser
72
- when :json
73
- JSON.parse(sqs_msg.body)
74
- when Proc
75
- body_parser.call(sqs_msg)
76
- when :text, nil
77
- sqs_msg.body
78
- else
79
- body_parser.load(sqs_msg.body) if body_parser.respond_to?(:load) # i.e. Oj.load(...) or MultiJson.load(...)
80
- end
81
- rescue => e
82
- logger.error "Error parsing the message body: #{e.message}\nbody_parser: #{body_parser}\nsqs_msg.body: #{sqs_msg.body}"
83
- nil
84
- end
85
- end
86
- end
data/lib/toiler/queue.rb DELETED
@@ -1,53 +0,0 @@
1
- module Toiler
2
- class Queue
3
- attr_accessor :name, :client, :url
4
-
5
- def initialize(name, client = nil)
6
- @name = name
7
- @client = client || ::Aws::SQS::Client.new
8
- @url = client.get_queue_url(queue_name: name).queue_url
9
- end
10
-
11
- def visibility_timeout
12
- client.get_queue_attributes(
13
- queue_url: url,
14
- attribute_names: ['VisibilityTimeout']
15
- ).attributes['VisibilityTimeout'].to_i
16
- end
17
-
18
- def delete_messages(options)
19
- client.delete_message_batch(options.merge(queue_url: url))
20
- end
21
-
22
- def send_message(options)
23
- client.send_message(sanitize_message_body(options.merge(queue_url: url)))
24
- end
25
-
26
- def send_messages(options)
27
- client.send_message_batch(sanitize_message_body(options.merge(queue_url: url)))
28
- end
29
-
30
- def receive_messages(options)
31
- client.receive_message(options.merge(queue_url: url))
32
- .messages
33
- .map { |m| Message.new(client, url, m) }
34
- end
35
-
36
- private
37
-
38
- def sanitize_message_body(options)
39
- messages = options[:entries] || [options]
40
-
41
- messages.each do |m|
42
- body = m[:message_body]
43
- if body.is_a?(Hash)
44
- m[:message_body] = JSON.dump(body)
45
- elsif !body.is_a? String
46
- fail ArgumentError, "The message body must be a String and you passed a #{body.class}"
47
- end
48
- end
49
-
50
- options
51
- end
52
- end
53
- end
@@ -1,16 +0,0 @@
1
- module Toiler
2
- class Scheduler
3
- include Celluloid
4
- include Celluloid::Internals::Logger
5
-
6
- execute_block_on_receiver :custom_every
7
-
8
- def custom_every(*args, block)
9
- period = args[0]
10
- block_args = args[1..-1]
11
- every(period) do
12
- block.call(*block_args)
13
- end
14
- end
15
- end
16
- end
@@ -1,66 +0,0 @@
1
- require 'toiler/manager'
2
-
3
- module Toiler
4
- class Supervisor
5
- attr_accessor :client, :config
6
-
7
- def initialize
8
- @client = ::Aws::SQS::Client.new
9
- @config = Celluloid::Supervision::Configuration.new
10
- define_manager
11
- define_schedulers
12
- define_processors
13
- define_fetchers
14
- @config.deploy
15
- end
16
-
17
- def queues
18
- Toiler.worker_class_registry
19
- end
20
-
21
- def define_manager
22
- @config.define type: Manager, as: :manager
23
- end
24
-
25
- def define_fetchers
26
- queues.each do |queue, _klass|
27
- @config.define type: Fetcher, as: "fetcher_#{queue}".to_sym, args: [queue, client]
28
- end
29
- end
30
-
31
- def define_schedulers
32
- queues.each do |queue, _klass|
33
- @config.define type: Scheduler, as: "scheduler_#{queue}".to_sym, args: []
34
- end
35
- end
36
-
37
- def define_processors
38
- queues.each do |q, klass|
39
- @config.define type: Celluloid::Supervision::Container::Pool,
40
- as: "processor_pool_#{q}".to_sym,
41
- args: [actors: Processor, size: klass.concurrency, args: [q]]
42
- end
43
- end
44
-
45
- def stop
46
- terminate_fetchers
47
- terminate_processors
48
- Toiler.manager.terminate if Toiler.manager && Toiler.manager.alive?
49
- @config.shutdown
50
- end
51
-
52
- def terminate_fetchers
53
- queues.each do |queue, _klass|
54
- fetcher = Toiler.fetcher(queue)
55
- fetcher.terminate if fetcher && fetcher.alive?
56
- end
57
- end
58
-
59
- def terminate_processors
60
- queues.each do |queue, _klass|
61
- processor_pool = Toiler.processor_pool(queue)
62
- processor_pool.terminate if processor_pool && processor_pool.alive?
63
- end
64
- end
65
- end
66
- end