toiler 0.1.5 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.ruby-version +1 -0
- data/Gemfile +7 -4
- data/Gemfile.lock +26 -57
- data/README.md +15 -0
- data/lib/toiler.rb +15 -44
- data/lib/toiler/actor/fetcher.rb +89 -0
- data/lib/toiler/actor/processor.rb +106 -0
- data/lib/toiler/actor/supervisor.rb +45 -0
- data/lib/toiler/actor/utils/actor_logging.rb +28 -0
- data/lib/toiler/aws/message.rb +64 -0
- data/lib/toiler/aws/queue.rb +61 -0
- data/lib/toiler/cli.rb +67 -94
- data/lib/toiler/utils/argument_parser.rb +50 -0
- data/lib/toiler/utils/environment_loader.rb +104 -0
- data/lib/toiler/utils/logging.rb +37 -0
- data/lib/toiler/version.rb +2 -1
- data/lib/toiler/worker.rb +35 -15
- data/toiler.gemspec +4 -4
- metadata +32 -32
- data/celluloid-task-pooledfiber/.gitignore +0 -9
- data/celluloid-task-pooledfiber/.rspec +0 -2
- data/celluloid-task-pooledfiber/.travis.yml +0 -5
- data/celluloid-task-pooledfiber/Gemfile +0 -11
- data/celluloid-task-pooledfiber/LICENSE.txt +0 -21
- data/celluloid-task-pooledfiber/README.md +0 -37
- data/celluloid-task-pooledfiber/Rakefile +0 -8
- data/celluloid-task-pooledfiber/celluloid-task-pooled-fiber.gemspec +0 -18
- data/celluloid-task-pooledfiber/lib/celluloid/task/pooled_fiber.rb +0 -26
- data/celluloid-task-pooledfiber/lib/celluloid/util/fiber_pool.rb +0 -95
- data/celluloid-task-pooledfiber/spec/celluloid/tasks/pooled_fiber_spec.rb +0 -5
- data/celluloid-task-pooledfiber/spec/spec_helper.rb +0 -60
- data/celluloid-task-pooledfiber/spec/support/shared_examples_for_task.rb +0 -49
- data/lib/toiler/core_ext.rb +0 -47
- data/lib/toiler/environment_loader.rb +0 -82
- data/lib/toiler/fetcher.rb +0 -56
- data/lib/toiler/logging.rb +0 -42
- data/lib/toiler/manager.rb +0 -71
- data/lib/toiler/message.rb +0 -60
- data/lib/toiler/processor.rb +0 -86
- data/lib/toiler/queue.rb +0 -53
- data/lib/toiler/scheduler.rb +0 -16
- data/lib/toiler/supervisor.rb +0 -66
data/lib/toiler/logging.rb
DELETED
@@ -1,42 +0,0 @@
|
|
1
|
-
require 'time'
|
2
|
-
require 'logger'
|
3
|
-
|
4
|
-
module Toiler
|
5
|
-
module Logging
|
6
|
-
class Pretty < Logger::Formatter
|
7
|
-
# Provide a call() method that returns the formatted message.
|
8
|
-
def call(severity, time, _program_name, message)
|
9
|
-
"#{time.utc.iso8601} #{Process.pid} TID-#{Thread.current.object_id.to_s(36)}#{context} #{severity}: #{message}\n"
|
10
|
-
end
|
11
|
-
|
12
|
-
def context
|
13
|
-
c = Thread.current[:toiler_context]
|
14
|
-
c ? " #{c}" : ''
|
15
|
-
end
|
16
|
-
end
|
17
|
-
|
18
|
-
module_function
|
19
|
-
|
20
|
-
def with_context(msg)
|
21
|
-
Thread.current[:toiler_context] = msg
|
22
|
-
yield
|
23
|
-
ensure
|
24
|
-
Thread.current[:toiler_context] = nil
|
25
|
-
end
|
26
|
-
|
27
|
-
def initialize_logger(log_target = STDOUT)
|
28
|
-
@logger = Logger.new(log_target)
|
29
|
-
@logger.level = Logger::INFO
|
30
|
-
@logger.formatter = Pretty.new
|
31
|
-
@logger
|
32
|
-
end
|
33
|
-
|
34
|
-
def logger
|
35
|
-
@logger || initialize_logger
|
36
|
-
end
|
37
|
-
|
38
|
-
def logger=(log)
|
39
|
-
@logger = (log ? log : Logger.new('/dev/null'))
|
40
|
-
end
|
41
|
-
end
|
42
|
-
end
|
data/lib/toiler/manager.rb
DELETED
@@ -1,71 +0,0 @@
|
|
1
|
-
require 'toiler/fetcher'
|
2
|
-
require 'toiler/processor'
|
3
|
-
|
4
|
-
module Toiler
|
5
|
-
class Manager
|
6
|
-
include Celluloid
|
7
|
-
include Celluloid::Internals::Logger
|
8
|
-
|
9
|
-
finalizer :shutdown
|
10
|
-
|
11
|
-
def initialize
|
12
|
-
async.init
|
13
|
-
end
|
14
|
-
|
15
|
-
def init
|
16
|
-
debug 'Initializing manager...'
|
17
|
-
init_workers
|
18
|
-
awake_fetchers
|
19
|
-
debug 'Finished initializing manager...'
|
20
|
-
end
|
21
|
-
|
22
|
-
def awake_fetchers
|
23
|
-
queues.each do |q, _klass|
|
24
|
-
fetcher = Toiler.fetcher(q)
|
25
|
-
fetcher.processor_finished if fetcher && fetcher.alive? && free_processors(q) > 0
|
26
|
-
end
|
27
|
-
end
|
28
|
-
|
29
|
-
def queues
|
30
|
-
Toiler.worker_class_registry
|
31
|
-
end
|
32
|
-
|
33
|
-
def shutdown
|
34
|
-
debug 'Manager shutting down...'
|
35
|
-
instance_variables.each { |iv| remove_instance_variable iv }
|
36
|
-
end
|
37
|
-
|
38
|
-
def processor_finished(queue)
|
39
|
-
fetcher = Toiler.fetcher(queue)
|
40
|
-
fetcher.processor_finished if fetcher && fetcher.alive?
|
41
|
-
end
|
42
|
-
|
43
|
-
def init_workers
|
44
|
-
queues.each do |q, klass|
|
45
|
-
Toiler.worker_registry[q] = klass.new
|
46
|
-
end
|
47
|
-
end
|
48
|
-
|
49
|
-
def free_processors(queue)
|
50
|
-
pool = Toiler.processor_pool(queue)
|
51
|
-
pool && pool.alive? ? pool.idle_size : 0
|
52
|
-
end
|
53
|
-
|
54
|
-
def assign_messages(queue, messages)
|
55
|
-
debug "Manager assigning #{messages.count} for queue #{queue}"
|
56
|
-
processor_pool = Toiler.processor_pool(queue)
|
57
|
-
if batch? queue
|
58
|
-
processor_pool.async.process(queue, messages)
|
59
|
-
else
|
60
|
-
messages.each do |m|
|
61
|
-
processor_pool.async.process(queue, m)
|
62
|
-
end
|
63
|
-
end
|
64
|
-
debug "Manager finished assigning #{messages.count} for queue #{queue}"
|
65
|
-
end
|
66
|
-
|
67
|
-
def batch?(queue)
|
68
|
-
queues[queue].batch?
|
69
|
-
end
|
70
|
-
end
|
71
|
-
end
|
data/lib/toiler/message.rb
DELETED
@@ -1,60 +0,0 @@
|
|
1
|
-
module Toiler
|
2
|
-
class Message
|
3
|
-
attr_accessor :client, :queue_url, :data
|
4
|
-
|
5
|
-
def initialize(client, queue_url, data)
|
6
|
-
@client = client
|
7
|
-
@queue_url = queue_url
|
8
|
-
@data = data
|
9
|
-
end
|
10
|
-
|
11
|
-
def delete
|
12
|
-
client.delete_message(
|
13
|
-
queue_url: queue_url,
|
14
|
-
receipt_handle: data.receipt_handle
|
15
|
-
)
|
16
|
-
end
|
17
|
-
|
18
|
-
def change_visibility(options)
|
19
|
-
client.change_message_visibility(
|
20
|
-
options.merge(queue_url: queue_url, receipt_handle: data.receipt_handle)
|
21
|
-
)
|
22
|
-
end
|
23
|
-
|
24
|
-
def visibility_timeout=(timeout)
|
25
|
-
client.change_message_visibility(
|
26
|
-
queue_url: queue_url,
|
27
|
-
receipt_handle: data.receipt_handle,
|
28
|
-
visibility_timeout: timeout
|
29
|
-
)
|
30
|
-
end
|
31
|
-
|
32
|
-
def message_id
|
33
|
-
data.message_id
|
34
|
-
end
|
35
|
-
|
36
|
-
def receipt_handle
|
37
|
-
data.receipt_handle
|
38
|
-
end
|
39
|
-
|
40
|
-
def md5_of_body
|
41
|
-
data.md5_of_body
|
42
|
-
end
|
43
|
-
|
44
|
-
def body
|
45
|
-
data.body
|
46
|
-
end
|
47
|
-
|
48
|
-
def attributes
|
49
|
-
data.attributes
|
50
|
-
end
|
51
|
-
|
52
|
-
def md5_of_message_attributes
|
53
|
-
data.md5_of_message_attributes
|
54
|
-
end
|
55
|
-
|
56
|
-
def message_attributes
|
57
|
-
data.message_attributes
|
58
|
-
end
|
59
|
-
end
|
60
|
-
end
|
data/lib/toiler/processor.rb
DELETED
@@ -1,86 +0,0 @@
|
|
1
|
-
require 'json'
|
2
|
-
require 'toiler/scheduler'
|
3
|
-
|
4
|
-
module Toiler
|
5
|
-
class Processor
|
6
|
-
include Celluloid
|
7
|
-
include Celluloid::Internals::Logger
|
8
|
-
|
9
|
-
attr_accessor :queue
|
10
|
-
|
11
|
-
finalizer :shutdown
|
12
|
-
|
13
|
-
def initialize(queue)
|
14
|
-
debug "Initializing Processor for queue #{queue}"
|
15
|
-
@queue = queue
|
16
|
-
processor_finished
|
17
|
-
debug "Finished initializing Processor for queue #{queue}"
|
18
|
-
end
|
19
|
-
|
20
|
-
def shutdown
|
21
|
-
debug "Processor for queue #{queue} shutting down..."
|
22
|
-
::ActiveRecord::Base.clear_active_connections! if defined? ActiveRecord
|
23
|
-
instance_variables.each { |iv| remove_instance_variable iv }
|
24
|
-
end
|
25
|
-
|
26
|
-
def process(queue, sqs_msg)
|
27
|
-
debug "Processor #{queue} begins processing..."
|
28
|
-
worker = Toiler.worker_registry[queue]
|
29
|
-
timer = auto_visibility_timeout(queue, sqs_msg, worker.class)
|
30
|
-
|
31
|
-
body = get_body(worker.class, sqs_msg)
|
32
|
-
worker.perform(sqs_msg, body)
|
33
|
-
sqs_msg.delete if worker.class.auto_delete?
|
34
|
-
rescue StandardError => e
|
35
|
-
error "Processor #{queue} faild processing msg: #{e.message}\n#{e.backtrace.join("\n")}"
|
36
|
-
ensure
|
37
|
-
timer.cancel if timer
|
38
|
-
::ActiveRecord::Base.clear_active_connections! if defined? ActiveRecord
|
39
|
-
processor_finished
|
40
|
-
debug "Processor #{queue} finishes processing..."
|
41
|
-
end
|
42
|
-
|
43
|
-
private
|
44
|
-
|
45
|
-
def processor_finished
|
46
|
-
Toiler.manager.async.processor_finished queue
|
47
|
-
end
|
48
|
-
|
49
|
-
def auto_visibility_timeout(queue, sqs_msg, worker_class)
|
50
|
-
return unless worker_class.auto_visibility_timeout?
|
51
|
-
queue_visibility_timeout = Toiler.fetcher(queue).queue.visibility_timeout
|
52
|
-
block = lambda do |msg, visibility_timeout, q|
|
53
|
-
debug "Processor #{q} updating visibility_timeout..."
|
54
|
-
msg.visibility_timeout = visibility_timeout
|
55
|
-
end
|
56
|
-
|
57
|
-
Toiler.scheduler(queue).custom_every(queue_visibility_timeout - 5, sqs_msg, queue_visibility_timeout, queue, block)
|
58
|
-
end
|
59
|
-
|
60
|
-
def get_body(worker_class, sqs_msg)
|
61
|
-
if sqs_msg.is_a? Array
|
62
|
-
sqs_msg.map { |m| parse_body(worker_class, m) }
|
63
|
-
else
|
64
|
-
parse_body(worker_class, sqs_msg)
|
65
|
-
end
|
66
|
-
end
|
67
|
-
|
68
|
-
def parse_body(worker_class, sqs_msg)
|
69
|
-
body_parser = worker_class.get_toiler_options[:parser]
|
70
|
-
|
71
|
-
case body_parser
|
72
|
-
when :json
|
73
|
-
JSON.parse(sqs_msg.body)
|
74
|
-
when Proc
|
75
|
-
body_parser.call(sqs_msg)
|
76
|
-
when :text, nil
|
77
|
-
sqs_msg.body
|
78
|
-
else
|
79
|
-
body_parser.load(sqs_msg.body) if body_parser.respond_to?(:load) # i.e. Oj.load(...) or MultiJson.load(...)
|
80
|
-
end
|
81
|
-
rescue => e
|
82
|
-
logger.error "Error parsing the message body: #{e.message}\nbody_parser: #{body_parser}\nsqs_msg.body: #{sqs_msg.body}"
|
83
|
-
nil
|
84
|
-
end
|
85
|
-
end
|
86
|
-
end
|
data/lib/toiler/queue.rb
DELETED
@@ -1,53 +0,0 @@
|
|
1
|
-
module Toiler
|
2
|
-
class Queue
|
3
|
-
attr_accessor :name, :client, :url
|
4
|
-
|
5
|
-
def initialize(name, client = nil)
|
6
|
-
@name = name
|
7
|
-
@client = client || ::Aws::SQS::Client.new
|
8
|
-
@url = client.get_queue_url(queue_name: name).queue_url
|
9
|
-
end
|
10
|
-
|
11
|
-
def visibility_timeout
|
12
|
-
client.get_queue_attributes(
|
13
|
-
queue_url: url,
|
14
|
-
attribute_names: ['VisibilityTimeout']
|
15
|
-
).attributes['VisibilityTimeout'].to_i
|
16
|
-
end
|
17
|
-
|
18
|
-
def delete_messages(options)
|
19
|
-
client.delete_message_batch(options.merge(queue_url: url))
|
20
|
-
end
|
21
|
-
|
22
|
-
def send_message(options)
|
23
|
-
client.send_message(sanitize_message_body(options.merge(queue_url: url)))
|
24
|
-
end
|
25
|
-
|
26
|
-
def send_messages(options)
|
27
|
-
client.send_message_batch(sanitize_message_body(options.merge(queue_url: url)))
|
28
|
-
end
|
29
|
-
|
30
|
-
def receive_messages(options)
|
31
|
-
client.receive_message(options.merge(queue_url: url))
|
32
|
-
.messages
|
33
|
-
.map { |m| Message.new(client, url, m) }
|
34
|
-
end
|
35
|
-
|
36
|
-
private
|
37
|
-
|
38
|
-
def sanitize_message_body(options)
|
39
|
-
messages = options[:entries] || [options]
|
40
|
-
|
41
|
-
messages.each do |m|
|
42
|
-
body = m[:message_body]
|
43
|
-
if body.is_a?(Hash)
|
44
|
-
m[:message_body] = JSON.dump(body)
|
45
|
-
elsif !body.is_a? String
|
46
|
-
fail ArgumentError, "The message body must be a String and you passed a #{body.class}"
|
47
|
-
end
|
48
|
-
end
|
49
|
-
|
50
|
-
options
|
51
|
-
end
|
52
|
-
end
|
53
|
-
end
|
data/lib/toiler/scheduler.rb
DELETED
@@ -1,16 +0,0 @@
|
|
1
|
-
module Toiler
|
2
|
-
class Scheduler
|
3
|
-
include Celluloid
|
4
|
-
include Celluloid::Internals::Logger
|
5
|
-
|
6
|
-
execute_block_on_receiver :custom_every
|
7
|
-
|
8
|
-
def custom_every(*args, block)
|
9
|
-
period = args[0]
|
10
|
-
block_args = args[1..-1]
|
11
|
-
every(period) do
|
12
|
-
block.call(*block_args)
|
13
|
-
end
|
14
|
-
end
|
15
|
-
end
|
16
|
-
end
|
data/lib/toiler/supervisor.rb
DELETED
@@ -1,66 +0,0 @@
|
|
1
|
-
require 'toiler/manager'
|
2
|
-
|
3
|
-
module Toiler
|
4
|
-
class Supervisor
|
5
|
-
attr_accessor :client, :config
|
6
|
-
|
7
|
-
def initialize
|
8
|
-
@client = ::Aws::SQS::Client.new
|
9
|
-
@config = Celluloid::Supervision::Configuration.new
|
10
|
-
define_manager
|
11
|
-
define_schedulers
|
12
|
-
define_processors
|
13
|
-
define_fetchers
|
14
|
-
@config.deploy
|
15
|
-
end
|
16
|
-
|
17
|
-
def queues
|
18
|
-
Toiler.worker_class_registry
|
19
|
-
end
|
20
|
-
|
21
|
-
def define_manager
|
22
|
-
@config.define type: Manager, as: :manager
|
23
|
-
end
|
24
|
-
|
25
|
-
def define_fetchers
|
26
|
-
queues.each do |queue, _klass|
|
27
|
-
@config.define type: Fetcher, as: "fetcher_#{queue}".to_sym, args: [queue, client]
|
28
|
-
end
|
29
|
-
end
|
30
|
-
|
31
|
-
def define_schedulers
|
32
|
-
queues.each do |queue, _klass|
|
33
|
-
@config.define type: Scheduler, as: "scheduler_#{queue}".to_sym, args: []
|
34
|
-
end
|
35
|
-
end
|
36
|
-
|
37
|
-
def define_processors
|
38
|
-
queues.each do |q, klass|
|
39
|
-
@config.define type: Celluloid::Supervision::Container::Pool,
|
40
|
-
as: "processor_pool_#{q}".to_sym,
|
41
|
-
args: [actors: Processor, size: klass.concurrency, args: [q]]
|
42
|
-
end
|
43
|
-
end
|
44
|
-
|
45
|
-
def stop
|
46
|
-
terminate_fetchers
|
47
|
-
terminate_processors
|
48
|
-
Toiler.manager.terminate if Toiler.manager && Toiler.manager.alive?
|
49
|
-
@config.shutdown
|
50
|
-
end
|
51
|
-
|
52
|
-
def terminate_fetchers
|
53
|
-
queues.each do |queue, _klass|
|
54
|
-
fetcher = Toiler.fetcher(queue)
|
55
|
-
fetcher.terminate if fetcher && fetcher.alive?
|
56
|
-
end
|
57
|
-
end
|
58
|
-
|
59
|
-
def terminate_processors
|
60
|
-
queues.each do |queue, _klass|
|
61
|
-
processor_pool = Toiler.processor_pool(queue)
|
62
|
-
processor_pool.terminate if processor_pool && processor_pool.alive?
|
63
|
-
end
|
64
|
-
end
|
65
|
-
end
|
66
|
-
end
|