toiler 0.3.5 → 0.3.6

Sign up to get free protection for your applications and to get access to all the features.
data/Rakefile CHANGED
@@ -1 +1 @@
1
- require 'bundler/gem_tasks'
1
+ require 'bundler/gem_tasks'
data/bin/toiler CHANGED
@@ -1,12 +1,12 @@
1
- #!/usr/bin/env ruby
2
-
3
- require 'toiler'
4
-
5
- begin
6
- Toiler::CLI.instance.run(ARGV)
7
- rescue => e
8
- raise e if $DEBUG
9
- STDERR.puts e.message
10
- STDERR.puts e.backtrace.join("\n")
11
- exit 1
12
- end
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'toiler'
4
+
5
+ begin
6
+ Toiler::CLI.instance.run(ARGV)
7
+ rescue => e
8
+ raise e if $DEBUG
9
+ STDERR.puts e.message
10
+ STDERR.puts e.backtrace.join("\n")
11
+ exit 1
12
+ end
@@ -1,55 +1,55 @@
1
- require 'aws-sdk'
2
- require 'toiler/utils/environment_loader'
3
- require 'toiler/utils/logging'
4
- require 'toiler/utils/argument_parser'
5
- require 'toiler/worker'
6
- require 'toiler/cli'
7
- require 'toiler/version'
8
-
9
- # Main module
10
- module Toiler
11
- @worker_class_registry = {}
12
- @options = {
13
- aws: {}
14
- }
15
- @fetchers = {}
16
- @processor_pools = {}
17
-
18
- attr_reader :worker_class_registry, :options, :fetchers, :processor_pools
19
- module_function :worker_class_registry, :options, :fetchers, :processor_pools
20
-
21
- module_function
22
-
23
- def logger
24
- Toiler::Utils::Logging.logger
25
- end
26
-
27
- def queues
28
- worker_class_registry.keys
29
- end
30
-
31
- def fetcher(queue)
32
- fetchers["fetcher_#{queue}".to_sym]
33
- end
34
-
35
- def set_fetcher(queue, val)
36
- fetchers["fetcher_#{queue}".to_sym] = val
37
- end
38
-
39
- def processor_pool(queue)
40
- processor_pools["processor_pool_#{queue}".to_sym]
41
- end
42
-
43
- def set_processor_pool(queue, val)
44
- processor_pools["processor_pool_#{queue}".to_sym] = val
45
- end
46
-
47
- def default_options
48
- {
49
- auto_visibility_timeout: false,
50
- concurrency: 1,
51
- auto_delete: false,
52
- batch: false
53
- }
54
- end
55
- end
1
+ require 'aws-sdk'
2
+ require 'toiler/utils/environment_loader'
3
+ require 'toiler/utils/logging'
4
+ require 'toiler/utils/argument_parser'
5
+ require 'toiler/worker'
6
+ require 'toiler/cli'
7
+ require 'toiler/version'
8
+
9
+ # Main module
10
+ module Toiler
11
+ @worker_class_registry = {}
12
+ @options = {
13
+ aws: {}
14
+ }
15
+ @fetchers = {}
16
+ @processor_pools = {}
17
+
18
+ attr_reader :worker_class_registry, :options, :fetchers, :processor_pools
19
+ module_function :worker_class_registry, :options, :fetchers, :processor_pools
20
+
21
+ module_function
22
+
23
+ def logger
24
+ Toiler::Utils::Logging.logger
25
+ end
26
+
27
+ def queues
28
+ worker_class_registry.keys
29
+ end
30
+
31
+ def fetcher(queue)
32
+ fetchers["fetcher_#{queue}".to_sym]
33
+ end
34
+
35
+ def set_fetcher(queue, val)
36
+ fetchers["fetcher_#{queue}".to_sym] = val
37
+ end
38
+
39
+ def processor_pool(queue)
40
+ processor_pools["processor_pool_#{queue}".to_sym]
41
+ end
42
+
43
+ def set_processor_pool(queue, val)
44
+ processor_pools["processor_pool_#{queue}".to_sym] = val
45
+ end
46
+
47
+ def default_options
48
+ {
49
+ auto_visibility_timeout: false,
50
+ concurrency: 1,
51
+ auto_delete: false,
52
+ batch: false
53
+ }
54
+ end
55
+ end
@@ -1,116 +1,116 @@
1
- require 'toiler/actor/utils/actor_logging'
2
- require 'toiler/aws/queue'
3
-
4
- module Toiler
5
- module Actor
6
- # Actor polling for messages only when processors are ready, otherwise idle
7
- class Fetcher < Concurrent::Actor::RestartingContext
8
- include Utils::ActorLogging
9
-
10
- FETCH_LIMIT = 10
11
-
12
- attr_accessor :queue, :wait, :visibility_timeout, :free_processors,
13
- :scheduled, :executing, :polling
14
-
15
- def initialize(queue, client)
16
- debug "Initializing Fetcher for queue #{queue}..."
17
- @queue = Toiler::Aws::Queue.new queue, client
18
- @wait = Toiler.options[:wait] || 20
19
- @free_processors = Concurrent::AtomicFixnum.new(0)
20
- @batch = Toiler.worker_class_registry[queue].batch?
21
- @visibility_timeout = @queue.visibility_timeout
22
- @scheduled = Concurrent::AtomicBoolean.new
23
- @executing = Concurrent::AtomicBoolean.new
24
- @polling = Concurrent::AtomicBoolean.new
25
- debug "Finished initializing Fetcher for queue #{queue}"
26
- end
27
-
28
- def default_executor
29
- Concurrent.global_io_executor
30
- end
31
-
32
- def on_message(msg)
33
- executing.make_true
34
- method, *args = msg
35
- send(method, *args)
36
- rescue StandardError => e
37
- error "Fetcher #{queue.name} raised exception #{e.class}: #{e.message}\n#{e.backtrace.join("\n")}"
38
- ensure
39
- executing.make_false
40
- end
41
-
42
- def executing?
43
- executing.value
44
- end
45
-
46
- def polling?
47
- polling.value
48
- end
49
-
50
- def scheduled?
51
- scheduled.value
52
- end
53
-
54
- def get_free_processors
55
- free_processors.value
56
- end
57
-
58
- private
59
-
60
- def batch?
61
- @batch
62
- end
63
-
64
- def processor_finished
65
- debug "Fetcher #{queue.name} received processor finished signal..."
66
- free_processors.increment
67
- schedule_poll
68
- end
69
-
70
- def max_messages
71
- batch? ? FETCH_LIMIT : [FETCH_LIMIT, free_processors.value].min
72
- end
73
-
74
- def poll_future
75
- Concurrent.future do
76
- queue.receive_messages message_attribute_names: %w(All),
77
- wait_time_seconds: wait,
78
- max_number_of_messages: max_messages
79
- end
80
- end
81
-
82
- def poll_messages
83
- return unless polling.make_true
84
- poll_future.on_completion! do |success, msgs, error|
85
- polling.make_false
86
- scheduled.make_false
87
- if success && !msgs.nil? && !msgs.empty?
88
- tell [:assign_messages, msgs]
89
- else
90
- tell :schedule_poll
91
- end
92
- end
93
- end
94
-
95
- def schedule_poll
96
- return unless free_processors.value > 0 && scheduled.make_true
97
- debug "Fetcher #{queue.name} scheduling polling..."
98
- tell :poll_messages
99
- end
100
-
101
- def processor_pool
102
- @processor_pool ||= Toiler.processor_pool queue.name
103
- end
104
-
105
- def assign_messages(messages)
106
- messages = [messages] if batch?
107
- messages.each do |m|
108
- processor_pool.tell [:process, visibility_timeout, m]
109
- free_processors.decrement
110
- end
111
- debug "Fetcher #{queue.name} assigned #{messages.count} messages"
112
- tell :schedule_poll
113
- end
114
- end
115
- end
116
- end
1
+ require 'toiler/actor/utils/actor_logging'
2
+ require 'toiler/aws/queue'
3
+
4
+ module Toiler
5
+ module Actor
6
+ # Actor polling for messages only when processors are ready, otherwise idle
7
+ class Fetcher < Concurrent::Actor::RestartingContext
8
+ include Utils::ActorLogging
9
+
10
+ FETCH_LIMIT = 10
11
+
12
+ attr_accessor :queue, :wait, :visibility_timeout, :free_processors,
13
+ :scheduled, :executing, :polling
14
+
15
+ def initialize(queue, client)
16
+ debug "Initializing Fetcher for queue #{queue}..."
17
+ @queue = Toiler::Aws::Queue.new queue, client
18
+ @wait = Toiler.options[:wait] || 20
19
+ @free_processors = Concurrent::AtomicFixnum.new(0)
20
+ @batch = Toiler.worker_class_registry[queue].batch?
21
+ @visibility_timeout = @queue.visibility_timeout
22
+ @scheduled = Concurrent::AtomicBoolean.new
23
+ @executing = Concurrent::AtomicBoolean.new
24
+ @polling = Concurrent::AtomicBoolean.new
25
+ debug "Finished initializing Fetcher for queue #{queue}"
26
+ end
27
+
28
+ def default_executor
29
+ Concurrent.global_io_executor
30
+ end
31
+
32
+ def on_message(msg)
33
+ executing.make_true
34
+ method, *args = msg
35
+ send(method, *args)
36
+ rescue StandardError => e
37
+ error "Fetcher #{queue.name} raised exception #{e.class}: #{e.message}\n#{e.backtrace.join("\n")}"
38
+ ensure
39
+ executing.make_false
40
+ end
41
+
42
+ def executing?
43
+ executing.value
44
+ end
45
+
46
+ def polling?
47
+ polling.value
48
+ end
49
+
50
+ def scheduled?
51
+ scheduled.value
52
+ end
53
+
54
+ def get_free_processors
55
+ free_processors.value
56
+ end
57
+
58
+ private
59
+
60
+ def batch?
61
+ @batch
62
+ end
63
+
64
+ def processor_finished
65
+ debug "Fetcher #{queue.name} received processor finished signal..."
66
+ free_processors.increment
67
+ schedule_poll
68
+ end
69
+
70
+ def max_messages
71
+ batch? ? FETCH_LIMIT : [FETCH_LIMIT, free_processors.value].min
72
+ end
73
+
74
+ def poll_future
75
+ Concurrent.future do
76
+ queue.receive_messages message_attribute_names: %w(All),
77
+ wait_time_seconds: wait,
78
+ max_number_of_messages: max_messages
79
+ end
80
+ end
81
+
82
+ def poll_messages
83
+ return unless polling.make_true
84
+ poll_future.on_completion! do |success, msgs, error|
85
+ polling.make_false
86
+ scheduled.make_false
87
+ if success && !msgs.nil? && !msgs.empty?
88
+ tell [:assign_messages, msgs]
89
+ else
90
+ tell :schedule_poll
91
+ end
92
+ end
93
+ end
94
+
95
+ def schedule_poll
96
+ return unless free_processors.value > 0 && scheduled.make_true
97
+ debug "Fetcher #{queue.name} scheduling polling..."
98
+ tell :poll_messages
99
+ end
100
+
101
+ def processor_pool
102
+ @processor_pool ||= Toiler.processor_pool queue.name
103
+ end
104
+
105
+ def assign_messages(messages)
106
+ messages = [messages] if batch?
107
+ messages.each do |m|
108
+ processor_pool.tell [:process, visibility_timeout, m]
109
+ free_processors.decrement
110
+ end
111
+ debug "Fetcher #{queue.name} assigned #{messages.count} messages"
112
+ tell :schedule_poll
113
+ end
114
+ end
115
+ end
116
+ end
@@ -1,123 +1,123 @@
1
- require 'json'
2
- require 'toiler/actor/utils/actor_logging'
3
-
4
- module Toiler
5
- module Actor
6
- # Responsible for processing sqs messages and notifying Fetcher when done
7
- class Processor < Concurrent::Actor::RestartingContext
8
- include Utils::ActorLogging
9
-
10
- attr_accessor :queue, :worker, :fetcher, :body_parser,
11
- :extend_callback, :executing, :thread
12
-
13
- def initialize(queue)
14
- @queue = queue
15
- @worker = Toiler.worker_class_registry[queue].new
16
- @fetcher = Toiler.fetcher queue
17
- @executing = Concurrent::AtomicBoolean.new
18
- @thread = nil
19
- init_options
20
- processor_finished
21
- end
22
-
23
- def default_executor
24
- Concurrent.global_io_executor
25
- end
26
-
27
- def on_message(msg)
28
- method, *args = msg
29
- send(method, *args)
30
- rescue StandardError => e
31
- error "Processor #{queue} failed processing, reason: #{e.class}: #{e.message}\n#{e.backtrace.join("\n")}"
32
- end
33
-
34
- def executing?
35
- executing.value
36
- end
37
-
38
- private
39
-
40
- def init_options
41
- @auto_visibility_timeout = @worker.class.auto_visibility_timeout?
42
- @auto_delete = @worker.class.auto_delete?
43
- toiler_options = @worker.class.toiler_options
44
- @body_parser = toiler_options[:parser]
45
- @extend_callback = toiler_options[:on_visibility_extend]
46
- end
47
-
48
- def auto_visibility_timeout?
49
- @auto_visibility_timeout
50
- end
51
-
52
- def auto_delete?
53
- @auto_delete
54
- end
55
-
56
- def process(visibility, sqs_msg)
57
- process_init
58
- body = get_body(sqs_msg)
59
- timer = visibility_extender visibility, sqs_msg, body, &extend_callback
60
-
61
- debug "Worker #{queue} starts performing..."
62
- worker.perform sqs_msg, body
63
- debug "Worker #{queue} finishes performing..."
64
- sqs_msg.delete if auto_delete?
65
- ensure
66
- process_cleanup timer
67
- end
68
-
69
- def process_init
70
- @executing.make_true
71
- @thread = Thread.current
72
- debug "Processor #{queue} begins processing..."
73
- end
74
-
75
- def process_cleanup(timer)
76
- debug "Processor #{queue} starts cleanup after perform..."
77
- timer.shutdown if timer
78
- ::ActiveRecord::Base.clear_active_connections! if defined? ActiveRecord
79
- processor_finished
80
- @executing.make_false
81
- @thread = nil
82
- debug "Processor #{queue} finished cleanup after perform..."
83
- end
84
-
85
- def processor_finished
86
- fetcher.tell :processor_finished
87
- end
88
-
89
- def visibility_extender(queue_visibility, sqs_msg, body)
90
- return unless auto_visibility_timeout?
91
- interval = [1,queue_visibility/3].max
92
- Concurrent::TimerTask.execute execution_interval: interval,
93
- timeout_interval: interval do
94
- begin
95
- sqs_msg.visibility_timeout = queue_visibility
96
- yield sqs_msg, body if block_given?
97
- rescue StandardError => e
98
- error "Processor #{queue} failed to extend visibility of message: #{e.message}\n#{e.backtrace.join("\n")}"
99
- end
100
- end
101
- end
102
-
103
- def get_body(sqs_msg)
104
- if sqs_msg.is_a? Array
105
- sqs_msg.map { |m| parse_body m }
106
- else
107
- parse_body sqs_msg
108
- end
109
- end
110
-
111
- def parse_body(sqs_msg)
112
- case body_parser
113
- when :json then JSON.parse sqs_msg.body
114
- when Proc then body_parser.call sqs_msg
115
- when :text, nil then sqs_msg.body
116
- else body_parser.load sqs_msg.body
117
- end
118
- rescue => e
119
- raise "Error parsing the message body: #{e.message}"
120
- end
121
- end
122
- end
123
- end
1
+ require 'json'
2
+ require 'toiler/actor/utils/actor_logging'
3
+
4
+ module Toiler
5
+ module Actor
6
+ # Responsible for processing sqs messages and notifying Fetcher when done
7
+ class Processor < Concurrent::Actor::RestartingContext
8
+ include Utils::ActorLogging
9
+
10
+ attr_accessor :queue, :worker, :fetcher, :body_parser,
11
+ :extend_callback, :executing, :thread
12
+
13
+ def initialize(queue)
14
+ @queue = queue
15
+ @worker = Toiler.worker_class_registry[queue].new
16
+ @fetcher = Toiler.fetcher queue
17
+ @executing = Concurrent::AtomicBoolean.new
18
+ @thread = nil
19
+ init_options
20
+ processor_finished
21
+ end
22
+
23
+ def default_executor
24
+ Concurrent.global_io_executor
25
+ end
26
+
27
+ def on_message(msg)
28
+ method, *args = msg
29
+ send(method, *args)
30
+ rescue StandardError => e
31
+ error "Processor #{queue} failed processing, reason: #{e.class}: #{e.message}\n#{e.backtrace.join("\n")}"
32
+ end
33
+
34
+ def executing?
35
+ executing.value
36
+ end
37
+
38
+ private
39
+
40
+ def init_options
41
+ @auto_visibility_timeout = @worker.class.auto_visibility_timeout?
42
+ @auto_delete = @worker.class.auto_delete?
43
+ toiler_options = @worker.class.toiler_options
44
+ @body_parser = toiler_options[:parser]
45
+ @extend_callback = toiler_options[:on_visibility_extend]
46
+ end
47
+
48
+ def auto_visibility_timeout?
49
+ @auto_visibility_timeout
50
+ end
51
+
52
+ def auto_delete?
53
+ @auto_delete
54
+ end
55
+
56
+ def process(visibility, sqs_msg)
57
+ process_init
58
+ body = get_body(sqs_msg)
59
+ timer = visibility_extender visibility, sqs_msg, body, &extend_callback
60
+
61
+ debug "Worker #{queue} starts performing..."
62
+ worker.perform sqs_msg, body
63
+ debug "Worker #{queue} finishes performing..."
64
+ sqs_msg.delete if auto_delete?
65
+ ensure
66
+ process_cleanup timer
67
+ end
68
+
69
+ def process_init
70
+ @executing.make_true
71
+ @thread = Thread.current
72
+ debug "Processor #{queue} begins processing..."
73
+ end
74
+
75
+ def process_cleanup(timer)
76
+ debug "Processor #{queue} starts cleanup after perform..."
77
+ timer.shutdown if timer
78
+ ::ActiveRecord::Base.clear_active_connections! if defined? ActiveRecord
79
+ processor_finished
80
+ @executing.make_false
81
+ @thread = nil
82
+ debug "Processor #{queue} finished cleanup after perform..."
83
+ end
84
+
85
+ def processor_finished
86
+ fetcher.tell :processor_finished
87
+ end
88
+
89
+ def visibility_extender(queue_visibility, sqs_msg, body)
90
+ return unless auto_visibility_timeout?
91
+ interval = [1,queue_visibility/3].max
92
+ Concurrent::TimerTask.execute execution_interval: interval,
93
+ timeout_interval: interval do
94
+ begin
95
+ sqs_msg.visibility_timeout = queue_visibility
96
+ yield sqs_msg, body if block_given?
97
+ rescue StandardError => e
98
+ error "Processor #{queue} failed to extend visibility of message: #{e.message}\n#{e.backtrace.join("\n")}"
99
+ end
100
+ end
101
+ end
102
+
103
+ def get_body(sqs_msg)
104
+ if sqs_msg.is_a? Array
105
+ sqs_msg.map { |m| parse_body m }
106
+ else
107
+ parse_body sqs_msg
108
+ end
109
+ end
110
+
111
+ def parse_body(sqs_msg)
112
+ case body_parser
113
+ when :json then JSON.parse sqs_msg.body
114
+ when Proc then body_parser.call sqs_msg
115
+ when :text, nil then sqs_msg.body
116
+ else body_parser.load sqs_msg.body
117
+ end
118
+ rescue => e
119
+ raise "Error parsing the message body: #{e.message}"
120
+ end
121
+ end
122
+ end
123
+ end