herdst_worker 0.1.6 → 0.1.9

Sign up to get free protection for your applications and to get access to all the features.
Files changed (14) hide show
  1. checksums.yaml +4 -4
  2. data/bin/herdst_worker_console +24 -0
  3. data/lib/herdst_worker/adapters/database.rb +6 -12
  4. data/lib/herdst_worker/adapters/facade.rb +20 -20
  5. data/lib/herdst_worker/application/facade.rb +98 -89
  6. data/lib/herdst_worker/autoload/facade.rb +67 -67
  7. data/lib/herdst_worker/configuration/paths.rb +41 -41
  8. data/lib/herdst_worker/log/facade.rb +87 -87
  9. data/lib/herdst_worker/queue/facade.rb +164 -162
  10. data/lib/herdst_worker/queue/processor.rb +225 -223
  11. data/lib/herdst_worker/queue/runner.rb +137 -137
  12. data/lib/herdst_worker/signals/facade.rb +35 -35
  13. data/lib/herdst_worker/version.rb +1 -1
  14. metadata +8 -6
data/lib/herdst_worker/queue/processor.rb CHANGED
@@ -1,223 +1,225 @@
1
- require 'aws-sdk-sqs'
2
-
3
- require_relative 'runner'
4
-
5
-
6
- module HerdstWorker
7
- module Queue
8
- class Processor < Runner
9
-
10
-
11
- attr_accessor :app, :enabled, :queue_url, :queue_wait_time, :poller
12
- attr_accessor :start_time, :restart_time
13
- attr_accessor :processor_status, :job_count, :max_jobs
14
- attr_accessor :attempt_threshold, :visibility_timeout, :ignored_notifications
15
-
16
-
17
- def initialize(app, enabled, queue_url, queue_wait_time)
18
- self.app = app
19
- self.enabled = enabled
20
- self.queue_url = queue_url
21
- self.queue_wait_time = queue_wait_time
22
- self.poller = Aws::SQS::QueuePoller.new(queue_url)
23
- self.job_count = 0
24
- self.max_jobs = 10
25
- self.attempt_threshold = 6
26
- self.visibility_timeout = 15
27
- self.ignored_notifications = [
28
- "AmazonSnsSubscriptionSucceeded"
29
- ]
30
-
31
- # Set the start time
32
- self.reset_time
33
-
34
- # Start the processor as working
35
- self.set_status "starting"
36
-
37
- # Log queue stats
38
- self.poller.before_request do |stats|
39
- before_request(stats)
40
- end
41
- end
42
-
43
-
44
- # Runs the poller
45
- def start_poller
46
- if self.enabled
47
- self.poller.poll(:wait_time_seconds => self.queue_wait_time, :skip_delete => false) do |msg|
48
- process_message(msg)
49
- end
50
- end
51
- end
52
-
53
-
54
- # Starts or resets the application to a working status
55
- def start
56
- if self.processor_status == "starting"
57
- self.set_status "working"
58
- self.reset_time
59
- self.start_poller
60
- else
61
- return if self.processor_status == "working"
62
-
63
- self.set_status "working"
64
- self.reset_time
65
- end
66
- end
67
-
68
-
69
- # Sets the processor status to finishing. The sqs before action will
70
- # take care of setting the idle state once all jobs have finished.
71
- def halt
72
- return if self.processor_status === "finishing"
73
- set_status "finishing"
74
- end
75
-
76
-
77
- # Sets the processor status to stopping. The sqs before action will
78
- # take care of stopping the application once all jobs have finished.
79
- def stop
80
- return if self.processor_status == "stopping"
81
- set_status "stopping"
82
- end
83
-
84
-
85
- # Set the processor status. The status is alos logged to file so services
86
- # like capastranio can see the current status
87
- def set_status(status)
88
- statuses = ["starting", "idle", "working", "finishing", "stopping", "stopped"]
89
-
90
- if statuses.include? status
91
- # Set status
92
- self.processor_status = status
93
-
94
- # Write the current status to file for capastranio to use
95
- process_file = self.app.config.paths.temp + "/process_status"
96
- File.open(process_file, "w") { |file| file.write(status) }
97
- else
98
- raise "Invalid status (#{status})"
99
- end
100
- end
101
-
102
-
103
- def before_request(stats)
104
- if self.app.config.is_dev?
105
- self.app.logger.queue_stats.info "STATS (#{self.processor_status}): #{stats.inspect}"
106
- end
107
-
108
- # After 1 hour of running terminate application.
109
- # The app will automatically restart in production
110
- current_time = Time.now.utc.to_i
111
- if (self.processor_status == "working") && (current_time >= self.restart_time)
112
- runtime = current_time - self.start_time
113
- self.app.logger.queue.info "Stopping after #{runtime} seconds of work"
114
- set_status "stopping"
115
-
116
- # On finishing wait for jobs to complete and then set status
117
- # to idle
118
- elsif self.processor_status == "finishing"
119
- if self.job_count == 0
120
- self.app.logger.queue.info "Setting processor status to idle"
121
- set_status "idle"
122
- end
123
-
124
- # On stopping wait for jobs to complete and then set status
125
- # to stopped. Once stopped the polling will terminate.
126
- elsif self.processor_status == "stopping"
127
- if self.job_count == 0
128
- self.app.logger.queue.info "Setting processor status to stopped"
129
- set_status "stopped"
130
- end
131
-
132
- end
133
-
134
- if self.processor_status == "stopped"
135
- self.app.logger.queue.info "Exiting program, Service requested to stop"
136
- throw :stop_polling
137
- end
138
- end
139
-
140
-
141
- def process_message(msg)
142
- if self.processor_status == "working"
143
- # If the app is already processing the max number of jobs
144
- # put the message back in the queue with a short wait time
145
- if self.job_count >= self.max_jobs
146
- self.poller.change_message_visibility_timeout(msg, self.visibility_timeout)
147
- throw :skip_delete
148
- end
149
-
150
- # Find out how many attempts there has been already for
151
- # the message.
152
- msg_attrs = msg.message_attributes.dup
153
- attempt_number = msg_attrs.include?("attempts") ? msg_attrs["attempts"]["string_value"].to_i + 1 : 1
154
- will_fail_permanently = attempt_number > self.attempt_threshold
155
-
156
- # Run the job and increase the job count
157
- # Once successful the job count is decreased by one
158
- # and the message is deleted.
159
- # If an error occured the job count is decreased by
160
- # one and the error is logged locally and with sentry
161
- self.job_count += 1
162
- message = JSON.parse(msg.body)
163
- process_message!(message, msg, will_fail_permanently).then {
164
- self.job_count -= 1
165
-
166
- }.rescue { |ex|
167
- if will_fail_permanently
168
- self.app.logger.queue.error "Message failed #{attempt_number} times, Reporting and failing permanently. \n#{ex.to_s} \n#{ex.backtrace.join("\n")}"
169
- Raven.capture_exception(ex, {
170
- :level => "fatal",
171
- :extra => {
172
- "queue_attempts" => attempt_number,
173
- "queue_message_body" => msg.body
174
- }
175
- })
176
-
177
- else
178
- self.app.logger.queue.error "Message failed #{attempt_number} times, Adding back to queue."
179
-
180
- if self.app.config.is_dev?
181
- puts ex.inspect
182
- puts ex.backtrace
183
- end
184
-
185
- replaced_message = {
186
- :queue_url => self.poller.queue_url,
187
- :message_body => msg.body,
188
- :delay_seconds => self.visibility_timeout,
189
- :message_attributes => msg_attrs.merge({
190
- "attempts" => {
191
- :string_value => attempt_number.to_s,
192
- :data_type => "Number"
193
- }
194
- })
195
- }
196
-
197
- self.poller.client.send_message replaced_message
198
- end
199
-
200
- if self.app.config.is_dev?
201
- self.app.logger.queue.error "Processor Error:"
202
- self.app.logger.queue.error ex.message
203
- self.app.logger.queue.error ex.backtrace
204
- end
205
-
206
- self.job_count -= 1
207
- }.execute
208
- else
209
- self.poller.change_message_visibility_timeout(msg, self.visibility_timeout * 2)
210
- throw :skip_delete
211
- end
212
- end
213
-
214
-
215
- private
216
- def reset_time
217
- self.start_time = Time.now.utc.to_i
218
- self.restart_time = self.start_time + (3600) # One hour
219
- end
220
-
221
- end
222
- end
223
- end
1
+ require 'aws-sdk-sqs'
2
+
3
+ require_relative 'runner'
4
+
5
+
6
+ module HerdstWorker
7
+ module Queue
8
+ class Processor < Runner
9
+
10
+
11
+ attr_accessor :app, :enabled, :queue_url, :queue_wait_time, :poller
12
+ attr_accessor :start_time, :restart_time
13
+ attr_accessor :processor_status, :job_count, :max_jobs
14
+ attr_accessor :attempt_threshold, :visibility_timeout, :ignored_notifications
15
+
16
+
17
+ def initialize(app, enabled, queue_url, queue_wait_time)
18
+ self.app = app
19
+ self.enabled = enabled
20
+ self.queue_url = queue_url
21
+ self.queue_wait_time = queue_wait_time
22
+ self.poller = Aws::SQS::QueuePoller.new(queue_url)
23
+ self.job_count = 0
24
+ self.max_jobs = 10
25
+ self.attempt_threshold = 6
26
+ self.visibility_timeout = 15
27
+ self.ignored_notifications = [
28
+ "AmazonSnsSubscriptionSucceeded"
29
+ ]
30
+
31
+ # Set the start time
32
+ self.reset_time
33
+
34
+ # Start the processor as working
35
+ self.set_status "starting"
36
+
37
+ # Log queue stats
38
+ self.poller.before_request do |stats|
39
+ before_request(stats)
40
+ end
41
+ end
42
+
43
+
44
+ # Runs the poller
45
+ def start_poller
46
+ if self.enabled
47
+ self.poller.poll(:wait_time_seconds => self.queue_wait_time, :skip_delete => false) do |msg|
48
+ process_message(msg)
49
+ end
50
+ else
51
+ raise "Cannot start a queue which is not enabled"
52
+ end
53
+ end
54
+
55
+
56
+ # Starts or resets the application to a working status
57
+ def start
58
+ if self.processor_status == "starting"
59
+ self.set_status "working"
60
+ self.reset_time
61
+ self.start_poller
62
+ else
63
+ return if self.processor_status == "working"
64
+
65
+ self.set_status "working"
66
+ self.reset_time
67
+ end
68
+ end
69
+
70
+
71
+ # Sets the processor status to finishing. The sqs before action will
72
+ # take care of setting the idle state once all jobs have finished.
73
+ def halt
74
+ return if self.processor_status === "finishing"
75
+ set_status "finishing"
76
+ end
77
+
78
+
79
+ # Sets the processor status to stopping. The sqs before action will
80
+ # take care of stopping the application once all jobs have finished.
81
+ def stop
82
+ return if self.processor_status == "stopping"
83
+ set_status "stopping"
84
+ end
85
+
86
+
87
+ # Set the processor status. The status is alos logged to file so services
88
+ # like capastranio can see the current status
89
+ def set_status(status)
90
+ statuses = ["starting", "idle", "working", "finishing", "stopping", "stopped"]
91
+
92
+ if statuses.include? status
93
+ # Set status
94
+ self.processor_status = status
95
+
96
+ # Write the current status to file for capastranio to use
97
+ process_file = self.app.config.paths.temp + "/process_status"
98
+ File.open(process_file, "w") { |file| file.write(status) }
99
+ else
100
+ raise "Invalid status (#{status})"
101
+ end
102
+ end
103
+
104
+
105
+ def before_request(stats)
106
+ if self.app.config.is_dev?
107
+ self.app.logger.queue_stats.info "STATS (#{self.processor_status}): #{stats.inspect}"
108
+ end
109
+
110
+ # After 1 hour of running terminate application.
111
+ # The app will automatically restart in production
112
+ current_time = Time.now.utc.to_i
113
+ if (self.processor_status == "working") && (current_time >= self.restart_time)
114
+ runtime = current_time - self.start_time
115
+ self.app.logger.queue.info "Stopping after #{runtime} seconds of work"
116
+ set_status "stopping"
117
+
118
+ # On finishing wait for jobs to complete and then set status
119
+ # to idle
120
+ elsif self.processor_status == "finishing"
121
+ if self.job_count == 0
122
+ self.app.logger.queue.info "Setting processor status to idle"
123
+ set_status "idle"
124
+ end
125
+
126
+ # On stopping wait for jobs to complete and then set status
127
+ # to stopped. Once stopped the polling will terminate.
128
+ elsif self.processor_status == "stopping"
129
+ if self.job_count == 0
130
+ self.app.logger.queue.info "Setting processor status to stopped"
131
+ set_status "stopped"
132
+ end
133
+
134
+ end
135
+
136
+ if self.processor_status == "stopped"
137
+ self.app.logger.queue.info "Exiting program, Service requested to stop"
138
+ throw :stop_polling
139
+ end
140
+ end
141
+
142
+
143
+ def process_message(msg)
144
+ if self.processor_status == "working"
145
+ # If the app is already processing the max number of jobs
146
+ # put the message back in the queue with a short wait time
147
+ if self.job_count >= self.max_jobs
148
+ self.poller.change_message_visibility_timeout(msg, self.visibility_timeout)
149
+ throw :skip_delete
150
+ end
151
+
152
+ # Find out how many attempts there has been already for
153
+ # the message.
154
+ msg_attrs = msg.message_attributes.dup
155
+ attempt_number = msg_attrs.include?("attempts") ? msg_attrs["attempts"]["string_value"].to_i + 1 : 1
156
+ will_fail_permanently = attempt_number > self.attempt_threshold
157
+
158
+ # Run the job and increase the job count
159
+ # Once successful the job count is decreased by one
160
+ # and the message is deleted.
161
+ # If an error occured the job count is decreased by
162
+ # one and the error is logged locally and with sentry
163
+ self.job_count += 1
164
+ message = JSON.parse(msg.body)
165
+ process_message!(message, msg, will_fail_permanently).then {
166
+ self.job_count -= 1
167
+
168
+ }.rescue { |ex|
169
+ if will_fail_permanently
170
+ self.app.logger.queue.error "Message failed #{attempt_number} times, Reporting and failing permanently. \n#{ex.to_s} \n#{ex.backtrace.join("\n")}"
171
+ Raven.capture_exception(ex, {
172
+ :level => "fatal",
173
+ :extra => {
174
+ "queue_attempts" => attempt_number,
175
+ "queue_message_body" => msg.body
176
+ }
177
+ })
178
+
179
+ else
180
+ self.app.logger.queue.error "Message failed #{attempt_number} times, Adding back to queue."
181
+
182
+ if self.app.config.is_dev?
183
+ puts ex.inspect
184
+ puts ex.backtrace
185
+ end
186
+
187
+ replaced_message = {
188
+ :queue_url => self.poller.queue_url,
189
+ :message_body => msg.body,
190
+ :delay_seconds => self.visibility_timeout,
191
+ :message_attributes => msg_attrs.merge({
192
+ "attempts" => {
193
+ :string_value => attempt_number.to_s,
194
+ :data_type => "Number"
195
+ }
196
+ })
197
+ }
198
+
199
+ self.poller.client.send_message replaced_message
200
+ end
201
+
202
+ if self.app.config.is_dev?
203
+ self.app.logger.queue.error "Processor Error:"
204
+ self.app.logger.queue.error ex.message
205
+ self.app.logger.queue.error ex.backtrace
206
+ end
207
+
208
+ self.job_count -= 1
209
+ }.execute
210
+ else
211
+ self.poller.change_message_visibility_timeout(msg, self.visibility_timeout * 2)
212
+ throw :skip_delete
213
+ end
214
+ end
215
+
216
+
217
+ private
218
+ def reset_time
219
+ self.start_time = Time.now.utc.to_i
220
+ self.restart_time = self.start_time + (3600) # One hour
221
+ end
222
+
223
+ end
224
+ end
225
+ end