herdst_worker 0.1.4 → 0.1.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (15) hide show
  1. checksums.yaml +4 -4
  2. data/lib/herdst_worker/adapters/database.rb +0 -1
  3. data/lib/herdst_worker/adapters/facade.rb +20 -20
  4. data/lib/herdst_worker/adapters/sentry.rb +1 -1
  5. data/lib/herdst_worker/application/facade.rb +98 -89
  6. data/lib/herdst_worker/autoload/facade.rb +67 -67
  7. data/lib/herdst_worker/configuration/metadata.rb +5 -1
  8. data/lib/herdst_worker/configuration/paths.rb +41 -41
  9. data/lib/herdst_worker/log/facade.rb +87 -87
  10. data/lib/herdst_worker/queue/facade.rb +164 -162
  11. data/lib/herdst_worker/queue/processor.rb +225 -223
  12. data/lib/herdst_worker/queue/runner.rb +137 -137
  13. data/lib/herdst_worker/signals/facade.rb +35 -35
  14. data/lib/herdst_worker/version.rb +1 -1
  15. metadata +6 -6
data/lib/herdst_worker/queue/processor.rb CHANGED
@@ -1,223 +1,225 @@
1
- require 'aws-sdk-sqs'
2
-
3
- require_relative 'runner'
4
-
5
-
6
- module HerdstWorker
7
- module Queue
8
- class Processor < Runner
9
-
10
-
11
- attr_accessor :app, :enabled, :queue_url, :queue_wait_time, :poller
12
- attr_accessor :start_time, :restart_time
13
- attr_accessor :processor_status, :job_count, :max_jobs
14
- attr_accessor :attempt_threshold, :visibility_timeout, :ignored_notifications
15
-
16
-
17
- def initialize(app, enabled, queue_url, queue_wait_time)
18
- self.app = app
19
- self.enabled = enabled
20
- self.queue_url = queue_url
21
- self.queue_wait_time = queue_wait_time
22
- self.poller = Aws::SQS::QueuePoller.new(queue_url)
23
- self.job_count = 0
24
- self.max_jobs = 10
25
- self.attempt_threshold = 6
26
- self.visibility_timeout = 15
27
- self.ignored_notifications = [
28
- "AmazonSnsSubscriptionSucceeded"
29
- ]
30
-
31
- # Set the start time
32
- self.reset_time
33
-
34
- # Start the processor as working
35
- self.set_status "starting"
36
-
37
- # Log queue stats
38
- self.poller.before_request do |stats|
39
- before_request(stats)
40
- end
41
- end
42
-
43
-
44
- # Runs the poller
45
- def start_poller
46
- if self.enabled
47
- self.poller.poll(:wait_time_seconds => self.queue_wait_time, :skip_delete => false) do |msg|
48
- process_message(msg)
49
- end
50
- end
51
- end
52
-
53
-
54
- # Starts or resets the application to a working status
55
- def start
56
- if self.processor_status == "starting"
57
- self.set_status "working"
58
- self.reset_time
59
- self.start_poller
60
- else
61
- return if self.processor_status == "working"
62
-
63
- self.set_status "working"
64
- self.reset_time
65
- end
66
- end
67
-
68
-
69
- # Sets the processor status to finishing. The sqs before action will
70
- # take care of setting the idle state once all jobs have finished.
71
- def halt
72
- return if self.processor_status === "finishing"
73
- set_status "finishing"
74
- end
75
-
76
-
77
- # Sets the processor status to stopping. The sqs before action will
78
- # take care of stopping the application once all jobs have finished.
79
- def stop
80
- return if self.processor_status == "stopping"
81
- set_status "stopping"
82
- end
83
-
84
-
85
- # Set the processor status. The status is alos logged to file so services
86
- # like capastranio can see the current status
87
- def set_status(status)
88
- statuses = ["starting", "idle", "working", "finishing", "stopping", "stopped"]
89
-
90
- if statuses.include? status
91
- # Set status
92
- self.processor_status = status
93
-
94
- # Write the current status to file for capastranio to use
95
- process_file = self.app.config.paths.temp + "/process_status"
96
- File.open(process_file, "w") { |file| file.write(status) }
97
- else
98
- raise "Invalid status (#{status})"
99
- end
100
- end
101
-
102
-
103
- def before_request(stats)
104
- if self.app.config.is_dev?
105
- self.app.logger.queue_stats.info "STATS (#{self.processor_status}): #{stats.inspect}"
106
- end
107
-
108
- # After 1 hour of running terminate application.
109
- # The app will automatically restart in production
110
- current_time = Time.now.utc.to_i
111
- if (self.processor_status == "working") && (current_time >= self.restart_time)
112
- runtime = current_time - self.start_time
113
- self.app.logger.queue.info "Stopping after #{runtime} seconds of work"
114
- set_status "stopping"
115
-
116
- # On finishing wait for jobs to complete and then set status
117
- # to idle
118
- elsif self.processor_status == "finishing"
119
- if self.job_count == 0
120
- self.app.logger.queue.info "Setting processor status to idle"
121
- set_status "idle"
122
- end
123
-
124
- # On stopping wait for jobs to complete and then set status
125
- # to stopped. Once stopped the polling will terminate.
126
- elsif self.processor_status == "stopping"
127
- if self.job_count == 0
128
- self.app.logger.queue.info "Setting processor status to stopped"
129
- set_status "stopped"
130
- end
131
-
132
- end
133
-
134
- if self.processor_status == "stopped"
135
- self.app.logger.queue.info "Exiting program, Service requested to stop"
136
- throw :stop_polling
137
- end
138
- end
139
-
140
-
141
- def process_message(msg)
142
- if self.processor_status == "working"
143
- # If the app is already processing the max number of jobs
144
- # put the message back in the queue with a short wait time
145
- if self.job_count >= self.max_jobs
146
- self.poller.change_message_visibility_timeout(msg, self.visibility_timeout)
147
- throw :skip_delete
148
- end
149
-
150
- # Find out how many attempts there has been already for
151
- # the message.
152
- msg_attrs = msg.message_attributes.dup
153
- attempt_number = msg_attrs.include?("attempts") ? msg_attrs["attempts"]["string_value"].to_i + 1 : 1
154
- will_fail_permanently = attempt_number > self.attempt_threshold
155
-
156
- # Run the job and increase the job count
157
- # Once successful the job count is decreased by one
158
- # and the message is deleted.
159
- # If an error occured the job count is decreased by
160
- # one and the error is logged locally and with sentry
161
- self.job_count += 1
162
- message = JSON.parse(msg.body)
163
- process_message!(message, msg, will_fail_permanently).then {
164
- self.job_count -= 1
165
-
166
- }.rescue { |ex|
167
- if will_fail_permanently
168
- self.app.logger.queue.error "Message failed #{attempt_number} times, Reporting and failing permanently. \n#{ex.to_s} \n#{ex.backtrace.join("\n")}"
169
- Raven.capture_exception(ex, {
170
- :level => "fatal",
171
- :extra => {
172
- "queue_attempts" => attempt_number,
173
- "queue_message_body" => msg.body
174
- }
175
- })
176
-
177
- else
178
- self.app.logger.queue.error "Message failed #{attempt_number} times, Adding back to queue."
179
-
180
- if self.app.config.is_dev?
181
- puts ex.inspect
182
- puts ex.backtrace
183
- end
184
-
185
- replaced_message = {
186
- :queue_url => self.poller.queue_url,
187
- :message_body => msg.body,
188
- :delay_seconds => self.visibility_timeout,
189
- :message_attributes => msg_attrs.merge({
190
- "attempts" => {
191
- :string_value => attempt_number.to_s,
192
- :data_type => "Number"
193
- }
194
- })
195
- }
196
-
197
- self.poller.client.send_message replaced_message
198
- end
199
-
200
- if self.app.config.is_dev?
201
- self.app.logger.queue.error "Processor Error:"
202
- self.app.logger.queue.error ex.message
203
- self.app.logger.queue.error ex.backtrace
204
- end
205
-
206
- self.job_count -= 1
207
- }.execute
208
- else
209
- self.poller.change_message_visibility_timeout(msg, self.visibility_timeout * 2)
210
- throw :skip_delete
211
- end
212
- end
213
-
214
-
215
- private
216
- def reset_time
217
- self.start_time = Time.now.utc.to_i
218
- self.restart_time = self.start_time + (3600) # One hour
219
- end
220
-
221
- end
222
- end
223
- end
1
+ require 'aws-sdk-sqs'
2
+
3
+ require_relative 'runner'
4
+
5
+
6
+ module HerdstWorker
7
+ module Queue
8
+ class Processor < Runner
9
+
10
+
11
+ attr_accessor :app, :enabled, :queue_url, :queue_wait_time, :poller
12
+ attr_accessor :start_time, :restart_time
13
+ attr_accessor :processor_status, :job_count, :max_jobs
14
+ attr_accessor :attempt_threshold, :visibility_timeout, :ignored_notifications
15
+
16
+
17
+ def initialize(app, enabled, queue_url, queue_wait_time)
18
+ self.app = app
19
+ self.enabled = enabled
20
+ self.queue_url = queue_url
21
+ self.queue_wait_time = queue_wait_time
22
+ self.poller = Aws::SQS::QueuePoller.new(queue_url)
23
+ self.job_count = 0
24
+ self.max_jobs = 10
25
+ self.attempt_threshold = 6
26
+ self.visibility_timeout = 15
27
+ self.ignored_notifications = [
28
+ "AmazonSnsSubscriptionSucceeded"
29
+ ]
30
+
31
+ # Set the start time
32
+ self.reset_time
33
+
34
+ # Start the processor as working
35
+ self.set_status "starting"
36
+
37
+ # Log queue stats
38
+ self.poller.before_request do |stats|
39
+ before_request(stats)
40
+ end
41
+ end
42
+
43
+
44
+ # Runs the poller
45
+ def start_poller
46
+ if self.enabled
47
+ self.poller.poll(:wait_time_seconds => self.queue_wait_time, :skip_delete => false) do |msg|
48
+ process_message(msg)
49
+ end
50
+ else
51
+ raise "Cannot start a queue which is not enabled"
52
+ end
53
+ end
54
+
55
+
56
+ # Starts or resets the application to a working status
57
+ def start
58
+ if self.processor_status == "starting"
59
+ self.set_status "working"
60
+ self.reset_time
61
+ self.start_poller
62
+ else
63
+ return if self.processor_status == "working"
64
+
65
+ self.set_status "working"
66
+ self.reset_time
67
+ end
68
+ end
69
+
70
+
71
+ # Sets the processor status to finishing. The sqs before action will
72
+ # take care of setting the idle state once all jobs have finished.
73
+ def halt
74
+ return if self.processor_status === "finishing"
75
+ set_status "finishing"
76
+ end
77
+
78
+
79
+ # Sets the processor status to stopping. The sqs before action will
80
+ # take care of stopping the application once all jobs have finished.
81
+ def stop
82
+ return if self.processor_status == "stopping"
83
+ set_status "stopping"
84
+ end
85
+
86
+
87
+ # Set the processor status. The status is alos logged to file so services
88
+ # like capastranio can see the current status
89
+ def set_status(status)
90
+ statuses = ["starting", "idle", "working", "finishing", "stopping", "stopped"]
91
+
92
+ if statuses.include? status
93
+ # Set status
94
+ self.processor_status = status
95
+
96
+ # Write the current status to file for capastranio to use
97
+ process_file = self.app.config.paths.temp + "/process_status"
98
+ File.open(process_file, "w") { |file| file.write(status) }
99
+ else
100
+ raise "Invalid status (#{status})"
101
+ end
102
+ end
103
+
104
+
105
+ def before_request(stats)
106
+ if self.app.config.is_dev?
107
+ self.app.logger.queue_stats.info "STATS (#{self.processor_status}): #{stats.inspect}"
108
+ end
109
+
110
+ # After 1 hour of running terminate application.
111
+ # The app will automatically restart in production
112
+ current_time = Time.now.utc.to_i
113
+ if (self.processor_status == "working") && (current_time >= self.restart_time)
114
+ runtime = current_time - self.start_time
115
+ self.app.logger.queue.info "Stopping after #{runtime} seconds of work"
116
+ set_status "stopping"
117
+
118
+ # On finishing wait for jobs to complete and then set status
119
+ # to idle
120
+ elsif self.processor_status == "finishing"
121
+ if self.job_count == 0
122
+ self.app.logger.queue.info "Setting processor status to idle"
123
+ set_status "idle"
124
+ end
125
+
126
+ # On stopping wait for jobs to complete and then set status
127
+ # to stopped. Once stopped the polling will terminate.
128
+ elsif self.processor_status == "stopping"
129
+ if self.job_count == 0
130
+ self.app.logger.queue.info "Setting processor status to stopped"
131
+ set_status "stopped"
132
+ end
133
+
134
+ end
135
+
136
+ if self.processor_status == "stopped"
137
+ self.app.logger.queue.info "Exiting program, Service requested to stop"
138
+ throw :stop_polling
139
+ end
140
+ end
141
+
142
+
143
+ def process_message(msg)
144
+ if self.processor_status == "working"
145
+ # If the app is already processing the max number of jobs
146
+ # put the message back in the queue with a short wait time
147
+ if self.job_count >= self.max_jobs
148
+ self.poller.change_message_visibility_timeout(msg, self.visibility_timeout)
149
+ throw :skip_delete
150
+ end
151
+
152
+ # Find out how many attempts there has been already for
153
+ # the message.
154
+ msg_attrs = msg.message_attributes.dup
155
+ attempt_number = msg_attrs.include?("attempts") ? msg_attrs["attempts"]["string_value"].to_i + 1 : 1
156
+ will_fail_permanently = attempt_number > self.attempt_threshold
157
+
158
+ # Run the job and increase the job count
159
+ # Once successful the job count is decreased by one
160
+ # and the message is deleted.
161
+ # If an error occured the job count is decreased by
162
+ # one and the error is logged locally and with sentry
163
+ self.job_count += 1
164
+ message = JSON.parse(msg.body)
165
+ process_message!(message, msg, will_fail_permanently).then {
166
+ self.job_count -= 1
167
+
168
+ }.rescue { |ex|
169
+ if will_fail_permanently
170
+ self.app.logger.queue.error "Message failed #{attempt_number} times, Reporting and failing permanently. \n#{ex.to_s} \n#{ex.backtrace.join("\n")}"
171
+ Raven.capture_exception(ex, {
172
+ :level => "fatal",
173
+ :extra => {
174
+ "queue_attempts" => attempt_number,
175
+ "queue_message_body" => msg.body
176
+ }
177
+ })
178
+
179
+ else
180
+ self.app.logger.queue.error "Message failed #{attempt_number} times, Adding back to queue."
181
+
182
+ if self.app.config.is_dev?
183
+ puts ex.inspect
184
+ puts ex.backtrace
185
+ end
186
+
187
+ replaced_message = {
188
+ :queue_url => self.poller.queue_url,
189
+ :message_body => msg.body,
190
+ :delay_seconds => self.visibility_timeout,
191
+ :message_attributes => msg_attrs.merge({
192
+ "attempts" => {
193
+ :string_value => attempt_number.to_s,
194
+ :data_type => "Number"
195
+ }
196
+ })
197
+ }
198
+
199
+ self.poller.client.send_message replaced_message
200
+ end
201
+
202
+ if self.app.config.is_dev?
203
+ self.app.logger.queue.error "Processor Error:"
204
+ self.app.logger.queue.error ex.message
205
+ self.app.logger.queue.error ex.backtrace
206
+ end
207
+
208
+ self.job_count -= 1
209
+ }.execute
210
+ else
211
+ self.poller.change_message_visibility_timeout(msg, self.visibility_timeout * 2)
212
+ throw :skip_delete
213
+ end
214
+ end
215
+
216
+
217
+ private
218
+ def reset_time
219
+ self.start_time = Time.now.utc.to_i
220
+ self.restart_time = self.start_time + (3600) # One hour
221
+ end
222
+
223
+ end
224
+ end
225
+ end