herdst_worker 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/Rakefile +7 -0
- data/bin/herdst_worker +16 -0
- data/lib/herdst_worker.rb +8 -0
- data/lib/herdst_worker/adapters/database.rb +31 -0
- data/lib/herdst_worker/adapters/facade.rb +20 -0
- data/lib/herdst_worker/adapters/sentry.rb +26 -0
- data/lib/herdst_worker/application/facade.rb +88 -0
- data/lib/herdst_worker/autoload/facade.rb +67 -0
- data/lib/herdst_worker/configuration/facade.rb +51 -0
- data/lib/herdst_worker/configuration/metadata.rb +183 -0
- data/lib/herdst_worker/configuration/paths.rb +41 -0
- data/lib/herdst_worker/log/facade.rb +87 -0
- data/lib/herdst_worker/queue/facade.rb +162 -0
- data/lib/herdst_worker/queue/processor.rb +223 -0
- data/lib/herdst_worker/queue/runner.rb +137 -0
- data/lib/herdst_worker/signals/facade.rb +35 -0
- data/lib/herdst_worker/version.rb +3 -0
- metadata +272 -0
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
require 'concurrent'
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
module HerdstWorker
|
|
5
|
+
module Queue
|
|
6
|
+
class Runner
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def process_message!(message, raw_message = nil, will_fail_permanently = false)
|
|
10
|
+
sent_timestamp = raw_message.attributes.include?("SentTimestamp") ?
|
|
11
|
+
raw_message.attributes["SentTimestamp"].to_i :
|
|
12
|
+
nil
|
|
13
|
+
trigger_timestamp = raw_message.message_attributes.include?("triggerTimestamp") ?
|
|
14
|
+
raw_message.message_attributes["triggerTimestamp"]["string_value"].to_i :
|
|
15
|
+
nil
|
|
16
|
+
expiry = raw_message.message_attributes.include?("expiry") ?
|
|
17
|
+
raw_message.message_attributes["expiry"]["string_value"].to_i :
|
|
18
|
+
nil
|
|
19
|
+
|
|
20
|
+
if expiry && (expiry > 0) && ((Time.now.utc.to_i * 1000) > expiry)
|
|
21
|
+
self.app.logger.queue.info "Job has expired, not running. #{message.inspect}"
|
|
22
|
+
|
|
23
|
+
return Concurrent::Promise.new {}
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
if message["Type"] != nil and message["Type"] == "Notification"
|
|
27
|
+
if message["Message"].is_a? String
|
|
28
|
+
message["Message"] = JSON.parse(message["Message"])
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
# Get the type
|
|
32
|
+
if (message.include? "Subject") && (message["Subject"].include? "Elastic Transcoder")
|
|
33
|
+
type = "Transcoder"
|
|
34
|
+
else
|
|
35
|
+
type = message["Message"]["notificationType"]
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# Update the message with sent and triggered timestamp
|
|
39
|
+
message["Message"]["sentTimestamp"] = sent_timestamp
|
|
40
|
+
message["Message"]["triggerTimestamp"] = trigger_timestamp || sent_timestamp
|
|
41
|
+
|
|
42
|
+
# Update the message with configuration Id
|
|
43
|
+
message["Message"]["configurationId"] = "notification#{type}"
|
|
44
|
+
|
|
45
|
+
# Since zips take a log time to process we might need to use:
|
|
46
|
+
# poller.change_message_visibility_timeout(msg, 60)
|
|
47
|
+
# To make sure other workers don't pick up the job
|
|
48
|
+
return Concurrent::Promise.new {
|
|
49
|
+
if !self.ignored_notifications.include? type
|
|
50
|
+
execute_message!(nil, nil, message["Message"])
|
|
51
|
+
end
|
|
52
|
+
}
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
if message["Records"].is_a? Array
|
|
56
|
+
execution_promise = nil
|
|
57
|
+
execution_data = []
|
|
58
|
+
|
|
59
|
+
message["Records"].each do |record|
|
|
60
|
+
data_source = record["eventSource"].split(":")
|
|
61
|
+
data_origin = data_source.first
|
|
62
|
+
data_operation = data_source.last
|
|
63
|
+
record_data = record[data_operation]
|
|
64
|
+
company_id = nil
|
|
65
|
+
user_id = nil
|
|
66
|
+
|
|
67
|
+
# Update the message with sent and triggered timestamp
|
|
68
|
+
record_data["sentTimestamp"] = sent_timestamp
|
|
69
|
+
record_data["triggerTimestamp"] = trigger_timestamp || sent_timestamp
|
|
70
|
+
|
|
71
|
+
execution_data << record_data
|
|
72
|
+
|
|
73
|
+
if data_origin === "application" and record.include? "userIdentity"
|
|
74
|
+
company_id = record["userIdentity"]["companyId"]
|
|
75
|
+
user_id = record["userIdentity"]["principalId"]
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
if execution_promise == nil
|
|
79
|
+
execution_promise = Concurrent::Promise.new {
|
|
80
|
+
execute_message!(company_id, user_id, record_data)
|
|
81
|
+
}
|
|
82
|
+
else
|
|
83
|
+
execution_promise = execution_promise.then {
|
|
84
|
+
execute_message!(company_id, user_id, record_data)
|
|
85
|
+
}
|
|
86
|
+
end
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
return Concurrent::Promise.new {} if execution_promise == nil
|
|
90
|
+
return execution_promise.rescue { |ex|
|
|
91
|
+
execution_data.each do |data|
|
|
92
|
+
fail_action_permanently(data) if will_fail_permanently
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
raise ex
|
|
96
|
+
}
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
return Concurrent::Promise.new {}
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def execute_message!(company_id, user_id, data)
|
|
104
|
+
action = data["configurationId"]
|
|
105
|
+
action_name = action.camelize
|
|
106
|
+
|
|
107
|
+
unless self.app.config.actions["enabled"].include?(action_name)
|
|
108
|
+
message = "Invalid action. #{action} is not an enabled action. Please add this action to the config file."
|
|
109
|
+
|
|
110
|
+
if self.app.config.is_dev?
|
|
111
|
+
raise message
|
|
112
|
+
else
|
|
113
|
+
Raven.capture_message(message)
|
|
114
|
+
return
|
|
115
|
+
end
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
action_name.constantize.send(:invoke, company_id, user_id, data)
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
protected
|
|
123
|
+
def fail_action_permanently(data)
|
|
124
|
+
if data.include? "action_id"
|
|
125
|
+
# action = Action.get(data["action_id"])
|
|
126
|
+
#
|
|
127
|
+
# if action
|
|
128
|
+
# action.update_data({ :stats => nil, :errors => nil }, :errored)
|
|
129
|
+
# action.cleanup
|
|
130
|
+
# end
|
|
131
|
+
end
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
end
|
|
136
|
+
end
|
|
137
|
+
end
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
module HerdstWorker
|
|
2
|
+
module Signals
|
|
3
|
+
class Facade
|
|
4
|
+
|
|
5
|
+
def self.listen(process_path)
|
|
6
|
+
write_process_file(process_path)
|
|
7
|
+
|
|
8
|
+
# Start
|
|
9
|
+
Signal.trap "USR1" do |x|
|
|
10
|
+
HerdstWorker::Queue::Facade.start
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
# Halt
|
|
14
|
+
Signal.trap "USR2" do |x|
|
|
15
|
+
HerdstWorker::Queue::Facade.halt
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
# Stop (abort)
|
|
19
|
+
Signal.trap "ABRT" do |x|
|
|
20
|
+
HerdstWorker::Queue::Facade.stop
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
private
|
|
25
|
+
def self.write_process_file(process_path)
|
|
26
|
+
# Write process id to file so we can signal the current process
|
|
27
|
+
process_id = "#{$$}"
|
|
28
|
+
process_file = process_path + "/process_id"
|
|
29
|
+
|
|
30
|
+
File.open(process_file, "w") { |file| file.write(process_id) }
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
metadata
ADDED
|
@@ -0,0 +1,272 @@
|
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
|
2
|
+
name: herdst_worker
|
|
3
|
+
version: !ruby/object:Gem::Version
|
|
4
|
+
version: 0.1.0
|
|
5
|
+
platform: ruby
|
|
6
|
+
authors:
|
|
7
|
+
- Herd.St
|
|
8
|
+
autorequire:
|
|
9
|
+
bindir: bin
|
|
10
|
+
cert_chain: []
|
|
11
|
+
date: 2021-07-02 00:00:00.000000000 Z
|
|
12
|
+
dependencies:
|
|
13
|
+
- !ruby/object:Gem::Dependency
|
|
14
|
+
name: activesupport
|
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
|
16
|
+
requirements:
|
|
17
|
+
- - "~>"
|
|
18
|
+
- !ruby/object:Gem::Version
|
|
19
|
+
version: '6.1'
|
|
20
|
+
- - ">="
|
|
21
|
+
- !ruby/object:Gem::Version
|
|
22
|
+
version: 6.1.4
|
|
23
|
+
type: :runtime
|
|
24
|
+
prerelease: false
|
|
25
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
26
|
+
requirements:
|
|
27
|
+
- - "~>"
|
|
28
|
+
- !ruby/object:Gem::Version
|
|
29
|
+
version: '6.1'
|
|
30
|
+
- - ">="
|
|
31
|
+
- !ruby/object:Gem::Version
|
|
32
|
+
version: 6.1.4
|
|
33
|
+
- !ruby/object:Gem::Dependency
|
|
34
|
+
name: activerecord
|
|
35
|
+
requirement: !ruby/object:Gem::Requirement
|
|
36
|
+
requirements:
|
|
37
|
+
- - "~>"
|
|
38
|
+
- !ruby/object:Gem::Version
|
|
39
|
+
version: '6.1'
|
|
40
|
+
- - ">="
|
|
41
|
+
- !ruby/object:Gem::Version
|
|
42
|
+
version: 6.1.4
|
|
43
|
+
type: :runtime
|
|
44
|
+
prerelease: false
|
|
45
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
46
|
+
requirements:
|
|
47
|
+
- - "~>"
|
|
48
|
+
- !ruby/object:Gem::Version
|
|
49
|
+
version: '6.1'
|
|
50
|
+
- - ">="
|
|
51
|
+
- !ruby/object:Gem::Version
|
|
52
|
+
version: 6.1.4
|
|
53
|
+
- !ruby/object:Gem::Dependency
|
|
54
|
+
name: concurrent-ruby
|
|
55
|
+
requirement: !ruby/object:Gem::Requirement
|
|
56
|
+
requirements:
|
|
57
|
+
- - "~>"
|
|
58
|
+
- !ruby/object:Gem::Version
|
|
59
|
+
version: '1.1'
|
|
60
|
+
- - ">="
|
|
61
|
+
- !ruby/object:Gem::Version
|
|
62
|
+
version: 1.1.9
|
|
63
|
+
type: :runtime
|
|
64
|
+
prerelease: false
|
|
65
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
66
|
+
requirements:
|
|
67
|
+
- - "~>"
|
|
68
|
+
- !ruby/object:Gem::Version
|
|
69
|
+
version: '1.1'
|
|
70
|
+
- - ">="
|
|
71
|
+
- !ruby/object:Gem::Version
|
|
72
|
+
version: 1.1.9
|
|
73
|
+
- !ruby/object:Gem::Dependency
|
|
74
|
+
name: mysql2
|
|
75
|
+
requirement: !ruby/object:Gem::Requirement
|
|
76
|
+
requirements:
|
|
77
|
+
- - "~>"
|
|
78
|
+
- !ruby/object:Gem::Version
|
|
79
|
+
version: 0.5.3
|
|
80
|
+
type: :runtime
|
|
81
|
+
prerelease: false
|
|
82
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
83
|
+
requirements:
|
|
84
|
+
- - "~>"
|
|
85
|
+
- !ruby/object:Gem::Version
|
|
86
|
+
version: 0.5.3
|
|
87
|
+
- !ruby/object:Gem::Dependency
|
|
88
|
+
name: aws-sdk-core
|
|
89
|
+
requirement: !ruby/object:Gem::Requirement
|
|
90
|
+
requirements:
|
|
91
|
+
- - "~>"
|
|
92
|
+
- !ruby/object:Gem::Version
|
|
93
|
+
version: '3.114'
|
|
94
|
+
- - ">="
|
|
95
|
+
- !ruby/object:Gem::Version
|
|
96
|
+
version: 3.114.3
|
|
97
|
+
type: :runtime
|
|
98
|
+
prerelease: false
|
|
99
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
100
|
+
requirements:
|
|
101
|
+
- - "~>"
|
|
102
|
+
- !ruby/object:Gem::Version
|
|
103
|
+
version: '3.114'
|
|
104
|
+
- - ">="
|
|
105
|
+
- !ruby/object:Gem::Version
|
|
106
|
+
version: 3.114.3
|
|
107
|
+
- !ruby/object:Gem::Dependency
|
|
108
|
+
name: aws-sdk-sqs
|
|
109
|
+
requirement: !ruby/object:Gem::Requirement
|
|
110
|
+
requirements:
|
|
111
|
+
- - "~>"
|
|
112
|
+
- !ruby/object:Gem::Version
|
|
113
|
+
version: '1.39'
|
|
114
|
+
type: :runtime
|
|
115
|
+
prerelease: false
|
|
116
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
117
|
+
requirements:
|
|
118
|
+
- - "~>"
|
|
119
|
+
- !ruby/object:Gem::Version
|
|
120
|
+
version: '1.39'
|
|
121
|
+
- !ruby/object:Gem::Dependency
|
|
122
|
+
name: aws-sdk-secretsmanager
|
|
123
|
+
requirement: !ruby/object:Gem::Requirement
|
|
124
|
+
requirements:
|
|
125
|
+
- - "~>"
|
|
126
|
+
- !ruby/object:Gem::Version
|
|
127
|
+
version: '1.46'
|
|
128
|
+
type: :runtime
|
|
129
|
+
prerelease: false
|
|
130
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
131
|
+
requirements:
|
|
132
|
+
- - "~>"
|
|
133
|
+
- !ruby/object:Gem::Version
|
|
134
|
+
version: '1.46'
|
|
135
|
+
- !ruby/object:Gem::Dependency
|
|
136
|
+
name: aws-sdk-ecs
|
|
137
|
+
requirement: !ruby/object:Gem::Requirement
|
|
138
|
+
requirements:
|
|
139
|
+
- - "~>"
|
|
140
|
+
- !ruby/object:Gem::Version
|
|
141
|
+
version: '1.80'
|
|
142
|
+
type: :runtime
|
|
143
|
+
prerelease: false
|
|
144
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
145
|
+
requirements:
|
|
146
|
+
- - "~>"
|
|
147
|
+
- !ruby/object:Gem::Version
|
|
148
|
+
version: '1.80'
|
|
149
|
+
- !ruby/object:Gem::Dependency
|
|
150
|
+
name: sentry-raven
|
|
151
|
+
requirement: !ruby/object:Gem::Requirement
|
|
152
|
+
requirements:
|
|
153
|
+
- - "~>"
|
|
154
|
+
- !ruby/object:Gem::Version
|
|
155
|
+
version: '3.1'
|
|
156
|
+
- - ">="
|
|
157
|
+
- !ruby/object:Gem::Version
|
|
158
|
+
version: 3.1.2
|
|
159
|
+
type: :runtime
|
|
160
|
+
prerelease: false
|
|
161
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
162
|
+
requirements:
|
|
163
|
+
- - "~>"
|
|
164
|
+
- !ruby/object:Gem::Version
|
|
165
|
+
version: '3.1'
|
|
166
|
+
- - ">="
|
|
167
|
+
- !ruby/object:Gem::Version
|
|
168
|
+
version: 3.1.2
|
|
169
|
+
- !ruby/object:Gem::Dependency
|
|
170
|
+
name: rake
|
|
171
|
+
requirement: !ruby/object:Gem::Requirement
|
|
172
|
+
requirements:
|
|
173
|
+
- - "~>"
|
|
174
|
+
- !ruby/object:Gem::Version
|
|
175
|
+
version: '13.0'
|
|
176
|
+
- - ">="
|
|
177
|
+
- !ruby/object:Gem::Version
|
|
178
|
+
version: 13.0.3
|
|
179
|
+
type: :development
|
|
180
|
+
prerelease: false
|
|
181
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
182
|
+
requirements:
|
|
183
|
+
- - "~>"
|
|
184
|
+
- !ruby/object:Gem::Version
|
|
185
|
+
version: '13.0'
|
|
186
|
+
- - ">="
|
|
187
|
+
- !ruby/object:Gem::Version
|
|
188
|
+
version: 13.0.3
|
|
189
|
+
- !ruby/object:Gem::Dependency
|
|
190
|
+
name: minitest
|
|
191
|
+
requirement: !ruby/object:Gem::Requirement
|
|
192
|
+
requirements:
|
|
193
|
+
- - "~>"
|
|
194
|
+
- !ruby/object:Gem::Version
|
|
195
|
+
version: '5.14'
|
|
196
|
+
- - ">="
|
|
197
|
+
- !ruby/object:Gem::Version
|
|
198
|
+
version: 5.14.4
|
|
199
|
+
type: :development
|
|
200
|
+
prerelease: false
|
|
201
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
202
|
+
requirements:
|
|
203
|
+
- - "~>"
|
|
204
|
+
- !ruby/object:Gem::Version
|
|
205
|
+
version: '5.14'
|
|
206
|
+
- - ">="
|
|
207
|
+
- !ruby/object:Gem::Version
|
|
208
|
+
version: 5.14.4
|
|
209
|
+
- !ruby/object:Gem::Dependency
|
|
210
|
+
name: fakeweb
|
|
211
|
+
requirement: !ruby/object:Gem::Requirement
|
|
212
|
+
requirements:
|
|
213
|
+
- - "~>"
|
|
214
|
+
- !ruby/object:Gem::Version
|
|
215
|
+
version: '1.3'
|
|
216
|
+
type: :development
|
|
217
|
+
prerelease: false
|
|
218
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
219
|
+
requirements:
|
|
220
|
+
- - "~>"
|
|
221
|
+
- !ruby/object:Gem::Version
|
|
222
|
+
version: '1.3'
|
|
223
|
+
description: Run background work triggered by aws sqs on AWS ECS
|
|
224
|
+
email:
|
|
225
|
+
- devops@herd.st
|
|
226
|
+
executables:
|
|
227
|
+
- herdst_worker
|
|
228
|
+
extensions: []
|
|
229
|
+
extra_rdoc_files: []
|
|
230
|
+
files:
|
|
231
|
+
- Rakefile
|
|
232
|
+
- bin/herdst_worker
|
|
233
|
+
- lib/herdst_worker.rb
|
|
234
|
+
- lib/herdst_worker/adapters/database.rb
|
|
235
|
+
- lib/herdst_worker/adapters/facade.rb
|
|
236
|
+
- lib/herdst_worker/adapters/sentry.rb
|
|
237
|
+
- lib/herdst_worker/application/facade.rb
|
|
238
|
+
- lib/herdst_worker/autoload/facade.rb
|
|
239
|
+
- lib/herdst_worker/configuration/facade.rb
|
|
240
|
+
- lib/herdst_worker/configuration/metadata.rb
|
|
241
|
+
- lib/herdst_worker/configuration/paths.rb
|
|
242
|
+
- lib/herdst_worker/log/facade.rb
|
|
243
|
+
- lib/herdst_worker/queue/facade.rb
|
|
244
|
+
- lib/herdst_worker/queue/processor.rb
|
|
245
|
+
- lib/herdst_worker/queue/runner.rb
|
|
246
|
+
- lib/herdst_worker/signals/facade.rb
|
|
247
|
+
- lib/herdst_worker/version.rb
|
|
248
|
+
homepage: https://herd.st
|
|
249
|
+
licenses:
|
|
250
|
+
- Nonstandard
|
|
251
|
+
metadata:
|
|
252
|
+
allowed_push_host: https://rubygems.org/
|
|
253
|
+
post_install_message:
|
|
254
|
+
rdoc_options: []
|
|
255
|
+
require_paths:
|
|
256
|
+
- lib
|
|
257
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
|
258
|
+
requirements:
|
|
259
|
+
- - ">="
|
|
260
|
+
- !ruby/object:Gem::Version
|
|
261
|
+
version: '0'
|
|
262
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
263
|
+
requirements:
|
|
264
|
+
- - ">="
|
|
265
|
+
- !ruby/object:Gem::Version
|
|
266
|
+
version: '0'
|
|
267
|
+
requirements: []
|
|
268
|
+
rubygems_version: 3.2.20
|
|
269
|
+
signing_key:
|
|
270
|
+
specification_version: 4
|
|
271
|
+
summary: Run background work triggered by aws sqs
|
|
272
|
+
test_files: []
|