herdst_worker 0.1.5 → 0.1.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/bin/herdst_worker_console +24 -0
- data/lib/herdst_worker/adapters/facade.rb +20 -20
- data/lib/herdst_worker/application/facade.rb +98 -89
- data/lib/herdst_worker/autoload/facade.rb +67 -67
- data/lib/herdst_worker/configuration/metadata.rb +5 -1
- data/lib/herdst_worker/configuration/paths.rb +41 -41
- data/lib/herdst_worker/log/facade.rb +87 -87
- data/lib/herdst_worker/queue/facade.rb +164 -162
- data/lib/herdst_worker/queue/processor.rb +225 -223
- data/lib/herdst_worker/queue/runner.rb +137 -137
- data/lib/herdst_worker/signals/facade.rb +35 -35
- data/lib/herdst_worker/version.rb +1 -1
- metadata +8 -6
|
@@ -1,137 +1,137 @@
|
|
|
1
|
-
require 'concurrent'
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
module HerdstWorker
|
|
5
|
-
module Queue
|
|
6
|
-
class Runner
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
def process_message!(message, raw_message = nil, will_fail_permanently = false)
|
|
10
|
-
sent_timestamp = raw_message.attributes.include?("SentTimestamp") ?
|
|
11
|
-
raw_message.attributes["SentTimestamp"].to_i :
|
|
12
|
-
nil
|
|
13
|
-
trigger_timestamp = raw_message.message_attributes.include?("triggerTimestamp") ?
|
|
14
|
-
raw_message.message_attributes["triggerTimestamp"]["string_value"].to_i :
|
|
15
|
-
nil
|
|
16
|
-
expiry = raw_message.message_attributes.include?("expiry") ?
|
|
17
|
-
raw_message.message_attributes["expiry"]["string_value"].to_i :
|
|
18
|
-
nil
|
|
19
|
-
|
|
20
|
-
if expiry && (expiry > 0) && ((Time.now.utc.to_i * 1000) > expiry)
|
|
21
|
-
self.app.logger.queue.info "Job has expired, not running. #{message.inspect}"
|
|
22
|
-
|
|
23
|
-
return Concurrent::Promise.new {}
|
|
24
|
-
end
|
|
25
|
-
|
|
26
|
-
if message["Type"] != nil and message["Type"] == "Notification"
|
|
27
|
-
if message["Message"].is_a? String
|
|
28
|
-
message["Message"] = JSON.parse(message["Message"])
|
|
29
|
-
end
|
|
30
|
-
|
|
31
|
-
# Get the type
|
|
32
|
-
if (message.include? "Subject") && (message["Subject"].include? "Elastic Transcoder")
|
|
33
|
-
type = "Transcoder"
|
|
34
|
-
else
|
|
35
|
-
type = message["Message"]["notificationType"]
|
|
36
|
-
end
|
|
37
|
-
|
|
38
|
-
# Update the message with sent and triggered timestamp
|
|
39
|
-
message["Message"]["sentTimestamp"] = sent_timestamp
|
|
40
|
-
message["Message"]["triggerTimestamp"] = trigger_timestamp || sent_timestamp
|
|
41
|
-
|
|
42
|
-
# Update the message with configuration Id
|
|
43
|
-
message["Message"]["configurationId"] = "notification#{type}"
|
|
44
|
-
|
|
45
|
-
# Since zips take a log time to process we might need to use:
|
|
46
|
-
# poller.change_message_visibility_timeout(msg, 60)
|
|
47
|
-
# To make sure other workers don't pick up the job
|
|
48
|
-
return Concurrent::Promise.new {
|
|
49
|
-
if !self.ignored_notifications.include? type
|
|
50
|
-
execute_message!(nil, nil, message["Message"])
|
|
51
|
-
end
|
|
52
|
-
}
|
|
53
|
-
end
|
|
54
|
-
|
|
55
|
-
if message["Records"].is_a? Array
|
|
56
|
-
execution_promise = nil
|
|
57
|
-
execution_data = []
|
|
58
|
-
|
|
59
|
-
message["Records"].each do |record|
|
|
60
|
-
data_source = record["eventSource"].split(":")
|
|
61
|
-
data_origin = data_source.first
|
|
62
|
-
data_operation = data_source.last
|
|
63
|
-
record_data = record[data_operation]
|
|
64
|
-
company_id = nil
|
|
65
|
-
user_id = nil
|
|
66
|
-
|
|
67
|
-
# Update the message with sent and triggered timestamp
|
|
68
|
-
record_data["sentTimestamp"] = sent_timestamp
|
|
69
|
-
record_data["triggerTimestamp"] = trigger_timestamp || sent_timestamp
|
|
70
|
-
|
|
71
|
-
execution_data << record_data
|
|
72
|
-
|
|
73
|
-
if data_origin === "application" and record.include? "userIdentity"
|
|
74
|
-
company_id = record["userIdentity"]["companyId"]
|
|
75
|
-
user_id = record["userIdentity"]["principalId"]
|
|
76
|
-
end
|
|
77
|
-
|
|
78
|
-
if execution_promise == nil
|
|
79
|
-
execution_promise = Concurrent::Promise.new {
|
|
80
|
-
execute_message!(company_id, user_id, record_data)
|
|
81
|
-
}
|
|
82
|
-
else
|
|
83
|
-
execution_promise = execution_promise.then {
|
|
84
|
-
execute_message!(company_id, user_id, record_data)
|
|
85
|
-
}
|
|
86
|
-
end
|
|
87
|
-
end
|
|
88
|
-
|
|
89
|
-
return Concurrent::Promise.new {} if execution_promise == nil
|
|
90
|
-
return execution_promise.rescue { |ex|
|
|
91
|
-
execution_data.each do |data|
|
|
92
|
-
fail_action_permanently(data) if will_fail_permanently
|
|
93
|
-
end
|
|
94
|
-
|
|
95
|
-
raise ex
|
|
96
|
-
}
|
|
97
|
-
end
|
|
98
|
-
|
|
99
|
-
return Concurrent::Promise.new {}
|
|
100
|
-
end
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
def execute_message!(company_id, user_id, data)
|
|
104
|
-
action = data["configurationId"]
|
|
105
|
-
action_name = action.camelize
|
|
106
|
-
|
|
107
|
-
unless self.app.config.actions["enabled"].include?(action_name)
|
|
108
|
-
message = "Invalid action. #{action} is not an enabled action. Please add this action to the config file."
|
|
109
|
-
|
|
110
|
-
if self.app.config.is_dev?
|
|
111
|
-
raise message
|
|
112
|
-
else
|
|
113
|
-
Raven.capture_message(message)
|
|
114
|
-
return
|
|
115
|
-
end
|
|
116
|
-
end
|
|
117
|
-
|
|
118
|
-
action_name.constantize.send(:invoke, company_id, user_id, data)
|
|
119
|
-
end
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
protected
|
|
123
|
-
def fail_action_permanently(data)
|
|
124
|
-
if data.include? "action_id"
|
|
125
|
-
# action = Action.get(data["action_id"])
|
|
126
|
-
#
|
|
127
|
-
# if action
|
|
128
|
-
# action.update_data({ :stats => nil, :errors => nil }, :errored)
|
|
129
|
-
# action.cleanup
|
|
130
|
-
# end
|
|
131
|
-
end
|
|
132
|
-
end
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
end
|
|
136
|
-
end
|
|
137
|
-
end
|
|
1
|
+
require 'concurrent'
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
module HerdstWorker
|
|
5
|
+
module Queue
|
|
6
|
+
class Runner
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def process_message!(message, raw_message = nil, will_fail_permanently = false)
|
|
10
|
+
sent_timestamp = raw_message.attributes.include?("SentTimestamp") ?
|
|
11
|
+
raw_message.attributes["SentTimestamp"].to_i :
|
|
12
|
+
nil
|
|
13
|
+
trigger_timestamp = raw_message.message_attributes.include?("triggerTimestamp") ?
|
|
14
|
+
raw_message.message_attributes["triggerTimestamp"]["string_value"].to_i :
|
|
15
|
+
nil
|
|
16
|
+
expiry = raw_message.message_attributes.include?("expiry") ?
|
|
17
|
+
raw_message.message_attributes["expiry"]["string_value"].to_i :
|
|
18
|
+
nil
|
|
19
|
+
|
|
20
|
+
if expiry && (expiry > 0) && ((Time.now.utc.to_i * 1000) > expiry)
|
|
21
|
+
self.app.logger.queue.info "Job has expired, not running. #{message.inspect}"
|
|
22
|
+
|
|
23
|
+
return Concurrent::Promise.new {}
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
if message["Type"] != nil and message["Type"] == "Notification"
|
|
27
|
+
if message["Message"].is_a? String
|
|
28
|
+
message["Message"] = JSON.parse(message["Message"])
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
# Get the type
|
|
32
|
+
if (message.include? "Subject") && (message["Subject"].include? "Elastic Transcoder")
|
|
33
|
+
type = "Transcoder"
|
|
34
|
+
else
|
|
35
|
+
type = message["Message"]["notificationType"]
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# Update the message with sent and triggered timestamp
|
|
39
|
+
message["Message"]["sentTimestamp"] = sent_timestamp
|
|
40
|
+
message["Message"]["triggerTimestamp"] = trigger_timestamp || sent_timestamp
|
|
41
|
+
|
|
42
|
+
# Update the message with configuration Id
|
|
43
|
+
message["Message"]["configurationId"] = "notification#{type}"
|
|
44
|
+
|
|
45
|
+
# Since zips take a log time to process we might need to use:
|
|
46
|
+
# poller.change_message_visibility_timeout(msg, 60)
|
|
47
|
+
# To make sure other workers don't pick up the job
|
|
48
|
+
return Concurrent::Promise.new {
|
|
49
|
+
if !self.ignored_notifications.include? type
|
|
50
|
+
execute_message!(nil, nil, message["Message"])
|
|
51
|
+
end
|
|
52
|
+
}
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
if message["Records"].is_a? Array
|
|
56
|
+
execution_promise = nil
|
|
57
|
+
execution_data = []
|
|
58
|
+
|
|
59
|
+
message["Records"].each do |record|
|
|
60
|
+
data_source = record["eventSource"].split(":")
|
|
61
|
+
data_origin = data_source.first
|
|
62
|
+
data_operation = data_source.last
|
|
63
|
+
record_data = record[data_operation]
|
|
64
|
+
company_id = nil
|
|
65
|
+
user_id = nil
|
|
66
|
+
|
|
67
|
+
# Update the message with sent and triggered timestamp
|
|
68
|
+
record_data["sentTimestamp"] = sent_timestamp
|
|
69
|
+
record_data["triggerTimestamp"] = trigger_timestamp || sent_timestamp
|
|
70
|
+
|
|
71
|
+
execution_data << record_data
|
|
72
|
+
|
|
73
|
+
if data_origin === "application" and record.include? "userIdentity"
|
|
74
|
+
company_id = record["userIdentity"]["companyId"]
|
|
75
|
+
user_id = record["userIdentity"]["principalId"]
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
if execution_promise == nil
|
|
79
|
+
execution_promise = Concurrent::Promise.new {
|
|
80
|
+
execute_message!(company_id, user_id, record_data)
|
|
81
|
+
}
|
|
82
|
+
else
|
|
83
|
+
execution_promise = execution_promise.then {
|
|
84
|
+
execute_message!(company_id, user_id, record_data)
|
|
85
|
+
}
|
|
86
|
+
end
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
return Concurrent::Promise.new {} if execution_promise == nil
|
|
90
|
+
return execution_promise.rescue { |ex|
|
|
91
|
+
execution_data.each do |data|
|
|
92
|
+
fail_action_permanently(data) if will_fail_permanently
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
raise ex
|
|
96
|
+
}
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
return Concurrent::Promise.new {}
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def execute_message!(company_id, user_id, data)
|
|
104
|
+
action = data["configurationId"]
|
|
105
|
+
action_name = action.camelize
|
|
106
|
+
|
|
107
|
+
unless self.app.config.actions["enabled"].include?(action_name)
|
|
108
|
+
message = "Invalid action. #{action} is not an enabled action. Please add this action to the config file."
|
|
109
|
+
|
|
110
|
+
if self.app.config.is_dev?
|
|
111
|
+
raise message
|
|
112
|
+
else
|
|
113
|
+
Raven.capture_message(message)
|
|
114
|
+
return
|
|
115
|
+
end
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
action_name.constantize.send(:invoke, company_id, user_id, data)
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
protected
|
|
123
|
+
def fail_action_permanently(data)
|
|
124
|
+
if data.include? "action_id"
|
|
125
|
+
# action = Action.get(data["action_id"])
|
|
126
|
+
#
|
|
127
|
+
# if action
|
|
128
|
+
# action.update_data({ :stats => nil, :errors => nil }, :errored)
|
|
129
|
+
# action.cleanup
|
|
130
|
+
# end
|
|
131
|
+
end
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
end
|
|
136
|
+
end
|
|
137
|
+
end
|
|
@@ -1,35 +1,35 @@
|
|
|
1
|
-
module HerdstWorker
|
|
2
|
-
module Signals
|
|
3
|
-
class Facade
|
|
4
|
-
|
|
5
|
-
def self.listen(process_path)
|
|
6
|
-
write_process_file(process_path)
|
|
7
|
-
|
|
8
|
-
# Start
|
|
9
|
-
Signal.trap "USR1" do |x|
|
|
10
|
-
HerdstWorker::Queue::Facade.start
|
|
11
|
-
end
|
|
12
|
-
|
|
13
|
-
# Halt
|
|
14
|
-
Signal.trap "USR2" do |x|
|
|
15
|
-
HerdstWorker::Queue::Facade.halt
|
|
16
|
-
end
|
|
17
|
-
|
|
18
|
-
# Stop (abort)
|
|
19
|
-
Signal.trap "ABRT" do |x|
|
|
20
|
-
HerdstWorker::Queue::Facade.stop
|
|
21
|
-
end
|
|
22
|
-
end
|
|
23
|
-
|
|
24
|
-
private
|
|
25
|
-
def self.write_process_file(process_path)
|
|
26
|
-
# Write process id to file so we can signal the current process
|
|
27
|
-
process_id = "#{$$}"
|
|
28
|
-
process_file = process_path + "/process_id"
|
|
29
|
-
|
|
30
|
-
File.open(process_file, "w") { |file| file.write(process_id) }
|
|
31
|
-
end
|
|
32
|
-
|
|
33
|
-
end
|
|
34
|
-
end
|
|
35
|
-
end
|
|
1
|
+
module HerdstWorker
|
|
2
|
+
module Signals
|
|
3
|
+
class Facade
|
|
4
|
+
|
|
5
|
+
def self.listen(process_path)
|
|
6
|
+
write_process_file(process_path)
|
|
7
|
+
|
|
8
|
+
# Start
|
|
9
|
+
Signal.trap "USR1" do |x|
|
|
10
|
+
HerdstWorker::Queue::Facade.start
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
# Halt
|
|
14
|
+
Signal.trap "USR2" do |x|
|
|
15
|
+
HerdstWorker::Queue::Facade.halt
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
# Stop (abort)
|
|
19
|
+
Signal.trap "ABRT" do |x|
|
|
20
|
+
HerdstWorker::Queue::Facade.stop
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
private
|
|
25
|
+
def self.write_process_file(process_path)
|
|
26
|
+
# Write process id to file so we can signal the current process
|
|
27
|
+
process_id = "#{$$}"
|
|
28
|
+
process_file = process_path + "/process_id"
|
|
29
|
+
|
|
30
|
+
File.open(process_file, "w") { |file| file.write(process_id) }
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: herdst_worker
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.8
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Herd.St
|
|
8
|
-
autorequire:
|
|
8
|
+
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2022-04-04 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: activesupport
|
|
@@ -225,11 +225,13 @@ email:
|
|
|
225
225
|
- devops@herd.st
|
|
226
226
|
executables:
|
|
227
227
|
- herdst_worker
|
|
228
|
+
- herdst_worker_console
|
|
228
229
|
extensions: []
|
|
229
230
|
extra_rdoc_files: []
|
|
230
231
|
files:
|
|
231
232
|
- Rakefile
|
|
232
233
|
- bin/herdst_worker
|
|
234
|
+
- bin/herdst_worker_console
|
|
233
235
|
- lib/herdst_worker.rb
|
|
234
236
|
- lib/herdst_worker/adapters/database.rb
|
|
235
237
|
- lib/herdst_worker/adapters/facade.rb
|
|
@@ -250,7 +252,7 @@ licenses:
|
|
|
250
252
|
- Nonstandard
|
|
251
253
|
metadata:
|
|
252
254
|
allowed_push_host: https://rubygems.org/
|
|
253
|
-
post_install_message:
|
|
255
|
+
post_install_message:
|
|
254
256
|
rdoc_options: []
|
|
255
257
|
require_paths:
|
|
256
258
|
- lib
|
|
@@ -265,8 +267,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
265
267
|
- !ruby/object:Gem::Version
|
|
266
268
|
version: '0'
|
|
267
269
|
requirements: []
|
|
268
|
-
rubygems_version: 3.
|
|
269
|
-
signing_key:
|
|
270
|
+
rubygems_version: 3.1.4
|
|
271
|
+
signing_key:
|
|
270
272
|
specification_version: 4
|
|
271
273
|
summary: Run background work triggered by aws sqs
|
|
272
274
|
test_files: []
|