herdst_worker 0.1.6 → 0.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/herdst_worker/adapters/facade.rb +20 -20
- data/lib/herdst_worker/application/facade.rb +98 -89
- data/lib/herdst_worker/autoload/facade.rb +67 -67
- data/lib/herdst_worker/configuration/paths.rb +41 -41
- data/lib/herdst_worker/log/facade.rb +87 -87
- data/lib/herdst_worker/queue/facade.rb +164 -162
- data/lib/herdst_worker/queue/processor.rb +225 -223
- data/lib/herdst_worker/queue/runner.rb +137 -137
- data/lib/herdst_worker/signals/facade.rb +35 -35
- data/lib/herdst_worker/version.rb +1 -1
- metadata +6 -6
|
@@ -1,137 +1,137 @@
|
|
|
1
|
-
require 'concurrent'
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
module HerdstWorker
|
|
5
|
-
module Queue
|
|
6
|
-
class Runner
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
def process_message!(message, raw_message = nil, will_fail_permanently = false)
|
|
10
|
-
sent_timestamp = raw_message.attributes.include?("SentTimestamp") ?
|
|
11
|
-
raw_message.attributes["SentTimestamp"].to_i :
|
|
12
|
-
nil
|
|
13
|
-
trigger_timestamp = raw_message.message_attributes.include?("triggerTimestamp") ?
|
|
14
|
-
raw_message.message_attributes["triggerTimestamp"]["string_value"].to_i :
|
|
15
|
-
nil
|
|
16
|
-
expiry = raw_message.message_attributes.include?("expiry") ?
|
|
17
|
-
raw_message.message_attributes["expiry"]["string_value"].to_i :
|
|
18
|
-
nil
|
|
19
|
-
|
|
20
|
-
if expiry && (expiry > 0) && ((Time.now.utc.to_i * 1000) > expiry)
|
|
21
|
-
self.app.logger.queue.info "Job has expired, not running. #{message.inspect}"
|
|
22
|
-
|
|
23
|
-
return Concurrent::Promise.new {}
|
|
24
|
-
end
|
|
25
|
-
|
|
26
|
-
if message["Type"] != nil and message["Type"] == "Notification"
|
|
27
|
-
if message["Message"].is_a? String
|
|
28
|
-
message["Message"] = JSON.parse(message["Message"])
|
|
29
|
-
end
|
|
30
|
-
|
|
31
|
-
# Get the type
|
|
32
|
-
if (message.include? "Subject") && (message["Subject"].include? "Elastic Transcoder")
|
|
33
|
-
type = "Transcoder"
|
|
34
|
-
else
|
|
35
|
-
type = message["Message"]["notificationType"]
|
|
36
|
-
end
|
|
37
|
-
|
|
38
|
-
# Update the message with sent and triggered timestamp
|
|
39
|
-
message["Message"]["sentTimestamp"] = sent_timestamp
|
|
40
|
-
message["Message"]["triggerTimestamp"] = trigger_timestamp || sent_timestamp
|
|
41
|
-
|
|
42
|
-
# Update the message with configuration Id
|
|
43
|
-
message["Message"]["configurationId"] = "notification#{type}"
|
|
44
|
-
|
|
45
|
-
# Since zips take a log time to process we might need to use:
|
|
46
|
-
# poller.change_message_visibility_timeout(msg, 60)
|
|
47
|
-
# To make sure other workers don't pick up the job
|
|
48
|
-
return Concurrent::Promise.new {
|
|
49
|
-
if !self.ignored_notifications.include? type
|
|
50
|
-
execute_message!(nil, nil, message["Message"])
|
|
51
|
-
end
|
|
52
|
-
}
|
|
53
|
-
end
|
|
54
|
-
|
|
55
|
-
if message["Records"].is_a? Array
|
|
56
|
-
execution_promise = nil
|
|
57
|
-
execution_data = []
|
|
58
|
-
|
|
59
|
-
message["Records"].each do |record|
|
|
60
|
-
data_source = record["eventSource"].split(":")
|
|
61
|
-
data_origin = data_source.first
|
|
62
|
-
data_operation = data_source.last
|
|
63
|
-
record_data = record[data_operation]
|
|
64
|
-
company_id = nil
|
|
65
|
-
user_id = nil
|
|
66
|
-
|
|
67
|
-
# Update the message with sent and triggered timestamp
|
|
68
|
-
record_data["sentTimestamp"] = sent_timestamp
|
|
69
|
-
record_data["triggerTimestamp"] = trigger_timestamp || sent_timestamp
|
|
70
|
-
|
|
71
|
-
execution_data << record_data
|
|
72
|
-
|
|
73
|
-
if data_origin === "application" and record.include? "userIdentity"
|
|
74
|
-
company_id = record["userIdentity"]["companyId"]
|
|
75
|
-
user_id = record["userIdentity"]["principalId"]
|
|
76
|
-
end
|
|
77
|
-
|
|
78
|
-
if execution_promise == nil
|
|
79
|
-
execution_promise = Concurrent::Promise.new {
|
|
80
|
-
execute_message!(company_id, user_id, record_data)
|
|
81
|
-
}
|
|
82
|
-
else
|
|
83
|
-
execution_promise = execution_promise.then {
|
|
84
|
-
execute_message!(company_id, user_id, record_data)
|
|
85
|
-
}
|
|
86
|
-
end
|
|
87
|
-
end
|
|
88
|
-
|
|
89
|
-
return Concurrent::Promise.new {} if execution_promise == nil
|
|
90
|
-
return execution_promise.rescue { |ex|
|
|
91
|
-
execution_data.each do |data|
|
|
92
|
-
fail_action_permanently(data) if will_fail_permanently
|
|
93
|
-
end
|
|
94
|
-
|
|
95
|
-
raise ex
|
|
96
|
-
}
|
|
97
|
-
end
|
|
98
|
-
|
|
99
|
-
return Concurrent::Promise.new {}
|
|
100
|
-
end
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
def execute_message!(company_id, user_id, data)
|
|
104
|
-
action = data["configurationId"]
|
|
105
|
-
action_name = action.camelize
|
|
106
|
-
|
|
107
|
-
unless self.app.config.actions["enabled"].include?(action_name)
|
|
108
|
-
message = "Invalid action. #{action} is not an enabled action. Please add this action to the config file."
|
|
109
|
-
|
|
110
|
-
if self.app.config.is_dev?
|
|
111
|
-
raise message
|
|
112
|
-
else
|
|
113
|
-
Raven.capture_message(message)
|
|
114
|
-
return
|
|
115
|
-
end
|
|
116
|
-
end
|
|
117
|
-
|
|
118
|
-
action_name.constantize.send(:invoke, company_id, user_id, data)
|
|
119
|
-
end
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
protected
|
|
123
|
-
def fail_action_permanently(data)
|
|
124
|
-
if data.include? "action_id"
|
|
125
|
-
# action = Action.get(data["action_id"])
|
|
126
|
-
#
|
|
127
|
-
# if action
|
|
128
|
-
# action.update_data({ :stats => nil, :errors => nil }, :errored)
|
|
129
|
-
# action.cleanup
|
|
130
|
-
# end
|
|
131
|
-
end
|
|
132
|
-
end
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
end
|
|
136
|
-
end
|
|
137
|
-
end
|
|
1
|
+
require 'concurrent'
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
module HerdstWorker
|
|
5
|
+
module Queue
|
|
6
|
+
class Runner
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def process_message!(message, raw_message = nil, will_fail_permanently = false)
|
|
10
|
+
sent_timestamp = raw_message.attributes.include?("SentTimestamp") ?
|
|
11
|
+
raw_message.attributes["SentTimestamp"].to_i :
|
|
12
|
+
nil
|
|
13
|
+
trigger_timestamp = raw_message.message_attributes.include?("triggerTimestamp") ?
|
|
14
|
+
raw_message.message_attributes["triggerTimestamp"]["string_value"].to_i :
|
|
15
|
+
nil
|
|
16
|
+
expiry = raw_message.message_attributes.include?("expiry") ?
|
|
17
|
+
raw_message.message_attributes["expiry"]["string_value"].to_i :
|
|
18
|
+
nil
|
|
19
|
+
|
|
20
|
+
if expiry && (expiry > 0) && ((Time.now.utc.to_i * 1000) > expiry)
|
|
21
|
+
self.app.logger.queue.info "Job has expired, not running. #{message.inspect}"
|
|
22
|
+
|
|
23
|
+
return Concurrent::Promise.new {}
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
if message["Type"] != nil and message["Type"] == "Notification"
|
|
27
|
+
if message["Message"].is_a? String
|
|
28
|
+
message["Message"] = JSON.parse(message["Message"])
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
# Get the type
|
|
32
|
+
if (message.include? "Subject") && (message["Subject"].include? "Elastic Transcoder")
|
|
33
|
+
type = "Transcoder"
|
|
34
|
+
else
|
|
35
|
+
type = message["Message"]["notificationType"]
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# Update the message with sent and triggered timestamp
|
|
39
|
+
message["Message"]["sentTimestamp"] = sent_timestamp
|
|
40
|
+
message["Message"]["triggerTimestamp"] = trigger_timestamp || sent_timestamp
|
|
41
|
+
|
|
42
|
+
# Update the message with configuration Id
|
|
43
|
+
message["Message"]["configurationId"] = "notification#{type}"
|
|
44
|
+
|
|
45
|
+
# Since zips take a log time to process we might need to use:
|
|
46
|
+
# poller.change_message_visibility_timeout(msg, 60)
|
|
47
|
+
# To make sure other workers don't pick up the job
|
|
48
|
+
return Concurrent::Promise.new {
|
|
49
|
+
if !self.ignored_notifications.include? type
|
|
50
|
+
execute_message!(nil, nil, message["Message"])
|
|
51
|
+
end
|
|
52
|
+
}
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
if message["Records"].is_a? Array
|
|
56
|
+
execution_promise = nil
|
|
57
|
+
execution_data = []
|
|
58
|
+
|
|
59
|
+
message["Records"].each do |record|
|
|
60
|
+
data_source = record["eventSource"].split(":")
|
|
61
|
+
data_origin = data_source.first
|
|
62
|
+
data_operation = data_source.last
|
|
63
|
+
record_data = record[data_operation]
|
|
64
|
+
company_id = nil
|
|
65
|
+
user_id = nil
|
|
66
|
+
|
|
67
|
+
# Update the message with sent and triggered timestamp
|
|
68
|
+
record_data["sentTimestamp"] = sent_timestamp
|
|
69
|
+
record_data["triggerTimestamp"] = trigger_timestamp || sent_timestamp
|
|
70
|
+
|
|
71
|
+
execution_data << record_data
|
|
72
|
+
|
|
73
|
+
if data_origin === "application" and record.include? "userIdentity"
|
|
74
|
+
company_id = record["userIdentity"]["companyId"]
|
|
75
|
+
user_id = record["userIdentity"]["principalId"]
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
if execution_promise == nil
|
|
79
|
+
execution_promise = Concurrent::Promise.new {
|
|
80
|
+
execute_message!(company_id, user_id, record_data)
|
|
81
|
+
}
|
|
82
|
+
else
|
|
83
|
+
execution_promise = execution_promise.then {
|
|
84
|
+
execute_message!(company_id, user_id, record_data)
|
|
85
|
+
}
|
|
86
|
+
end
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
return Concurrent::Promise.new {} if execution_promise == nil
|
|
90
|
+
return execution_promise.rescue { |ex|
|
|
91
|
+
execution_data.each do |data|
|
|
92
|
+
fail_action_permanently(data) if will_fail_permanently
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
raise ex
|
|
96
|
+
}
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
return Concurrent::Promise.new {}
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def execute_message!(company_id, user_id, data)
|
|
104
|
+
action = data["configurationId"]
|
|
105
|
+
action_name = action.camelize
|
|
106
|
+
|
|
107
|
+
unless self.app.config.actions["enabled"].include?(action_name)
|
|
108
|
+
message = "Invalid action. #{action} is not an enabled action. Please add this action to the config file."
|
|
109
|
+
|
|
110
|
+
if self.app.config.is_dev?
|
|
111
|
+
raise message
|
|
112
|
+
else
|
|
113
|
+
Raven.capture_message(message)
|
|
114
|
+
return
|
|
115
|
+
end
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
action_name.constantize.send(:invoke, company_id, user_id, data)
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
protected
|
|
123
|
+
def fail_action_permanently(data)
|
|
124
|
+
if data.include? "action_id"
|
|
125
|
+
# action = Action.get(data["action_id"])
|
|
126
|
+
#
|
|
127
|
+
# if action
|
|
128
|
+
# action.update_data({ :stats => nil, :errors => nil }, :errored)
|
|
129
|
+
# action.cleanup
|
|
130
|
+
# end
|
|
131
|
+
end
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
end
|
|
136
|
+
end
|
|
137
|
+
end
|
|
@@ -1,35 +1,35 @@
|
|
|
1
|
-
module HerdstWorker
|
|
2
|
-
module Signals
|
|
3
|
-
class Facade
|
|
4
|
-
|
|
5
|
-
def self.listen(process_path)
|
|
6
|
-
write_process_file(process_path)
|
|
7
|
-
|
|
8
|
-
# Start
|
|
9
|
-
Signal.trap "USR1" do |x|
|
|
10
|
-
HerdstWorker::Queue::Facade.start
|
|
11
|
-
end
|
|
12
|
-
|
|
13
|
-
# Halt
|
|
14
|
-
Signal.trap "USR2" do |x|
|
|
15
|
-
HerdstWorker::Queue::Facade.halt
|
|
16
|
-
end
|
|
17
|
-
|
|
18
|
-
# Stop (abort)
|
|
19
|
-
Signal.trap "ABRT" do |x|
|
|
20
|
-
HerdstWorker::Queue::Facade.stop
|
|
21
|
-
end
|
|
22
|
-
end
|
|
23
|
-
|
|
24
|
-
private
|
|
25
|
-
def self.write_process_file(process_path)
|
|
26
|
-
# Write process id to file so we can signal the current process
|
|
27
|
-
process_id = "#{$$}"
|
|
28
|
-
process_file = process_path + "/process_id"
|
|
29
|
-
|
|
30
|
-
File.open(process_file, "w") { |file| file.write(process_id) }
|
|
31
|
-
end
|
|
32
|
-
|
|
33
|
-
end
|
|
34
|
-
end
|
|
35
|
-
end
|
|
1
|
+
module HerdstWorker
|
|
2
|
+
module Signals
|
|
3
|
+
class Facade
|
|
4
|
+
|
|
5
|
+
def self.listen(process_path)
|
|
6
|
+
write_process_file(process_path)
|
|
7
|
+
|
|
8
|
+
# Start
|
|
9
|
+
Signal.trap "USR1" do |x|
|
|
10
|
+
HerdstWorker::Queue::Facade.start
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
# Halt
|
|
14
|
+
Signal.trap "USR2" do |x|
|
|
15
|
+
HerdstWorker::Queue::Facade.halt
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
# Stop (abort)
|
|
19
|
+
Signal.trap "ABRT" do |x|
|
|
20
|
+
HerdstWorker::Queue::Facade.stop
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
private
|
|
25
|
+
def self.write_process_file(process_path)
|
|
26
|
+
# Write process id to file so we can signal the current process
|
|
27
|
+
process_id = "#{$$}"
|
|
28
|
+
process_file = process_path + "/process_id"
|
|
29
|
+
|
|
30
|
+
File.open(process_file, "w") { |file| file.write(process_id) }
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: herdst_worker
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.
|
|
4
|
+
version: 0.1.7
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Herd.St
|
|
8
|
-
autorequire:
|
|
8
|
+
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2022-04-04 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: activesupport
|
|
@@ -250,7 +250,7 @@ licenses:
|
|
|
250
250
|
- Nonstandard
|
|
251
251
|
metadata:
|
|
252
252
|
allowed_push_host: https://rubygems.org/
|
|
253
|
-
post_install_message:
|
|
253
|
+
post_install_message:
|
|
254
254
|
rdoc_options: []
|
|
255
255
|
require_paths:
|
|
256
256
|
- lib
|
|
@@ -265,8 +265,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
265
265
|
- !ruby/object:Gem::Version
|
|
266
266
|
version: '0'
|
|
267
267
|
requirements: []
|
|
268
|
-
rubygems_version: 3.
|
|
269
|
-
signing_key:
|
|
268
|
+
rubygems_version: 3.1.4
|
|
269
|
+
signing_key:
|
|
270
270
|
specification_version: 4
|
|
271
271
|
summary: Run background work triggered by aws sqs
|
|
272
272
|
test_files: []
|