herdst_worker 0.1.4 → 0.1.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (15) hide show
  1. checksums.yaml +4 -4
  2. data/lib/herdst_worker/adapters/database.rb +0 -1
  3. data/lib/herdst_worker/adapters/facade.rb +20 -20
  4. data/lib/herdst_worker/adapters/sentry.rb +1 -1
  5. data/lib/herdst_worker/application/facade.rb +98 -89
  6. data/lib/herdst_worker/autoload/facade.rb +67 -67
  7. data/lib/herdst_worker/configuration/metadata.rb +5 -1
  8. data/lib/herdst_worker/configuration/paths.rb +41 -41
  9. data/lib/herdst_worker/log/facade.rb +87 -87
  10. data/lib/herdst_worker/queue/facade.rb +164 -162
  11. data/lib/herdst_worker/queue/processor.rb +225 -223
  12. data/lib/herdst_worker/queue/runner.rb +137 -137
  13. data/lib/herdst_worker/signals/facade.rb +35 -35
  14. data/lib/herdst_worker/version.rb +1 -1
  15. metadata +6 -6
data/lib/herdst_worker/queue/runner.rb CHANGED
@@ -1,137 +1,137 @@
1
- require 'concurrent'
2
-
3
-
4
- module HerdstWorker
5
- module Queue
6
- class Runner
7
-
8
-
9
- def process_message!(message, raw_message = nil, will_fail_permanently = false)
10
- sent_timestamp = raw_message.attributes.include?("SentTimestamp") ?
11
- raw_message.attributes["SentTimestamp"].to_i :
12
- nil
13
- trigger_timestamp = raw_message.message_attributes.include?("triggerTimestamp") ?
14
- raw_message.message_attributes["triggerTimestamp"]["string_value"].to_i :
15
- nil
16
- expiry = raw_message.message_attributes.include?("expiry") ?
17
- raw_message.message_attributes["expiry"]["string_value"].to_i :
18
- nil
19
-
20
- if expiry && (expiry > 0) && ((Time.now.utc.to_i * 1000) > expiry)
21
- self.app.logger.queue.info "Job has expired, not running. #{message.inspect}"
22
-
23
- return Concurrent::Promise.new {}
24
- end
25
-
26
- if message["Type"] != nil and message["Type"] == "Notification"
27
- if message["Message"].is_a? String
28
- message["Message"] = JSON.parse(message["Message"])
29
- end
30
-
31
- # Get the type
32
- if (message.include? "Subject") && (message["Subject"].include? "Elastic Transcoder")
33
- type = "Transcoder"
34
- else
35
- type = message["Message"]["notificationType"]
36
- end
37
-
38
- # Update the message with sent and triggered timestamp
39
- message["Message"]["sentTimestamp"] = sent_timestamp
40
- message["Message"]["triggerTimestamp"] = trigger_timestamp || sent_timestamp
41
-
42
- # Update the message with configuration Id
43
- message["Message"]["configurationId"] = "notification#{type}"
44
-
45
- # Since zips take a log time to process we might need to use:
46
- # poller.change_message_visibility_timeout(msg, 60)
47
- # To make sure other workers don't pick up the job
48
- return Concurrent::Promise.new {
49
- if !self.ignored_notifications.include? type
50
- execute_message!(nil, nil, message["Message"])
51
- end
52
- }
53
- end
54
-
55
- if message["Records"].is_a? Array
56
- execution_promise = nil
57
- execution_data = []
58
-
59
- message["Records"].each do |record|
60
- data_source = record["eventSource"].split(":")
61
- data_origin = data_source.first
62
- data_operation = data_source.last
63
- record_data = record[data_operation]
64
- company_id = nil
65
- user_id = nil
66
-
67
- # Update the message with sent and triggered timestamp
68
- record_data["sentTimestamp"] = sent_timestamp
69
- record_data["triggerTimestamp"] = trigger_timestamp || sent_timestamp
70
-
71
- execution_data << record_data
72
-
73
- if data_origin === "application" and record.include? "userIdentity"
74
- company_id = record["userIdentity"]["companyId"]
75
- user_id = record["userIdentity"]["principalId"]
76
- end
77
-
78
- if execution_promise == nil
79
- execution_promise = Concurrent::Promise.new {
80
- execute_message!(company_id, user_id, record_data)
81
- }
82
- else
83
- execution_promise = execution_promise.then {
84
- execute_message!(company_id, user_id, record_data)
85
- }
86
- end
87
- end
88
-
89
- return Concurrent::Promise.new {} if execution_promise == nil
90
- return execution_promise.rescue { |ex|
91
- execution_data.each do |data|
92
- fail_action_permanently(data) if will_fail_permanently
93
- end
94
-
95
- raise ex
96
- }
97
- end
98
-
99
- return Concurrent::Promise.new {}
100
- end
101
-
102
-
103
- def execute_message!(company_id, user_id, data)
104
- action = data["configurationId"]
105
- action_name = action.camelize
106
-
107
- unless self.app.config.actions["enabled"].include?(action_name)
108
- message = "Invalid action. #{action} is not an enabled action. Please add this action to the config file."
109
-
110
- if self.app.config.is_dev?
111
- raise message
112
- else
113
- Raven.capture_message(message)
114
- return
115
- end
116
- end
117
-
118
- action_name.constantize.send(:invoke, company_id, user_id, data)
119
- end
120
-
121
-
122
- protected
123
- def fail_action_permanently(data)
124
- if data.include? "action_id"
125
- # action = Action.get(data["action_id"])
126
- #
127
- # if action
128
- # action.update_data({ :stats => nil, :errors => nil }, :errored)
129
- # action.cleanup
130
- # end
131
- end
132
- end
133
-
134
-
135
- end
136
- end
137
- end
1
+ require 'concurrent'
2
+
3
+
4
+ module HerdstWorker
5
+ module Queue
6
+ class Runner
7
+
8
+
9
+ def process_message!(message, raw_message = nil, will_fail_permanently = false)
10
+ sent_timestamp = raw_message.attributes.include?("SentTimestamp") ?
11
+ raw_message.attributes["SentTimestamp"].to_i :
12
+ nil
13
+ trigger_timestamp = raw_message.message_attributes.include?("triggerTimestamp") ?
14
+ raw_message.message_attributes["triggerTimestamp"]["string_value"].to_i :
15
+ nil
16
+ expiry = raw_message.message_attributes.include?("expiry") ?
17
+ raw_message.message_attributes["expiry"]["string_value"].to_i :
18
+ nil
19
+
20
+ if expiry && (expiry > 0) && ((Time.now.utc.to_i * 1000) > expiry)
21
+ self.app.logger.queue.info "Job has expired, not running. #{message.inspect}"
22
+
23
+ return Concurrent::Promise.new {}
24
+ end
25
+
26
+ if message["Type"] != nil and message["Type"] == "Notification"
27
+ if message["Message"].is_a? String
28
+ message["Message"] = JSON.parse(message["Message"])
29
+ end
30
+
31
+ # Get the type
32
+ if (message.include? "Subject") && (message["Subject"].include? "Elastic Transcoder")
33
+ type = "Transcoder"
34
+ else
35
+ type = message["Message"]["notificationType"]
36
+ end
37
+
38
+ # Update the message with sent and triggered timestamp
39
+ message["Message"]["sentTimestamp"] = sent_timestamp
40
+ message["Message"]["triggerTimestamp"] = trigger_timestamp || sent_timestamp
41
+
42
+ # Update the message with configuration Id
43
+ message["Message"]["configurationId"] = "notification#{type}"
44
+
45
+ # Since zips take a log time to process we might need to use:
46
+ # poller.change_message_visibility_timeout(msg, 60)
47
+ # To make sure other workers don't pick up the job
48
+ return Concurrent::Promise.new {
49
+ if !self.ignored_notifications.include? type
50
+ execute_message!(nil, nil, message["Message"])
51
+ end
52
+ }
53
+ end
54
+
55
+ if message["Records"].is_a? Array
56
+ execution_promise = nil
57
+ execution_data = []
58
+
59
+ message["Records"].each do |record|
60
+ data_source = record["eventSource"].split(":")
61
+ data_origin = data_source.first
62
+ data_operation = data_source.last
63
+ record_data = record[data_operation]
64
+ company_id = nil
65
+ user_id = nil
66
+
67
+ # Update the message with sent and triggered timestamp
68
+ record_data["sentTimestamp"] = sent_timestamp
69
+ record_data["triggerTimestamp"] = trigger_timestamp || sent_timestamp
70
+
71
+ execution_data << record_data
72
+
73
+ if data_origin === "application" and record.include? "userIdentity"
74
+ company_id = record["userIdentity"]["companyId"]
75
+ user_id = record["userIdentity"]["principalId"]
76
+ end
77
+
78
+ if execution_promise == nil
79
+ execution_promise = Concurrent::Promise.new {
80
+ execute_message!(company_id, user_id, record_data)
81
+ }
82
+ else
83
+ execution_promise = execution_promise.then {
84
+ execute_message!(company_id, user_id, record_data)
85
+ }
86
+ end
87
+ end
88
+
89
+ return Concurrent::Promise.new {} if execution_promise == nil
90
+ return execution_promise.rescue { |ex|
91
+ execution_data.each do |data|
92
+ fail_action_permanently(data) if will_fail_permanently
93
+ end
94
+
95
+ raise ex
96
+ }
97
+ end
98
+
99
+ return Concurrent::Promise.new {}
100
+ end
101
+
102
+
103
+ def execute_message!(company_id, user_id, data)
104
+ action = data["configurationId"]
105
+ action_name = action.camelize
106
+
107
+ unless self.app.config.actions["enabled"].include?(action_name)
108
+ message = "Invalid action. #{action} is not an enabled action. Please add this action to the config file."
109
+
110
+ if self.app.config.is_dev?
111
+ raise message
112
+ else
113
+ Raven.capture_message(message)
114
+ return
115
+ end
116
+ end
117
+
118
+ action_name.constantize.send(:invoke, company_id, user_id, data)
119
+ end
120
+
121
+
122
+ protected
123
+ def fail_action_permanently(data)
124
+ if data.include? "action_id"
125
+ # action = Action.get(data["action_id"])
126
+ #
127
+ # if action
128
+ # action.update_data({ :stats => nil, :errors => nil }, :errored)
129
+ # action.cleanup
130
+ # end
131
+ end
132
+ end
133
+
134
+
135
+ end
136
+ end
137
+ end
data/lib/herdst_worker/signals/facade.rb CHANGED
@@ -1,35 +1,35 @@
1
- module HerdstWorker
2
- module Signals
3
- class Facade
4
-
5
- def self.listen(process_path)
6
- write_process_file(process_path)
7
-
8
- # Start
9
- Signal.trap "USR1" do |x|
10
- HerdstWorker::Queue::Facade.start
11
- end
12
-
13
- # Halt
14
- Signal.trap "USR2" do |x|
15
- HerdstWorker::Queue::Facade.halt
16
- end
17
-
18
- # Stop (abort)
19
- Signal.trap "ABRT" do |x|
20
- HerdstWorker::Queue::Facade.stop
21
- end
22
- end
23
-
24
- private
25
- def self.write_process_file(process_path)
26
- # Write process id to file so we can signal the current process
27
- process_id = "#{$$}"
28
- process_file = process_path + "/process_id"
29
-
30
- File.open(process_file, "w") { |file| file.write(process_id) }
31
- end
32
-
33
- end
34
- end
35
- end
1
+ module HerdstWorker
2
+ module Signals
3
+ class Facade
4
+
5
+ def self.listen(process_path)
6
+ write_process_file(process_path)
7
+
8
+ # Start
9
+ Signal.trap "USR1" do |x|
10
+ HerdstWorker::Queue::Facade.start
11
+ end
12
+
13
+ # Halt
14
+ Signal.trap "USR2" do |x|
15
+ HerdstWorker::Queue::Facade.halt
16
+ end
17
+
18
+ # Stop (abort)
19
+ Signal.trap "ABRT" do |x|
20
+ HerdstWorker::Queue::Facade.stop
21
+ end
22
+ end
23
+
24
+ private
25
+ def self.write_process_file(process_path)
26
+ # Write process id to file so we can signal the current process
27
+ process_id = "#{$$}"
28
+ process_file = process_path + "/process_id"
29
+
30
+ File.open(process_file, "w") { |file| file.write(process_id) }
31
+ end
32
+
33
+ end
34
+ end
35
+ end
data/lib/herdst_worker/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module HerdstWorker
2
- VERSION = '0.1.4'
2
+ VERSION = '0.1.7'
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: herdst_worker
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.4
4
+ version: 0.1.7
5
5
  platform: ruby
6
6
  authors:
7
7
  - Herd.St
8
- autorequire:
8
+ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2021-07-05 00:00:00.000000000 Z
11
+ date: 2022-04-04 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: activesupport
@@ -250,7 +250,7 @@ licenses:
250
250
  - Nonstandard
251
251
  metadata:
252
252
  allowed_push_host: https://rubygems.org/
253
- post_install_message:
253
+ post_install_message:
254
254
  rdoc_options: []
255
255
  require_paths:
256
256
  - lib
@@ -265,8 +265,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
265
265
  - !ruby/object:Gem::Version
266
266
  version: '0'
267
267
  requirements: []
268
- rubygems_version: 3.2.20
269
- signing_key:
268
+ rubygems_version: 3.1.4
269
+ signing_key:
270
270
  specification_version: 4
271
271
  summary: Run background work triggered by aws sqs
272
272
  test_files: []