herdst_worker 0.1.4 → 0.1.7

Sign up to get free protection for your applications and to get access to all the features.
Files changed (15) hide show
  1. checksums.yaml +4 -4
  2. data/lib/herdst_worker/adapters/database.rb +0 -1
  3. data/lib/herdst_worker/adapters/facade.rb +20 -20
  4. data/lib/herdst_worker/adapters/sentry.rb +1 -1
  5. data/lib/herdst_worker/application/facade.rb +98 -89
  6. data/lib/herdst_worker/autoload/facade.rb +67 -67
  7. data/lib/herdst_worker/configuration/metadata.rb +5 -1
  8. data/lib/herdst_worker/configuration/paths.rb +41 -41
  9. data/lib/herdst_worker/log/facade.rb +87 -87
  10. data/lib/herdst_worker/queue/facade.rb +164 -162
  11. data/lib/herdst_worker/queue/processor.rb +225 -223
  12. data/lib/herdst_worker/queue/runner.rb +137 -137
  13. data/lib/herdst_worker/signals/facade.rb +35 -35
  14. data/lib/herdst_worker/version.rb +1 -1
  15. metadata +6 -6
data/lib/herdst_worker/queue/runner.rb CHANGED
@@ -1,137 +1,137 @@
1
- require 'concurrent'
2
-
3
-
4
- module HerdstWorker
5
- module Queue
6
- class Runner
7
-
8
-
9
- def process_message!(message, raw_message = nil, will_fail_permanently = false)
10
- sent_timestamp = raw_message.attributes.include?("SentTimestamp") ?
11
- raw_message.attributes["SentTimestamp"].to_i :
12
- nil
13
- trigger_timestamp = raw_message.message_attributes.include?("triggerTimestamp") ?
14
- raw_message.message_attributes["triggerTimestamp"]["string_value"].to_i :
15
- nil
16
- expiry = raw_message.message_attributes.include?("expiry") ?
17
- raw_message.message_attributes["expiry"]["string_value"].to_i :
18
- nil
19
-
20
- if expiry && (expiry > 0) && ((Time.now.utc.to_i * 1000) > expiry)
21
- self.app.logger.queue.info "Job has expired, not running. #{message.inspect}"
22
-
23
- return Concurrent::Promise.new {}
24
- end
25
-
26
- if message["Type"] != nil and message["Type"] == "Notification"
27
- if message["Message"].is_a? String
28
- message["Message"] = JSON.parse(message["Message"])
29
- end
30
-
31
- # Get the type
32
- if (message.include? "Subject") && (message["Subject"].include? "Elastic Transcoder")
33
- type = "Transcoder"
34
- else
35
- type = message["Message"]["notificationType"]
36
- end
37
-
38
- # Update the message with sent and triggered timestamp
39
- message["Message"]["sentTimestamp"] = sent_timestamp
40
- message["Message"]["triggerTimestamp"] = trigger_timestamp || sent_timestamp
41
-
42
- # Update the message with configuration Id
43
- message["Message"]["configurationId"] = "notification#{type}"
44
-
45
- # Since zips take a log time to process we might need to use:
46
- # poller.change_message_visibility_timeout(msg, 60)
47
- # To make sure other workers don't pick up the job
48
- return Concurrent::Promise.new {
49
- if !self.ignored_notifications.include? type
50
- execute_message!(nil, nil, message["Message"])
51
- end
52
- }
53
- end
54
-
55
- if message["Records"].is_a? Array
56
- execution_promise = nil
57
- execution_data = []
58
-
59
- message["Records"].each do |record|
60
- data_source = record["eventSource"].split(":")
61
- data_origin = data_source.first
62
- data_operation = data_source.last
63
- record_data = record[data_operation]
64
- company_id = nil
65
- user_id = nil
66
-
67
- # Update the message with sent and triggered timestamp
68
- record_data["sentTimestamp"] = sent_timestamp
69
- record_data["triggerTimestamp"] = trigger_timestamp || sent_timestamp
70
-
71
- execution_data << record_data
72
-
73
- if data_origin === "application" and record.include? "userIdentity"
74
- company_id = record["userIdentity"]["companyId"]
75
- user_id = record["userIdentity"]["principalId"]
76
- end
77
-
78
- if execution_promise == nil
79
- execution_promise = Concurrent::Promise.new {
80
- execute_message!(company_id, user_id, record_data)
81
- }
82
- else
83
- execution_promise = execution_promise.then {
84
- execute_message!(company_id, user_id, record_data)
85
- }
86
- end
87
- end
88
-
89
- return Concurrent::Promise.new {} if execution_promise == nil
90
- return execution_promise.rescue { |ex|
91
- execution_data.each do |data|
92
- fail_action_permanently(data) if will_fail_permanently
93
- end
94
-
95
- raise ex
96
- }
97
- end
98
-
99
- return Concurrent::Promise.new {}
100
- end
101
-
102
-
103
- def execute_message!(company_id, user_id, data)
104
- action = data["configurationId"]
105
- action_name = action.camelize
106
-
107
- unless self.app.config.actions["enabled"].include?(action_name)
108
- message = "Invalid action. #{action} is not an enabled action. Please add this action to the config file."
109
-
110
- if self.app.config.is_dev?
111
- raise message
112
- else
113
- Raven.capture_message(message)
114
- return
115
- end
116
- end
117
-
118
- action_name.constantize.send(:invoke, company_id, user_id, data)
119
- end
120
-
121
-
122
- protected
123
- def fail_action_permanently(data)
124
- if data.include? "action_id"
125
- # action = Action.get(data["action_id"])
126
- #
127
- # if action
128
- # action.update_data({ :stats => nil, :errors => nil }, :errored)
129
- # action.cleanup
130
- # end
131
- end
132
- end
133
-
134
-
135
- end
136
- end
137
- end
1
+ require 'concurrent'
2
+
3
+
4
+ module HerdstWorker
5
+ module Queue
6
+ class Runner
7
+
8
+
9
+ def process_message!(message, raw_message = nil, will_fail_permanently = false)
10
+ sent_timestamp = raw_message.attributes.include?("SentTimestamp") ?
11
+ raw_message.attributes["SentTimestamp"].to_i :
12
+ nil
13
+ trigger_timestamp = raw_message.message_attributes.include?("triggerTimestamp") ?
14
+ raw_message.message_attributes["triggerTimestamp"]["string_value"].to_i :
15
+ nil
16
+ expiry = raw_message.message_attributes.include?("expiry") ?
17
+ raw_message.message_attributes["expiry"]["string_value"].to_i :
18
+ nil
19
+
20
+ if expiry && (expiry > 0) && ((Time.now.utc.to_i * 1000) > expiry)
21
+ self.app.logger.queue.info "Job has expired, not running. #{message.inspect}"
22
+
23
+ return Concurrent::Promise.new {}
24
+ end
25
+
26
+ if message["Type"] != nil and message["Type"] == "Notification"
27
+ if message["Message"].is_a? String
28
+ message["Message"] = JSON.parse(message["Message"])
29
+ end
30
+
31
+ # Get the type
32
+ if (message.include? "Subject") && (message["Subject"].include? "Elastic Transcoder")
33
+ type = "Transcoder"
34
+ else
35
+ type = message["Message"]["notificationType"]
36
+ end
37
+
38
+ # Update the message with sent and triggered timestamp
39
+ message["Message"]["sentTimestamp"] = sent_timestamp
40
+ message["Message"]["triggerTimestamp"] = trigger_timestamp || sent_timestamp
41
+
42
+ # Update the message with configuration Id
43
+ message["Message"]["configurationId"] = "notification#{type}"
44
+
45
+ # Since zips take a log time to process we might need to use:
46
+ # poller.change_message_visibility_timeout(msg, 60)
47
+ # To make sure other workers don't pick up the job
48
+ return Concurrent::Promise.new {
49
+ if !self.ignored_notifications.include? type
50
+ execute_message!(nil, nil, message["Message"])
51
+ end
52
+ }
53
+ end
54
+
55
+ if message["Records"].is_a? Array
56
+ execution_promise = nil
57
+ execution_data = []
58
+
59
+ message["Records"].each do |record|
60
+ data_source = record["eventSource"].split(":")
61
+ data_origin = data_source.first
62
+ data_operation = data_source.last
63
+ record_data = record[data_operation]
64
+ company_id = nil
65
+ user_id = nil
66
+
67
+ # Update the message with sent and triggered timestamp
68
+ record_data["sentTimestamp"] = sent_timestamp
69
+ record_data["triggerTimestamp"] = trigger_timestamp || sent_timestamp
70
+
71
+ execution_data << record_data
72
+
73
+ if data_origin === "application" and record.include? "userIdentity"
74
+ company_id = record["userIdentity"]["companyId"]
75
+ user_id = record["userIdentity"]["principalId"]
76
+ end
77
+
78
+ if execution_promise == nil
79
+ execution_promise = Concurrent::Promise.new {
80
+ execute_message!(company_id, user_id, record_data)
81
+ }
82
+ else
83
+ execution_promise = execution_promise.then {
84
+ execute_message!(company_id, user_id, record_data)
85
+ }
86
+ end
87
+ end
88
+
89
+ return Concurrent::Promise.new {} if execution_promise == nil
90
+ return execution_promise.rescue { |ex|
91
+ execution_data.each do |data|
92
+ fail_action_permanently(data) if will_fail_permanently
93
+ end
94
+
95
+ raise ex
96
+ }
97
+ end
98
+
99
+ return Concurrent::Promise.new {}
100
+ end
101
+
102
+
103
+ def execute_message!(company_id, user_id, data)
104
+ action = data["configurationId"]
105
+ action_name = action.camelize
106
+
107
+ unless self.app.config.actions["enabled"].include?(action_name)
108
+ message = "Invalid action. #{action} is not an enabled action. Please add this action to the config file."
109
+
110
+ if self.app.config.is_dev?
111
+ raise message
112
+ else
113
+ Raven.capture_message(message)
114
+ return
115
+ end
116
+ end
117
+
118
+ action_name.constantize.send(:invoke, company_id, user_id, data)
119
+ end
120
+
121
+
122
+ protected
123
+ def fail_action_permanently(data)
124
+ if data.include? "action_id"
125
+ # action = Action.get(data["action_id"])
126
+ #
127
+ # if action
128
+ # action.update_data({ :stats => nil, :errors => nil }, :errored)
129
+ # action.cleanup
130
+ # end
131
+ end
132
+ end
133
+
134
+
135
+ end
136
+ end
137
+ end
data/lib/herdst_worker/signals/facade.rb CHANGED
@@ -1,35 +1,35 @@
1
- module HerdstWorker
2
- module Signals
3
- class Facade
4
-
5
- def self.listen(process_path)
6
- write_process_file(process_path)
7
-
8
- # Start
9
- Signal.trap "USR1" do |x|
10
- HerdstWorker::Queue::Facade.start
11
- end
12
-
13
- # Halt
14
- Signal.trap "USR2" do |x|
15
- HerdstWorker::Queue::Facade.halt
16
- end
17
-
18
- # Stop (abort)
19
- Signal.trap "ABRT" do |x|
20
- HerdstWorker::Queue::Facade.stop
21
- end
22
- end
23
-
24
- private
25
- def self.write_process_file(process_path)
26
- # Write process id to file so we can signal the current process
27
- process_id = "#{$$}"
28
- process_file = process_path + "/process_id"
29
-
30
- File.open(process_file, "w") { |file| file.write(process_id) }
31
- end
32
-
33
- end
34
- end
35
- end
1
+ module HerdstWorker
2
+ module Signals
3
+ class Facade
4
+
5
+ def self.listen(process_path)
6
+ write_process_file(process_path)
7
+
8
+ # Start
9
+ Signal.trap "USR1" do |x|
10
+ HerdstWorker::Queue::Facade.start
11
+ end
12
+
13
+ # Halt
14
+ Signal.trap "USR2" do |x|
15
+ HerdstWorker::Queue::Facade.halt
16
+ end
17
+
18
+ # Stop (abort)
19
+ Signal.trap "ABRT" do |x|
20
+ HerdstWorker::Queue::Facade.stop
21
+ end
22
+ end
23
+
24
+ private
25
+ def self.write_process_file(process_path)
26
+ # Write process id to file so we can signal the current process
27
+ process_id = "#{$$}"
28
+ process_file = process_path + "/process_id"
29
+
30
+ File.open(process_file, "w") { |file| file.write(process_id) }
31
+ end
32
+
33
+ end
34
+ end
35
+ end
data/lib/herdst_worker/version.rb CHANGED
@@ -1,3 +1,3 @@
1
1
  module HerdstWorker
2
- VERSION = '0.1.4'
2
+ VERSION = '0.1.7'
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: herdst_worker
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.4
4
+ version: 0.1.7
5
5
  platform: ruby
6
6
  authors:
7
7
  - Herd.St
8
- autorequire:
8
+ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2021-07-05 00:00:00.000000000 Z
11
+ date: 2022-04-04 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: activesupport
@@ -250,7 +250,7 @@ licenses:
250
250
  - Nonstandard
251
251
  metadata:
252
252
  allowed_push_host: https://rubygems.org/
253
- post_install_message:
253
+ post_install_message:
254
254
  rdoc_options: []
255
255
  require_paths:
256
256
  - lib
@@ -265,8 +265,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
265
265
  - !ruby/object:Gem::Version
266
266
  version: '0'
267
267
  requirements: []
268
- rubygems_version: 3.2.20
269
- signing_key:
268
+ rubygems_version: 3.1.4
269
+ signing_key:
270
270
  specification_version: 4
271
271
  summary: Run background work triggered by aws sqs
272
272
  test_files: []