fk-redis-sqspoller 0.1.1 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/bin/fk-redis-sqspoller +3 -3
- data/lib/sqspoller/message_delegator.rb +3 -5
- data/lib/sqspoller/queue_controller.rb +44 -21
- data/lib/sqspoller/sns_forwarder.rb +21 -0
- data/lib/sqspoller/sqs_poller.rb +127 -43
- data/lib/sqspoller/worker_task.rb +1 -4
- data/sqspoller.gemspec +3 -1
- metadata +16 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 2e565f6d7c27ea7b1400460f7b0579a39c6debb1
|
4
|
+
data.tar.gz: bf80d0b650f6870ad0e86c5ba4886f14ba31ffae
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: cb01da2c8db24e135c9de4e72bc913ad8107e52fe57a8737687685aadc7227fce334c05975cadbda011e9df385ec0ff023019f25ca331f1c39dc2b85de003804
|
7
|
+
data.tar.gz: a7d39a8b52e6e7d193f15b1a025d34e375c652adc2d79e675fed905ab4740bc272568c20e71a4dce2eb7999796ae088f94af732490f95ab3ddeab845bf4df068
|
data/bin/fk-redis-sqspoller
CHANGED
@@ -6,6 +6,6 @@ puts "SQSPoller arguments check"
|
|
6
6
|
ARGV.each do |arg|
|
7
7
|
puts "arg: #{arg}"
|
8
8
|
end
|
9
|
-
puts "
|
10
|
-
Sqspoller::SqsPoller.start_poller ARGV[0], ARGV[1], ARGV[2], ARGV[3], ARGV[4], ARGV[5]
|
11
|
-
puts "poller done"
|
9
|
+
puts "poller start"
|
10
|
+
Sqspoller::SqsPoller.start_poller ARGV[0], ARGV[1], ARGV[2], ARGV[3], ARGV[4], ARGV[5], ARGV[6]
|
11
|
+
puts "poller done"
|
@@ -28,23 +28,21 @@ module Sqspoller
|
|
28
28
|
@logger.info "Exiting wait state, connection_pool size reached below worker_thread_pool_size, pending_schedule_tasks=#{@pending_schedule_tasks}"
|
29
29
|
end
|
30
30
|
end
|
31
|
-
@logger.info "Scheduling worker task for message: #{message.message_id}"
|
32
31
|
|
33
32
|
begin
|
34
33
|
@connection_pool.post do
|
35
34
|
begin
|
36
|
-
@logger.info "Starting worker task for message: #{message.message_id}"
|
35
|
+
@logger.info " Starting worker task for message: #{message.message_id} on queue #{queue_name}"
|
37
36
|
@worker_task.process(message.body, message.message_id)
|
38
|
-
@logger.info "Finished worker task for message: #{message.message_id}"
|
39
37
|
queue_controller.delete_message message.receipt_handle
|
40
38
|
rescue Exception => e
|
41
|
-
@logger.info "Caught error for message: #{message}, error: #{e.message}, #{e.backtrace.join("\n")}"
|
39
|
+
@logger.info " Caught error for message: #{message}, error: #{e.message}, #{e.backtrace.join("\n")}"
|
42
40
|
end
|
43
41
|
@pending_schedule_tasks -= 1
|
44
42
|
end
|
45
43
|
rescue Concurrent::RejectedExecutionError => e
|
46
44
|
@pending_schedule_tasks -= 1
|
47
|
-
@logger.info "Caught Concurrent::RejectedExecutionError for #{e.message}"
|
45
|
+
@logger.info " Caught Concurrent::RejectedExecutionError for #{e.message} on queue #{queue_name}"
|
48
46
|
end
|
49
47
|
end
|
50
48
|
end
|
@@ -1,21 +1,35 @@
|
|
1
1
|
require "logger"
|
2
2
|
require "aws-sdk"
|
3
|
+
require 'base64'
|
4
|
+
require 'openssl'
|
3
5
|
|
4
6
|
module Sqspoller
|
5
7
|
class QueueController
|
6
8
|
|
7
|
-
|
9
|
+
# expecting the queue name to be <environment>_outbound_messages_<company_id>
|
10
|
+
REGEXP = /(?<environment>\w+)-outbound-messages-(?<window_identifier>[\w-]+)/
|
8
11
|
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
@
|
14
|
-
|
15
|
-
|
12
|
+
attr_accessor :threads,
|
13
|
+
:queue_name
|
14
|
+
|
15
|
+
def initialize args
|
16
|
+
@logger = args[:logger]
|
17
|
+
self.queue_name = args[:queue_name]
|
18
|
+
@polling_threads_count = args[:polling_threads_count]
|
19
|
+
@sqs = Aws::SQS::Client.new access_key_id: args[:access_key_id],
|
20
|
+
secret_access_key: args[:secret_access_key],
|
21
|
+
region: args[:region]
|
16
22
|
@queue_details = @sqs.get_queue_url(queue_name: queue_name)
|
17
23
|
self.threads = []
|
18
|
-
@task_delegator = task_delegator
|
24
|
+
@task_delegator = args[:task_delegator]
|
25
|
+
match_data = REGEXP.match queue_name
|
26
|
+
@maintenance_window = if match_data
|
27
|
+
@cache_key = "#{match_data[:environment]}::OutboundMessages::MaintenanceWindowOpen::#{match_data[:window_identifier]}"
|
28
|
+
@window_identifier = match_data[:window_identifier]
|
29
|
+
true
|
30
|
+
else
|
31
|
+
false
|
32
|
+
end
|
19
33
|
end
|
20
34
|
|
21
35
|
def start
|
@@ -28,23 +42,19 @@ module Sqspoller
|
|
28
42
|
|
29
43
|
def start_thread(queue_url)
|
30
44
|
Thread.new do
|
31
|
-
@logger.info "Poller thread started for queue: #{queue_url}"
|
32
45
|
poller = Aws::SQS::QueuePoller.new(queue_url)
|
46
|
+
poller.before_request do |stats|
|
47
|
+
block_on_maintenance_window
|
48
|
+
end
|
33
49
|
|
34
50
|
loop do
|
35
|
-
@logger.info "Polling queue #{
|
36
|
-
|
37
|
-
msgs = @sqs.receive_message :queue_url => queue_url
|
38
|
-
rescue Exception => e
|
39
|
-
@logger.info "Error receiving messages from queue #{@queue_name}: #{e.message}"
|
40
|
-
next
|
41
|
-
end
|
42
|
-
msgs.messages.each do |received_message|
|
51
|
+
@logger.info " Polling queue #{queue_name} for messages"
|
52
|
+
poller.poll do |received_message|
|
43
53
|
begin
|
44
|
-
@
|
45
|
-
@task_delegator.process self, received_message, @queue_name
|
54
|
+
@task_delegator.process self, received_message, queue_name
|
46
55
|
rescue Exception => e
|
47
|
-
@logger.info "Encountered error #{e.message} while submitting message from queue #{
|
56
|
+
@logger.info " Encountered error #{e.message} while submitting message to worker from queue #{queue_name}"
|
57
|
+
throw :skip_delete
|
48
58
|
end
|
49
59
|
end
|
50
60
|
end
|
@@ -56,5 +66,18 @@ module Sqspoller
|
|
56
66
|
receipt_handle: receipt_handle
|
57
67
|
end
|
58
68
|
|
69
|
+
def block_on_maintenance_window
|
70
|
+
if @maintenance_window
|
71
|
+
loop do
|
72
|
+
window_open = REDIS.get @cache_key
|
73
|
+
if window_open
|
74
|
+
@logger.info " Maintenance Window is open for #{@window_identifier}, sleeping for 5 minutes"
|
75
|
+
sleep 300
|
76
|
+
else
|
77
|
+
break
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
59
82
|
end
|
60
83
|
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
require 'aws-sdk'
|
2
|
+
|
3
|
+
module Sqspoller
|
4
|
+
class SnsForwarder
|
5
|
+
def initialize args
|
6
|
+
@sns = Aws::SNS::Client.new access_key_id: args[:access_key_id],
|
7
|
+
secret_access_key: args[:secret_access_key],
|
8
|
+
region: args[:region]
|
9
|
+
@topic_arn = args[:sns_topic_arn]
|
10
|
+
@logger = args[:logger]
|
11
|
+
@logger.info " Inializing SnsForwarder with SNS topic arn: #{@topic_arn}"
|
12
|
+
end
|
13
|
+
|
14
|
+
def process message, message_id
|
15
|
+
@logger.info " Processing message"
|
16
|
+
@logger.info " Publishing to #{@topic_arn}"
|
17
|
+
response = @sns.publish topic_arn: @topic_arn, message: message
|
18
|
+
@logger.info " SNS response message id: #{response.message_id}"
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
data/lib/sqspoller/sqs_poller.rb
CHANGED
@@ -3,11 +3,17 @@ require "concurrent"
|
|
3
3
|
require "net/http"
|
4
4
|
require "yaml"
|
5
5
|
require "erb"
|
6
|
+
require 'redis'
|
6
7
|
require "sqspoller/worker_task"
|
8
|
+
require "sqspoller/sns_forwarder"
|
7
9
|
require "sqspoller/message_delegator"
|
8
10
|
require "sqspoller/queue_controller"
|
9
11
|
|
10
12
|
module Sqspoller
|
13
|
+
REDIS = Redis.new host: ENV['REDIS_HOST'], port: ENV['REDIS_PORT']
|
14
|
+
HEADERS = { 'Content-Type' => 'application/json',
|
15
|
+
'Accept' => 'application/json'
|
16
|
+
}
|
11
17
|
|
12
18
|
class SqsPoller
|
13
19
|
class << self
|
@@ -42,70 +48,148 @@ module Sqspoller
|
|
42
48
|
STDERR.reopen STDOUT
|
43
49
|
end
|
44
50
|
|
45
|
-
def
|
46
|
-
puts "
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
51
|
+
def start_poller content_name, queue_config_name, access_key_id, secret_access_key, region, log_filename=nil, redis_or_file=false
|
52
|
+
puts "StartPoller"
|
53
|
+
|
54
|
+
poller_args = {content_name: content_name,
|
55
|
+
queue_config_name: queue_config_name.to_sym,
|
56
|
+
access_key_id: access_key_id,
|
57
|
+
secret_access_key: secret_access_key,
|
58
|
+
region: region,
|
59
|
+
log_filename: log_filename,
|
60
|
+
redis_or_file: redis_or_file
|
61
|
+
}
|
62
|
+
|
63
|
+
if redis_or_file
|
64
|
+
if REDIS.get('random_key') == nil
|
65
|
+
puts "Able to connect to Redis"
|
58
66
|
else
|
59
|
-
|
60
|
-
|
61
|
-
queues_config[queue][:polling_threads],
|
62
|
-
message_delegator,
|
63
|
-
access_key_id,
|
64
|
-
secret_access_key,
|
65
|
-
region,
|
66
|
-
logger_file
|
67
|
-
qc.start
|
68
|
-
qc.threads.each do |thread|
|
69
|
-
thread.join
|
70
|
-
end
|
67
|
+
puts "*** Unable to connect to Redis"
|
68
|
+
exit -1
|
71
69
|
end
|
72
70
|
end
|
71
|
+
if log_filename.nil? || log_filename.empty?
|
72
|
+
poller_args[:log_filename] = STDOUT
|
73
|
+
puts "Did not receive log file name"
|
74
|
+
fork do
|
75
|
+
Process.daemon nil, :noclose
|
76
|
+
start_queues_with_config poller_args
|
77
|
+
end
|
78
|
+
else
|
79
|
+
puts "Did receive log file name"
|
80
|
+
daemonize log_filename
|
81
|
+
start_queues_with_config poller_args
|
82
|
+
end
|
73
83
|
end
|
74
84
|
|
75
|
-
def
|
85
|
+
def load_config_from_file filename
|
76
86
|
content = IO.read filename
|
77
|
-
|
87
|
+
symbolize_from_content content
|
88
|
+
end
|
89
|
+
|
90
|
+
def load_config_from_redis redis_key
|
91
|
+
content = REDIS.get redis_key
|
92
|
+
symbolize_from_content content
|
78
93
|
end
|
79
94
|
|
80
|
-
def
|
81
|
-
puts "Starting poller"
|
95
|
+
def symbolize_from_content content
|
82
96
|
erb = ERB.new content
|
83
|
-
|
84
|
-
|
97
|
+
yaml = YAML.load erb.result
|
98
|
+
symbolize yaml
|
99
|
+
end
|
85
100
|
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
101
|
+
def start_queues_with_config poller_args
|
102
|
+
@logger = Logger.new(poller_args[:log_filename])
|
103
|
+
poller_args[:logger] = @logger
|
104
|
+
@logger.info "Get config"
|
105
|
+
config = if poller_args[:redis_or_file]
|
106
|
+
load_config_from_redis poller_args[:content_name]
|
107
|
+
else
|
108
|
+
load_config_from_file poller_args[:content_name]
|
109
|
+
end
|
110
|
+
@logger.info "Config: #{config.inspect}"
|
111
|
+
queues_config = config[poller_args[:queue_config_name]]
|
112
|
+
if queues_config
|
113
|
+
@logger.info "QueuesConfig: #{queues_config.inspect}"
|
114
|
+
else
|
115
|
+
@logger.error "Unable to fetch Queue Config"
|
116
|
+
end
|
117
|
+
|
118
|
+
@logger.info "Started poller method"
|
119
|
+
|
120
|
+
message_delegator = worker_pool_init config[:worker_configuration].merge(poller_args), queues_config, poller_args[:log_filename]
|
121
|
+
|
122
|
+
if poller_args[:redis_or_file]
|
123
|
+
start_all_queues_with_refresh queues_config, message_delegator, poller_args, config[:worker_configuration][:refresh_interval_in_seconds] || 3600
|
124
|
+
else
|
125
|
+
start_all_queues queues_config, message_delegator, poller_args
|
126
|
+
end
|
127
|
+
end
|
128
|
+
|
129
|
+
def start_all_queues_with_refresh queues_config, message_delegator, poller_args, refresh_interval
|
130
|
+
@logger.info "Start all queues with refresh"
|
131
|
+
queues = {}
|
132
|
+
loop do
|
133
|
+
Thread.pass
|
134
|
+
queue_names = queues_config.keys
|
135
|
+
queues_config.keys.each do |queue|
|
136
|
+
@logger.info " Checking queue #{queue}"
|
137
|
+
if queues[queue]
|
138
|
+
@logger.info " Queue: #{queue} not created, already initialized and running"
|
139
|
+
else
|
140
|
+
queues[queue] = start_queue_controller queues_config, queue, message_delegator, poller_args
|
141
|
+
end
|
91
142
|
end
|
143
|
+
@logger.info " Done creating queues, sleeping for #{refresh_interval} seconds"
|
144
|
+
sleep refresh_interval
|
145
|
+
@logger.info " Refreshing config"
|
146
|
+
config = load_config_from_redis poller_args[:content_name]
|
147
|
+
queues_config = config[poller_args[:queue_config_name]]
|
148
|
+
end
|
149
|
+
end
|
150
|
+
|
151
|
+
def start_all_queues queues_config, message_delegator, poller_args
|
152
|
+
@logger.info "Start all queues "
|
153
|
+
queues_config.keys.each do |queue|
|
154
|
+
start_queue_controller queues_config, queue, message_delegator, poller_args
|
155
|
+
end
|
156
|
+
end
|
157
|
+
|
158
|
+
def worker_pool_init worker_config, queues_config, logger_file
|
159
|
+
total_poller_threads = queues_config.keys.reduce(0) do |sum, queue|
|
160
|
+
sum += queues_config[queue][:polling_threads]
|
161
|
+
end
|
162
|
+
initialize_worker worker_config, total_poller_threads, logger_file
|
163
|
+
end
|
164
|
+
|
165
|
+
def start_queue_controller queues_config, queue, message_delegator, poller_args
|
166
|
+
if queues_config[queue][:polling_threads] == 0
|
167
|
+
@logger.info " Polling disabled for queue: #{queue}"
|
168
|
+
nil
|
92
169
|
else
|
93
|
-
|
94
|
-
|
95
|
-
|
170
|
+
@logger.info " Creating QueueController object for queue: #{queue}"
|
171
|
+
qc = QueueController.new queue_name: queue,
|
172
|
+
polling_threads_count: queues_config[queue][:polling_threads],
|
173
|
+
task_delegator: message_delegator,
|
174
|
+
access_key_id: poller_args[:access_key_id],
|
175
|
+
secret_access_key: poller_args[:secret_access_key],
|
176
|
+
region: poller_args[:region],
|
177
|
+
logger_file: poller_args[:log_filename],
|
178
|
+
logger: poller_args[:logger]
|
179
|
+
qc.start
|
180
|
+
qc
|
96
181
|
end
|
97
182
|
end
|
98
183
|
|
99
|
-
def initialize_worker
|
184
|
+
def initialize_worker worker_configuration, total_poller_threads, logger_file
|
100
185
|
worker_thread_count = worker_configuration[:concurrency] || total_poller_threads
|
101
186
|
waiting_tasks_ratio = worker_configuration[:waiting_tasks_ratio] || 1
|
102
187
|
|
103
|
-
klass = worker_configuration[:worker_class].split('::').
|
104
|
-
worker_task = klass.new
|
188
|
+
klass = worker_configuration[:worker_class].split('::').reduce(Object, :const_get)
|
189
|
+
worker_task = klass.new worker_configuration
|
105
190
|
|
106
191
|
MessageDelegator.new worker_thread_count, waiting_tasks_ratio, worker_task, logger_file
|
107
192
|
end
|
108
193
|
end
|
109
194
|
end
|
110
|
-
|
111
195
|
end
|
@@ -4,14 +4,11 @@ require "rest-client"
|
|
4
4
|
module Sqspoller
|
5
5
|
class WorkerTask
|
6
6
|
|
7
|
-
HEADERS = { 'Content-Type' => 'application/json',
|
8
|
-
'Accept' => 'application/json'
|
9
|
-
}
|
10
7
|
ALLOWED_METHODS = { 'post' => :post,
|
11
8
|
'get' => :get
|
12
9
|
}
|
13
10
|
|
14
|
-
def initialize
|
11
|
+
def initialize worker_configuration
|
15
12
|
@http_method = ALLOWED_METHODS[worker_configuration[:http_method].downcase]
|
16
13
|
@http_url = worker_configuration[:http_url]
|
17
14
|
@timeout = worker_configuration[:timeout] && worker_configuration[:timeout].to_i || 450
|
data/sqspoller.gemspec
CHANGED
@@ -5,7 +5,7 @@ require 'sqspoller/version'
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |spec|
|
7
7
|
spec.name = "fk-redis-sqspoller"
|
8
|
-
spec.version = '0.1.
|
8
|
+
spec.version = '0.1.2'
|
9
9
|
spec.authors = ["FourKites"]
|
10
10
|
spec.email = ["developer@fourkites.com "]
|
11
11
|
|
@@ -24,5 +24,7 @@ Gem::Specification.new do |spec|
|
|
24
24
|
spec.add_runtime_dependency "concurrent-ruby"
|
25
25
|
spec.add_runtime_dependency "rest-client"
|
26
26
|
spec.add_runtime_dependency 'aws-sdk', '~> 2'
|
27
|
+
spec.add_runtime_dependency 'redis'
|
28
|
+
|
27
29
|
|
28
30
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fk-redis-sqspoller
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- FourKites
|
@@ -80,6 +80,20 @@ dependencies:
|
|
80
80
|
- - "~>"
|
81
81
|
- !ruby/object:Gem::Version
|
82
82
|
version: '2'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: redis
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - ">="
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '0'
|
90
|
+
type: :runtime
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - ">="
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '0'
|
83
97
|
description: ''
|
84
98
|
email:
|
85
99
|
- 'developer@fourkites.com '
|
@@ -99,6 +113,7 @@ files:
|
|
99
113
|
- lib/sqspoller.rb
|
100
114
|
- lib/sqspoller/message_delegator.rb
|
101
115
|
- lib/sqspoller/queue_controller.rb
|
116
|
+
- lib/sqspoller/sns_forwarder.rb
|
102
117
|
- lib/sqspoller/sqs_poller.rb
|
103
118
|
- lib/sqspoller/version.rb
|
104
119
|
- lib/sqspoller/worker_task.rb
|