fk-redis-sqspoller 0.1.1 → 0.1.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/bin/fk-redis-sqspoller +3 -3
- data/lib/sqspoller/message_delegator.rb +3 -5
- data/lib/sqspoller/queue_controller.rb +44 -21
- data/lib/sqspoller/sns_forwarder.rb +21 -0
- data/lib/sqspoller/sqs_poller.rb +127 -43
- data/lib/sqspoller/worker_task.rb +1 -4
- data/sqspoller.gemspec +3 -1
- metadata +16 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 2e565f6d7c27ea7b1400460f7b0579a39c6debb1
|
4
|
+
data.tar.gz: bf80d0b650f6870ad0e86c5ba4886f14ba31ffae
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: cb01da2c8db24e135c9de4e72bc913ad8107e52fe57a8737687685aadc7227fce334c05975cadbda011e9df385ec0ff023019f25ca331f1c39dc2b85de003804
|
7
|
+
data.tar.gz: a7d39a8b52e6e7d193f15b1a025d34e375c652adc2d79e675fed905ab4740bc272568c20e71a4dce2eb7999796ae088f94af732490f95ab3ddeab845bf4df068
|
data/bin/fk-redis-sqspoller
CHANGED
@@ -6,6 +6,6 @@ puts "SQSPoller arguments check"
|
|
6
6
|
ARGV.each do |arg|
|
7
7
|
puts "arg: #{arg}"
|
8
8
|
end
|
9
|
-
puts "
|
10
|
-
Sqspoller::SqsPoller.start_poller ARGV[0], ARGV[1], ARGV[2], ARGV[3], ARGV[4], ARGV[5]
|
11
|
-
puts "poller done"
|
9
|
+
puts "poller start"
|
10
|
+
Sqspoller::SqsPoller.start_poller ARGV[0], ARGV[1], ARGV[2], ARGV[3], ARGV[4], ARGV[5], ARGV[6]
|
11
|
+
puts "poller done"
|
@@ -28,23 +28,21 @@ module Sqspoller
|
|
28
28
|
@logger.info "Exiting wait state, connection_pool size reached below worker_thread_pool_size, pending_schedule_tasks=#{@pending_schedule_tasks}"
|
29
29
|
end
|
30
30
|
end
|
31
|
-
@logger.info "Scheduling worker task for message: #{message.message_id}"
|
32
31
|
|
33
32
|
begin
|
34
33
|
@connection_pool.post do
|
35
34
|
begin
|
36
|
-
@logger.info "Starting worker task for message: #{message.message_id}"
|
35
|
+
@logger.info " Starting worker task for message: #{message.message_id} on queue #{queue_name}"
|
37
36
|
@worker_task.process(message.body, message.message_id)
|
38
|
-
@logger.info "Finished worker task for message: #{message.message_id}"
|
39
37
|
queue_controller.delete_message message.receipt_handle
|
40
38
|
rescue Exception => e
|
41
|
-
@logger.info "Caught error for message: #{message}, error: #{e.message}, #{e.backtrace.join("\n")}"
|
39
|
+
@logger.info " Caught error for message: #{message}, error: #{e.message}, #{e.backtrace.join("\n")}"
|
42
40
|
end
|
43
41
|
@pending_schedule_tasks -= 1
|
44
42
|
end
|
45
43
|
rescue Concurrent::RejectedExecutionError => e
|
46
44
|
@pending_schedule_tasks -= 1
|
47
|
-
@logger.info "Caught Concurrent::RejectedExecutionError for #{e.message}"
|
45
|
+
@logger.info " Caught Concurrent::RejectedExecutionError for #{e.message} on queue #{queue_name}"
|
48
46
|
end
|
49
47
|
end
|
50
48
|
end
|
@@ -1,21 +1,35 @@
|
|
1
1
|
require "logger"
|
2
2
|
require "aws-sdk"
|
3
|
+
require 'base64'
|
4
|
+
require 'openssl'
|
3
5
|
|
4
6
|
module Sqspoller
|
5
7
|
class QueueController
|
6
8
|
|
7
|
-
|
9
|
+
# expecting the queue name to be <environment>_outbound_messages_<company_id>
|
10
|
+
REGEXP = /(?<environment>\w+)-outbound-messages-(?<window_identifier>[\w-]+)/
|
8
11
|
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
@
|
14
|
-
|
15
|
-
|
12
|
+
attr_accessor :threads,
|
13
|
+
:queue_name
|
14
|
+
|
15
|
+
def initialize args
|
16
|
+
@logger = args[:logger]
|
17
|
+
self.queue_name = args[:queue_name]
|
18
|
+
@polling_threads_count = args[:polling_threads_count]
|
19
|
+
@sqs = Aws::SQS::Client.new access_key_id: args[:access_key_id],
|
20
|
+
secret_access_key: args[:secret_access_key],
|
21
|
+
region: args[:region]
|
16
22
|
@queue_details = @sqs.get_queue_url(queue_name: queue_name)
|
17
23
|
self.threads = []
|
18
|
-
@task_delegator = task_delegator
|
24
|
+
@task_delegator = args[:task_delegator]
|
25
|
+
match_data = REGEXP.match queue_name
|
26
|
+
@maintenance_window = if match_data
|
27
|
+
@cache_key = "#{match_data[:environment]}::OutboundMessages::MaintenanceWindowOpen::#{match_data[:window_identifier]}"
|
28
|
+
@window_identifier = match_data[:window_identifier]
|
29
|
+
true
|
30
|
+
else
|
31
|
+
false
|
32
|
+
end
|
19
33
|
end
|
20
34
|
|
21
35
|
def start
|
@@ -28,23 +42,19 @@ module Sqspoller
|
|
28
42
|
|
29
43
|
def start_thread(queue_url)
|
30
44
|
Thread.new do
|
31
|
-
@logger.info "Poller thread started for queue: #{queue_url}"
|
32
45
|
poller = Aws::SQS::QueuePoller.new(queue_url)
|
46
|
+
poller.before_request do |stats|
|
47
|
+
block_on_maintenance_window
|
48
|
+
end
|
33
49
|
|
34
50
|
loop do
|
35
|
-
@logger.info "Polling queue #{
|
36
|
-
|
37
|
-
msgs = @sqs.receive_message :queue_url => queue_url
|
38
|
-
rescue Exception => e
|
39
|
-
@logger.info "Error receiving messages from queue #{@queue_name}: #{e.message}"
|
40
|
-
next
|
41
|
-
end
|
42
|
-
msgs.messages.each do |received_message|
|
51
|
+
@logger.info " Polling queue #{queue_name} for messages"
|
52
|
+
poller.poll do |received_message|
|
43
53
|
begin
|
44
|
-
@
|
45
|
-
@task_delegator.process self, received_message, @queue_name
|
54
|
+
@task_delegator.process self, received_message, queue_name
|
46
55
|
rescue Exception => e
|
47
|
-
@logger.info "Encountered error #{e.message} while submitting message from queue #{
|
56
|
+
@logger.info " Encountered error #{e.message} while submitting message to worker from queue #{queue_name}"
|
57
|
+
throw :skip_delete
|
48
58
|
end
|
49
59
|
end
|
50
60
|
end
|
@@ -56,5 +66,18 @@ module Sqspoller
|
|
56
66
|
receipt_handle: receipt_handle
|
57
67
|
end
|
58
68
|
|
69
|
+
def block_on_maintenance_window
|
70
|
+
if @maintenance_window
|
71
|
+
loop do
|
72
|
+
window_open = REDIS.get @cache_key
|
73
|
+
if window_open
|
74
|
+
@logger.info " Maintenance Window is open for #{@window_identifier}, sleeping for 5 minutes"
|
75
|
+
sleep 300
|
76
|
+
else
|
77
|
+
break
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
end
|
59
82
|
end
|
60
83
|
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
require 'aws-sdk'
|
2
|
+
|
3
|
+
module Sqspoller
|
4
|
+
class SnsForwarder
|
5
|
+
def initialize args
|
6
|
+
@sns = Aws::SNS::Client.new access_key_id: args[:access_key_id],
|
7
|
+
secret_access_key: args[:secret_access_key],
|
8
|
+
region: args[:region]
|
9
|
+
@topic_arn = args[:sns_topic_arn]
|
10
|
+
@logger = args[:logger]
|
11
|
+
@logger.info " Inializing SnsForwarder with SNS topic arn: #{@topic_arn}"
|
12
|
+
end
|
13
|
+
|
14
|
+
def process message, message_id
|
15
|
+
@logger.info " Processing message"
|
16
|
+
@logger.info " Publishing to #{@topic_arn}"
|
17
|
+
response = @sns.publish topic_arn: @topic_arn, message: message
|
18
|
+
@logger.info " SNS response message id: #{response.message_id}"
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
data/lib/sqspoller/sqs_poller.rb
CHANGED
@@ -3,11 +3,17 @@ require "concurrent"
|
|
3
3
|
require "net/http"
|
4
4
|
require "yaml"
|
5
5
|
require "erb"
|
6
|
+
require 'redis'
|
6
7
|
require "sqspoller/worker_task"
|
8
|
+
require "sqspoller/sns_forwarder"
|
7
9
|
require "sqspoller/message_delegator"
|
8
10
|
require "sqspoller/queue_controller"
|
9
11
|
|
10
12
|
module Sqspoller
|
13
|
+
REDIS = Redis.new host: ENV['REDIS_HOST'], port: ENV['REDIS_PORT']
|
14
|
+
HEADERS = { 'Content-Type' => 'application/json',
|
15
|
+
'Accept' => 'application/json'
|
16
|
+
}
|
11
17
|
|
12
18
|
class SqsPoller
|
13
19
|
class << self
|
@@ -42,70 +48,148 @@ module Sqspoller
|
|
42
48
|
STDERR.reopen STDOUT
|
43
49
|
end
|
44
50
|
|
45
|
-
def
|
46
|
-
puts "
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
51
|
+
def start_poller content_name, queue_config_name, access_key_id, secret_access_key, region, log_filename=nil, redis_or_file=false
|
52
|
+
puts "StartPoller"
|
53
|
+
|
54
|
+
poller_args = {content_name: content_name,
|
55
|
+
queue_config_name: queue_config_name.to_sym,
|
56
|
+
access_key_id: access_key_id,
|
57
|
+
secret_access_key: secret_access_key,
|
58
|
+
region: region,
|
59
|
+
log_filename: log_filename,
|
60
|
+
redis_or_file: redis_or_file
|
61
|
+
}
|
62
|
+
|
63
|
+
if redis_or_file
|
64
|
+
if REDIS.get('random_key') == nil
|
65
|
+
puts "Able to connect to Redis"
|
58
66
|
else
|
59
|
-
|
60
|
-
|
61
|
-
queues_config[queue][:polling_threads],
|
62
|
-
message_delegator,
|
63
|
-
access_key_id,
|
64
|
-
secret_access_key,
|
65
|
-
region,
|
66
|
-
logger_file
|
67
|
-
qc.start
|
68
|
-
qc.threads.each do |thread|
|
69
|
-
thread.join
|
70
|
-
end
|
67
|
+
puts "*** Unable to connect to Redis"
|
68
|
+
exit -1
|
71
69
|
end
|
72
70
|
end
|
71
|
+
if log_filename.nil? || log_filename.empty?
|
72
|
+
poller_args[:log_filename] = STDOUT
|
73
|
+
puts "Did not receive log file name"
|
74
|
+
fork do
|
75
|
+
Process.daemon nil, :noclose
|
76
|
+
start_queues_with_config poller_args
|
77
|
+
end
|
78
|
+
else
|
79
|
+
puts "Did receive log file name"
|
80
|
+
daemonize log_filename
|
81
|
+
start_queues_with_config poller_args
|
82
|
+
end
|
73
83
|
end
|
74
84
|
|
75
|
-
def
|
85
|
+
def load_config_from_file filename
|
76
86
|
content = IO.read filename
|
77
|
-
|
87
|
+
symbolize_from_content content
|
88
|
+
end
|
89
|
+
|
90
|
+
def load_config_from_redis redis_key
|
91
|
+
content = REDIS.get redis_key
|
92
|
+
symbolize_from_content content
|
78
93
|
end
|
79
94
|
|
80
|
-
def
|
81
|
-
puts "Starting poller"
|
95
|
+
def symbolize_from_content content
|
82
96
|
erb = ERB.new content
|
83
|
-
|
84
|
-
|
97
|
+
yaml = YAML.load erb.result
|
98
|
+
symbolize yaml
|
99
|
+
end
|
85
100
|
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
101
|
+
def start_queues_with_config poller_args
|
102
|
+
@logger = Logger.new(poller_args[:log_filename])
|
103
|
+
poller_args[:logger] = @logger
|
104
|
+
@logger.info "Get config"
|
105
|
+
config = if poller_args[:redis_or_file]
|
106
|
+
load_config_from_redis poller_args[:content_name]
|
107
|
+
else
|
108
|
+
load_config_from_file poller_args[:content_name]
|
109
|
+
end
|
110
|
+
@logger.info "Config: #{config.inspect}"
|
111
|
+
queues_config = config[poller_args[:queue_config_name]]
|
112
|
+
if queues_config
|
113
|
+
@logger.info "QueuesConfig: #{queues_config.inspect}"
|
114
|
+
else
|
115
|
+
@logger.error "Unable to fetch Queue Config"
|
116
|
+
end
|
117
|
+
|
118
|
+
@logger.info "Started poller method"
|
119
|
+
|
120
|
+
message_delegator = worker_pool_init config[:worker_configuration].merge(poller_args), queues_config, poller_args[:log_filename]
|
121
|
+
|
122
|
+
if poller_args[:redis_or_file]
|
123
|
+
start_all_queues_with_refresh queues_config, message_delegator, poller_args, config[:worker_configuration][:refresh_interval_in_seconds] || 3600
|
124
|
+
else
|
125
|
+
start_all_queues queues_config, message_delegator, poller_args
|
126
|
+
end
|
127
|
+
end
|
128
|
+
|
129
|
+
def start_all_queues_with_refresh queues_config, message_delegator, poller_args, refresh_interval
|
130
|
+
@logger.info "Start all queues with refresh"
|
131
|
+
queues = {}
|
132
|
+
loop do
|
133
|
+
Thread.pass
|
134
|
+
queue_names = queues_config.keys
|
135
|
+
queues_config.keys.each do |queue|
|
136
|
+
@logger.info " Checking queue #{queue}"
|
137
|
+
if queues[queue]
|
138
|
+
@logger.info " Queue: #{queue} not created, already initialized and running"
|
139
|
+
else
|
140
|
+
queues[queue] = start_queue_controller queues_config, queue, message_delegator, poller_args
|
141
|
+
end
|
91
142
|
end
|
143
|
+
@logger.info " Done creating queues, sleeping for #{refresh_interval} seconds"
|
144
|
+
sleep refresh_interval
|
145
|
+
@logger.info " Refreshing config"
|
146
|
+
config = load_config_from_redis poller_args[:content_name]
|
147
|
+
queues_config = config[poller_args[:queue_config_name]]
|
148
|
+
end
|
149
|
+
end
|
150
|
+
|
151
|
+
def start_all_queues queues_config, message_delegator, poller_args
|
152
|
+
@logger.info "Start all queues "
|
153
|
+
queues_config.keys.each do |queue|
|
154
|
+
start_queue_controller queues_config, queue, message_delegator, poller_args
|
155
|
+
end
|
156
|
+
end
|
157
|
+
|
158
|
+
def worker_pool_init worker_config, queues_config, logger_file
|
159
|
+
total_poller_threads = queues_config.keys.reduce(0) do |sum, queue|
|
160
|
+
sum += queues_config[queue][:polling_threads]
|
161
|
+
end
|
162
|
+
initialize_worker worker_config, total_poller_threads, logger_file
|
163
|
+
end
|
164
|
+
|
165
|
+
def start_queue_controller queues_config, queue, message_delegator, poller_args
|
166
|
+
if queues_config[queue][:polling_threads] == 0
|
167
|
+
@logger.info " Polling disabled for queue: #{queue}"
|
168
|
+
nil
|
92
169
|
else
|
93
|
-
|
94
|
-
|
95
|
-
|
170
|
+
@logger.info " Creating QueueController object for queue: #{queue}"
|
171
|
+
qc = QueueController.new queue_name: queue,
|
172
|
+
polling_threads_count: queues_config[queue][:polling_threads],
|
173
|
+
task_delegator: message_delegator,
|
174
|
+
access_key_id: poller_args[:access_key_id],
|
175
|
+
secret_access_key: poller_args[:secret_access_key],
|
176
|
+
region: poller_args[:region],
|
177
|
+
logger_file: poller_args[:log_filename],
|
178
|
+
logger: poller_args[:logger]
|
179
|
+
qc.start
|
180
|
+
qc
|
96
181
|
end
|
97
182
|
end
|
98
183
|
|
99
|
-
def initialize_worker
|
184
|
+
def initialize_worker worker_configuration, total_poller_threads, logger_file
|
100
185
|
worker_thread_count = worker_configuration[:concurrency] || total_poller_threads
|
101
186
|
waiting_tasks_ratio = worker_configuration[:waiting_tasks_ratio] || 1
|
102
187
|
|
103
|
-
klass = worker_configuration[:worker_class].split('::').
|
104
|
-
worker_task = klass.new
|
188
|
+
klass = worker_configuration[:worker_class].split('::').reduce(Object, :const_get)
|
189
|
+
worker_task = klass.new worker_configuration
|
105
190
|
|
106
191
|
MessageDelegator.new worker_thread_count, waiting_tasks_ratio, worker_task, logger_file
|
107
192
|
end
|
108
193
|
end
|
109
194
|
end
|
110
|
-
|
111
195
|
end
|
@@ -4,14 +4,11 @@ require "rest-client"
|
|
4
4
|
module Sqspoller
|
5
5
|
class WorkerTask
|
6
6
|
|
7
|
-
HEADERS = { 'Content-Type' => 'application/json',
|
8
|
-
'Accept' => 'application/json'
|
9
|
-
}
|
10
7
|
ALLOWED_METHODS = { 'post' => :post,
|
11
8
|
'get' => :get
|
12
9
|
}
|
13
10
|
|
14
|
-
def initialize
|
11
|
+
def initialize worker_configuration
|
15
12
|
@http_method = ALLOWED_METHODS[worker_configuration[:http_method].downcase]
|
16
13
|
@http_url = worker_configuration[:http_url]
|
17
14
|
@timeout = worker_configuration[:timeout] && worker_configuration[:timeout].to_i || 450
|
data/sqspoller.gemspec
CHANGED
@@ -5,7 +5,7 @@ require 'sqspoller/version'
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |spec|
|
7
7
|
spec.name = "fk-redis-sqspoller"
|
8
|
-
spec.version = '0.1.
|
8
|
+
spec.version = '0.1.2'
|
9
9
|
spec.authors = ["FourKites"]
|
10
10
|
spec.email = ["developer@fourkites.com "]
|
11
11
|
|
@@ -24,5 +24,7 @@ Gem::Specification.new do |spec|
|
|
24
24
|
spec.add_runtime_dependency "concurrent-ruby"
|
25
25
|
spec.add_runtime_dependency "rest-client"
|
26
26
|
spec.add_runtime_dependency 'aws-sdk', '~> 2'
|
27
|
+
spec.add_runtime_dependency 'redis'
|
28
|
+
|
27
29
|
|
28
30
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: fk-redis-sqspoller
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- FourKites
|
@@ -80,6 +80,20 @@ dependencies:
|
|
80
80
|
- - "~>"
|
81
81
|
- !ruby/object:Gem::Version
|
82
82
|
version: '2'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: redis
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - ">="
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '0'
|
90
|
+
type: :runtime
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - ">="
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '0'
|
83
97
|
description: ''
|
84
98
|
email:
|
85
99
|
- 'developer@fourkites.com '
|
@@ -99,6 +113,7 @@ files:
|
|
99
113
|
- lib/sqspoller.rb
|
100
114
|
- lib/sqspoller/message_delegator.rb
|
101
115
|
- lib/sqspoller/queue_controller.rb
|
116
|
+
- lib/sqspoller/sns_forwarder.rb
|
102
117
|
- lib/sqspoller/sqs_poller.rb
|
103
118
|
- lib/sqspoller/version.rb
|
104
119
|
- lib/sqspoller/worker_task.rb
|