fk-redis-sqspoller 0.1.1 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 6227bb64adfc2ba80e9cdec095a1dcda19b3d1a1
4
- data.tar.gz: 15f81fb35c96bbfcda9314e723ff964df6d6fd54
3
+ metadata.gz: 2e565f6d7c27ea7b1400460f7b0579a39c6debb1
4
+ data.tar.gz: bf80d0b650f6870ad0e86c5ba4886f14ba31ffae
5
5
  SHA512:
6
- metadata.gz: e7af5196432728282835c3c46bc5c40958c82561d1bea5e64a6cc1134222fe97ab8679758cad3da5c66d3388105b71d412c5a9ded652558b1eb90b07ff0e89eb
7
- data.tar.gz: 69d76ff9c2e808cf00b37e483e5a438d08dc0a66753f43d460adde2c20f6a421b6536b3d16f000f5a6b1e02bf4e46460c0105ae11bb8375118166e59167114e7
6
+ metadata.gz: cb01da2c8db24e135c9de4e72bc913ad8107e52fe57a8737687685aadc7227fce334c05975cadbda011e9df385ec0ff023019f25ca331f1c39dc2b85de003804
7
+ data.tar.gz: a7d39a8b52e6e7d193f15b1a025d34e375c652adc2d79e675fed905ab4740bc272568c20e71a4dce2eb7999796ae088f94af732490f95ab3ddeab845bf4df068
@@ -6,6 +6,6 @@ puts "SQSPoller arguments check"
6
6
  ARGV.each do |arg|
7
7
  puts "arg: #{arg}"
8
8
  end
9
- puts "Starting poller"
10
- Sqspoller::SqsPoller.start_poller ARGV[0], ARGV[1], ARGV[2], ARGV[3], ARGV[4], ARGV[5]
11
- puts "poller done"
9
+ puts "poller start"
10
+ Sqspoller::SqsPoller.start_poller ARGV[0], ARGV[1], ARGV[2], ARGV[3], ARGV[4], ARGV[5], ARGV[6]
11
+ puts "poller done"
@@ -28,23 +28,21 @@ module Sqspoller
28
28
  @logger.info "Exiting wait state, connection_pool size reached below worker_thread_pool_size, pending_schedule_tasks=#{@pending_schedule_tasks}"
29
29
  end
30
30
  end
31
- @logger.info "Scheduling worker task for message: #{message.message_id}"
32
31
 
33
32
  begin
34
33
  @connection_pool.post do
35
34
  begin
36
- @logger.info "Starting worker task for message: #{message.message_id}"
35
+ @logger.info " Starting worker task for message: #{message.message_id} on queue #{queue_name}"
37
36
  @worker_task.process(message.body, message.message_id)
38
- @logger.info "Finished worker task for message: #{message.message_id}"
39
37
  queue_controller.delete_message message.receipt_handle
40
38
  rescue Exception => e
41
- @logger.info "Caught error for message: #{message}, error: #{e.message}, #{e.backtrace.join("\n")}"
39
+ @logger.info " Caught error for message: #{message}, error: #{e.message}, #{e.backtrace.join("\n")}"
42
40
  end
43
41
  @pending_schedule_tasks -= 1
44
42
  end
45
43
  rescue Concurrent::RejectedExecutionError => e
46
44
  @pending_schedule_tasks -= 1
47
- @logger.info "Caught Concurrent::RejectedExecutionError for #{e.message}"
45
+ @logger.info " Caught Concurrent::RejectedExecutionError for #{e.message} on queue #{queue_name}"
48
46
  end
49
47
  end
50
48
  end
@@ -1,21 +1,35 @@
1
1
  require "logger"
2
2
  require "aws-sdk"
3
+ require 'base64'
4
+ require 'openssl'
3
5
 
4
6
  module Sqspoller
5
7
  class QueueController
6
8
 
7
- attr_accessor :threads
9
+ # expecting the queue name to be <environment>_outbound_messages_<company_id>
10
+ REGEXP = /(?<environment>\w+)-outbound-messages-(?<window_identifier>[\w-]+)/
8
11
 
9
- def initialize(queue_name, polling_threads_count, task_delegator, access_key_id, secret_access_key, region, logger_file)
10
- @logger = Logger.new(logger_file)
11
- @queue_name = queue_name
12
- @polling_threads_count = polling_threads_count
13
- @sqs = Aws::SQS::Client.new access_key_id: access_key_id,
14
- secret_access_key: secret_access_key,
15
- region: region
12
+ attr_accessor :threads,
13
+ :queue_name
14
+
15
+ def initialize args
16
+ @logger = args[:logger]
17
+ self.queue_name = args[:queue_name]
18
+ @polling_threads_count = args[:polling_threads_count]
19
+ @sqs = Aws::SQS::Client.new access_key_id: args[:access_key_id],
20
+ secret_access_key: args[:secret_access_key],
21
+ region: args[:region]
16
22
  @queue_details = @sqs.get_queue_url(queue_name: queue_name)
17
23
  self.threads = []
18
- @task_delegator = task_delegator
24
+ @task_delegator = args[:task_delegator]
25
+ match_data = REGEXP.match queue_name
26
+ @maintenance_window = if match_data
27
+ @cache_key = "#{match_data[:environment]}::OutboundMessages::MaintenanceWindowOpen::#{match_data[:window_identifier]}"
28
+ @window_identifier = match_data[:window_identifier]
29
+ true
30
+ else
31
+ false
32
+ end
19
33
  end
20
34
 
21
35
  def start
@@ -28,23 +42,19 @@ module Sqspoller
28
42
 
29
43
  def start_thread(queue_url)
30
44
  Thread.new do
31
- @logger.info "Poller thread started for queue: #{queue_url}"
32
45
  poller = Aws::SQS::QueuePoller.new(queue_url)
46
+ poller.before_request do |stats|
47
+ block_on_maintenance_window
48
+ end
33
49
 
34
50
  loop do
35
- @logger.info "Polling queue #{@queue_name} for messages"
36
- begin
37
- msgs = @sqs.receive_message :queue_url => queue_url
38
- rescue Exception => e
39
- @logger.info "Error receiving messages from queue #{@queue_name}: #{e.message}"
40
- next
41
- end
42
- msgs.messages.each do |received_message|
51
+ @logger.info " Polling queue #{queue_name} for messages"
52
+ poller.poll do |received_message|
43
53
  begin
44
- @logger.info "Received message #{@queue_name} : #{received_message.message_id}"
45
- @task_delegator.process self, received_message, @queue_name
54
+ @task_delegator.process self, received_message, queue_name
46
55
  rescue Exception => e
47
- @logger.info "Encountered error #{e.message} while submitting message from queue #{queue_url}"
56
+ @logger.info " Encountered error #{e.message} while submitting message to worker from queue #{queue_name}"
57
+ throw :skip_delete
48
58
  end
49
59
  end
50
60
  end
@@ -56,5 +66,18 @@ module Sqspoller
56
66
  receipt_handle: receipt_handle
57
67
  end
58
68
 
69
+ def block_on_maintenance_window
70
+ if @maintenance_window
71
+ loop do
72
+ window_open = REDIS.get @cache_key
73
+ if window_open
74
+ @logger.info " Maintenance Window is open for #{@window_identifier}, sleeping for 5 minutes"
75
+ sleep 300
76
+ else
77
+ break
78
+ end
79
+ end
80
+ end
81
+ end
59
82
  end
60
83
  end
@@ -0,0 +1,21 @@
1
+ require 'aws-sdk'
2
+
3
+ module Sqspoller
4
+ class SnsForwarder
5
+ def initialize args
6
+ @sns = Aws::SNS::Client.new access_key_id: args[:access_key_id],
7
+ secret_access_key: args[:secret_access_key],
8
+ region: args[:region]
9
+ @topic_arn = args[:sns_topic_arn]
10
+ @logger = args[:logger]
11
+ @logger.info " Inializing SnsForwarder with SNS topic arn: #{@topic_arn}"
12
+ end
13
+
14
+ def process message, message_id
15
+ @logger.info " Processing message"
16
+ @logger.info " Publishing to #{@topic_arn}"
17
+ response = @sns.publish topic_arn: @topic_arn, message: message
18
+ @logger.info " SNS response message id: #{response.message_id}"
19
+ end
20
+ end
21
+ end
@@ -3,11 +3,17 @@ require "concurrent"
3
3
  require "net/http"
4
4
  require "yaml"
5
5
  require "erb"
6
+ require 'redis'
6
7
  require "sqspoller/worker_task"
8
+ require "sqspoller/sns_forwarder"
7
9
  require "sqspoller/message_delegator"
8
10
  require "sqspoller/queue_controller"
9
11
 
10
12
  module Sqspoller
13
+ REDIS = Redis.new host: ENV['REDIS_HOST'], port: ENV['REDIS_PORT']
14
+ HEADERS = { 'Content-Type' => 'application/json',
15
+ 'Accept' => 'application/json'
16
+ }
11
17
 
12
18
  class SqsPoller
13
19
  class << self
@@ -42,70 +48,148 @@ module Sqspoller
42
48
  STDERR.reopen STDOUT
43
49
  end
44
50
 
45
- def start_poller_with_config(config, queue_config_name, access_key_id, secret_access_key, region, logger_file)
46
- puts "Started poller method"
47
- @logger = Logger.new(logger_file)
48
-
49
- qcs = []
50
- queues_config = config[queue_config_name] || config[queue_config_name.to_sym]
51
- total_poller_threads = queues_config.keys.reduce(0) do |sum, queue|
52
- sum += queues_config[queue][:polling_threads]
53
- end
54
- message_delegator = initialize_worker config[:worker_configuration], total_poller_threads, logger_file
55
- queues_config.keys.each do |queue|
56
- if queues_config[queue][:polling_threads] == 0
57
- @logger.info "Polling disabled for queue: #{queue}"
51
+ def start_poller content_name, queue_config_name, access_key_id, secret_access_key, region, log_filename=nil, redis_or_file=false
52
+ puts "StartPoller"
53
+
54
+ poller_args = {content_name: content_name,
55
+ queue_config_name: queue_config_name.to_sym,
56
+ access_key_id: access_key_id,
57
+ secret_access_key: secret_access_key,
58
+ region: region,
59
+ log_filename: log_filename,
60
+ redis_or_file: redis_or_file
61
+ }
62
+
63
+ if redis_or_file
64
+ if REDIS.get('random_key') == nil
65
+ puts "Able to connect to Redis"
58
66
  else
59
- @logger.info "Creating QueueController object for queue: #{queue}"
60
- qc = QueueController.new queue,
61
- queues_config[queue][:polling_threads],
62
- message_delegator,
63
- access_key_id,
64
- secret_access_key,
65
- region,
66
- logger_file
67
- qc.start
68
- qc.threads.each do |thread|
69
- thread.join
70
- end
67
+ puts "*** Unable to connect to Redis"
68
+ exit -1
71
69
  end
72
70
  end
71
+ if log_filename.nil? || log_filename.empty?
72
+ poller_args[:log_filename] = STDOUT
73
+ puts "Did not receive log file name"
74
+ fork do
75
+ Process.daemon nil, :noclose
76
+ start_queues_with_config poller_args
77
+ end
78
+ else
79
+ puts "Did receive log file name"
80
+ daemonize log_filename
81
+ start_queues_with_config poller_args
82
+ end
73
83
  end
74
84
 
75
- def start_poller(filename, queue_config_name, access_key_id, secret_access_key, region, log_filename=nil)
85
+ def load_config_from_file filename
76
86
  content = IO.read filename
77
- start_poller_from_content content, queue_config_name, access_key_id, secret_access_key, region, log_filename
87
+ symbolize_from_content content
88
+ end
89
+
90
+ def load_config_from_redis redis_key
91
+ content = REDIS.get redis_key
92
+ symbolize_from_content content
78
93
  end
79
94
 
80
- def start_poller_from_content(content, queue_config_name, access_key_id, secret_access_key, region, log_filename)
81
- puts "Starting poller"
95
+ def symbolize_from_content content
82
96
  erb = ERB.new content
83
- config = YAML.load erb.result
84
- config = symbolize config
97
+ yaml = YAML.load erb.result
98
+ symbolize yaml
99
+ end
85
100
 
86
- if log_filename.nil? || log_filename.empty?
87
- puts "Did not receive log file name"
88
- fork do
89
- Process.daemon
90
- start_poller_with_config config, queue_config_name, access_key_id, secret_access_key, region, STDOUT
101
+ def start_queues_with_config poller_args
102
+ @logger = Logger.new(poller_args[:log_filename])
103
+ poller_args[:logger] = @logger
104
+ @logger.info "Get config"
105
+ config = if poller_args[:redis_or_file]
106
+ load_config_from_redis poller_args[:content_name]
107
+ else
108
+ load_config_from_file poller_args[:content_name]
109
+ end
110
+ @logger.info "Config: #{config.inspect}"
111
+ queues_config = config[poller_args[:queue_config_name]]
112
+ if queues_config
113
+ @logger.info "QueuesConfig: #{queues_config.inspect}"
114
+ else
115
+ @logger.error "Unable to fetch Queue Config"
116
+ end
117
+
118
+ @logger.info "Started poller method"
119
+
120
+ message_delegator = worker_pool_init config[:worker_configuration].merge(poller_args), queues_config, poller_args[:log_filename]
121
+
122
+ if poller_args[:redis_or_file]
123
+ start_all_queues_with_refresh queues_config, message_delegator, poller_args, config[:worker_configuration][:refresh_interval_in_seconds] || 3600
124
+ else
125
+ start_all_queues queues_config, message_delegator, poller_args
126
+ end
127
+ end
128
+
129
+ def start_all_queues_with_refresh queues_config, message_delegator, poller_args, refresh_interval
130
+ @logger.info "Start all queues with refresh"
131
+ queues = {}
132
+ loop do
133
+ Thread.pass
134
+ queue_names = queues_config.keys
135
+ queues_config.keys.each do |queue|
136
+ @logger.info " Checking queue #{queue}"
137
+ if queues[queue]
138
+ @logger.info " Queue: #{queue} not created, already initialized and running"
139
+ else
140
+ queues[queue] = start_queue_controller queues_config, queue, message_delegator, poller_args
141
+ end
91
142
  end
143
+ @logger.info " Done creating queues, sleeping for #{refresh_interval} seconds"
144
+ sleep refresh_interval
145
+ @logger.info " Refreshing config"
146
+ config = load_config_from_redis poller_args[:content_name]
147
+ queues_config = config[poller_args[:queue_config_name]]
148
+ end
149
+ end
150
+
151
+ def start_all_queues queues_config, message_delegator, poller_args
152
+ @logger.info "Start all queues "
153
+ queues_config.keys.each do |queue|
154
+ start_queue_controller queues_config, queue, message_delegator, poller_args
155
+ end
156
+ end
157
+
158
+ def worker_pool_init worker_config, queues_config, logger_file
159
+ total_poller_threads = queues_config.keys.reduce(0) do |sum, queue|
160
+ sum += queues_config[queue][:polling_threads]
161
+ end
162
+ initialize_worker worker_config, total_poller_threads, logger_file
163
+ end
164
+
165
+ def start_queue_controller queues_config, queue, message_delegator, poller_args
166
+ if queues_config[queue][:polling_threads] == 0
167
+ @logger.info " Polling disabled for queue: #{queue}"
168
+ nil
92
169
  else
93
- puts "Did receive log file name"
94
- daemonize log_filename
95
- start_poller_with_config config, queue_config_name, access_key_id, secret_access_key, region, log_filename
170
+ @logger.info " Creating QueueController object for queue: #{queue}"
171
+ qc = QueueController.new queue_name: queue,
172
+ polling_threads_count: queues_config[queue][:polling_threads],
173
+ task_delegator: message_delegator,
174
+ access_key_id: poller_args[:access_key_id],
175
+ secret_access_key: poller_args[:secret_access_key],
176
+ region: poller_args[:region],
177
+ logger_file: poller_args[:log_filename],
178
+ logger: poller_args[:logger]
179
+ qc.start
180
+ qc
96
181
  end
97
182
  end
98
183
 
99
- def initialize_worker(worker_configuration, total_poller_threads, logger_file)
184
+ def initialize_worker worker_configuration, total_poller_threads, logger_file
100
185
  worker_thread_count = worker_configuration[:concurrency] || total_poller_threads
101
186
  waiting_tasks_ratio = worker_configuration[:waiting_tasks_ratio] || 1
102
187
 
103
- klass = worker_configuration[:worker_class].split('::').inject(Object) {|o,c| o.const_get c}
104
- worker_task = klass.new(worker_configuration, logger_file)
188
+ klass = worker_configuration[:worker_class].split('::').reduce(Object, :const_get)
189
+ worker_task = klass.new worker_configuration
105
190
 
106
191
  MessageDelegator.new worker_thread_count, waiting_tasks_ratio, worker_task, logger_file
107
192
  end
108
193
  end
109
194
  end
110
-
111
195
  end
@@ -4,14 +4,11 @@ require "rest-client"
4
4
  module Sqspoller
5
5
  class WorkerTask
6
6
 
7
- HEADERS = { 'Content-Type' => 'application/json',
8
- 'Accept' => 'application/json'
9
- }
10
7
  ALLOWED_METHODS = { 'post' => :post,
11
8
  'get' => :get
12
9
  }
13
10
 
14
- def initialize(worker_configuration, logger_file)
11
+ def initialize worker_configuration
15
12
  @http_method = ALLOWED_METHODS[worker_configuration[:http_method].downcase]
16
13
  @http_url = worker_configuration[:http_url]
17
14
  @timeout = worker_configuration[:timeout] && worker_configuration[:timeout].to_i || 450
data/sqspoller.gemspec CHANGED
@@ -5,7 +5,7 @@ require 'sqspoller/version'
5
5
 
6
6
  Gem::Specification.new do |spec|
7
7
  spec.name = "fk-redis-sqspoller"
8
- spec.version = '0.1.1'
8
+ spec.version = '0.1.2'
9
9
  spec.authors = ["FourKites"]
10
10
  spec.email = ["developer@fourkites.com "]
11
11
 
@@ -24,5 +24,7 @@ Gem::Specification.new do |spec|
24
24
  spec.add_runtime_dependency "concurrent-ruby"
25
25
  spec.add_runtime_dependency "rest-client"
26
26
  spec.add_runtime_dependency 'aws-sdk', '~> 2'
27
+ spec.add_runtime_dependency 'redis'
28
+
27
29
 
28
30
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fk-redis-sqspoller
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.1
4
+ version: 0.1.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - FourKites
@@ -80,6 +80,20 @@ dependencies:
80
80
  - - "~>"
81
81
  - !ruby/object:Gem::Version
82
82
  version: '2'
83
+ - !ruby/object:Gem::Dependency
84
+ name: redis
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - ">="
88
+ - !ruby/object:Gem::Version
89
+ version: '0'
90
+ type: :runtime
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - ">="
95
+ - !ruby/object:Gem::Version
96
+ version: '0'
83
97
  description: ''
84
98
  email:
85
99
  - 'developer@fourkites.com '
@@ -99,6 +113,7 @@ files:
99
113
  - lib/sqspoller.rb
100
114
  - lib/sqspoller/message_delegator.rb
101
115
  - lib/sqspoller/queue_controller.rb
116
+ - lib/sqspoller/sns_forwarder.rb
102
117
  - lib/sqspoller/sqs_poller.rb
103
118
  - lib/sqspoller/version.rb
104
119
  - lib/sqspoller/worker_task.rb