fk-redis-sqspoller 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6227bb64adfc2ba80e9cdec095a1dcda19b3d1a1
-  data.tar.gz: 15f81fb35c96bbfcda9314e723ff964df6d6fd54
+  metadata.gz: 2e565f6d7c27ea7b1400460f7b0579a39c6debb1
+  data.tar.gz: bf80d0b650f6870ad0e86c5ba4886f14ba31ffae
 SHA512:
-  metadata.gz: e7af5196432728282835c3c46bc5c40958c82561d1bea5e64a6cc1134222fe97ab8679758cad3da5c66d3388105b71d412c5a9ded652558b1eb90b07ff0e89eb
-  data.tar.gz: 69d76ff9c2e808cf00b37e483e5a438d08dc0a66753f43d460adde2c20f6a421b6536b3d16f000f5a6b1e02bf4e46460c0105ae11bb8375118166e59167114e7
+  metadata.gz: cb01da2c8db24e135c9de4e72bc913ad8107e52fe57a8737687685aadc7227fce334c05975cadbda011e9df385ec0ff023019f25ca331f1c39dc2b85de003804
+  data.tar.gz: a7d39a8b52e6e7d193f15b1a025d34e375c652adc2d79e675fed905ab4740bc272568c20e71a4dce2eb7999796ae088f94af732490f95ab3ddeab845bf4df068

data/bin/fk-redis-sqspoller

@@ -6,6 +6,6 @@ puts "SQSPoller arguments check"
 ARGV.each do |arg|
   puts "arg: #{arg}"
 end
-puts "Starting poller"
-Sqspoller::SqsPoller.start_poller ARGV[0], ARGV[1], ARGV[2], ARGV[3], ARGV[4], ARGV[5]
-puts "poller done"
+puts "poller start"
+Sqspoller::SqsPoller.start_poller ARGV[0], ARGV[1], ARGV[2], ARGV[3], ARGV[4], ARGV[5], ARGV[6]
+puts "poller done"

data/lib/sqspoller/message_delegator.rb

@@ -28,23 +28,21 @@ module Sqspoller
           @logger.info "Exiting wait state, connection_pool size reached below worker_thread_pool_size, pending_schedule_tasks=#{@pending_schedule_tasks}"
         end
       end
-      @logger.info "Scheduling worker task for message: #{message.message_id}"
 
       begin
         @connection_pool.post do
           begin
-            @logger.info "Starting worker task for message: #{message.message_id}"
+            @logger.info "  Starting worker task for message: #{message.message_id} on queue #{queue_name}"
             @worker_task.process(message.body, message.message_id)
-            @logger.info "Finished worker task for message: #{message.message_id}"
             queue_controller.delete_message message.receipt_handle
           rescue Exception => e
-            @logger.info "Caught error for message: #{message}, error: #{e.message}, #{e.backtrace.join("\n")}"
+            @logger.info "    Caught error for message: #{message}, error: #{e.message}, #{e.backtrace.join("\n")}"
           end
           @pending_schedule_tasks -= 1
         end
       rescue Concurrent::RejectedExecutionError => e
         @pending_schedule_tasks -= 1
-        @logger.info  "Caught Concurrent::RejectedExecutionError for #{e.message}"
+        @logger.info  "  Caught Concurrent::RejectedExecutionError for #{e.message} on queue #{queue_name}"
       end
     end
   end

data/lib/sqspoller/queue_controller.rb

@@ -1,21 +1,35 @@
 require "logger"
 require "aws-sdk"
+require 'base64'
+require 'openssl'
 
 module Sqspoller
   class QueueController
 
-    attr_accessor :threads
+    # expecting the queue name to be <environment>_outbound_messages_<company_id>
+    REGEXP = /(?<environment>\w+)-outbound-messages-(?<window_identifier>[\w-]+)/
 
-    def initialize(queue_name, polling_threads_count, task_delegator, access_key_id, secret_access_key, region, logger_file)
-      @logger = Logger.new(logger_file)
-      @queue_name = queue_name
-      @polling_threads_count = polling_threads_count
-      @sqs = Aws::SQS::Client.new access_key_id: access_key_id,
-                                  secret_access_key: secret_access_key,
-                                  region: region
+    attr_accessor :threads,
+                  :queue_name
+
+    def initialize args
+      @logger = args[:logger]
+      self.queue_name = args[:queue_name]
+      @polling_threads_count = args[:polling_threads_count]
+      @sqs = Aws::SQS::Client.new access_key_id: args[:access_key_id],
+                                  secret_access_key: args[:secret_access_key],
+                                  region: args[:region]
       @queue_details = @sqs.get_queue_url(queue_name: queue_name)
       self.threads = []
-      @task_delegator = task_delegator
+      @task_delegator = args[:task_delegator]
+      match_data = REGEXP.match queue_name
+      @maintenance_window = if match_data
+                              @cache_key = "#{match_data[:environment]}::OutboundMessages::MaintenanceWindowOpen::#{match_data[:window_identifier]}"
+                              @window_identifier = match_data[:window_identifier]
+                              true
+                            else
+                              false
+                            end
     end
 
     def start
@@ -28,23 +42,19 @@ module Sqspoller
 
     def start_thread(queue_url)
       Thread.new do
-        @logger.info "Poller thread started for queue: #{queue_url}"
         poller = Aws::SQS::QueuePoller.new(queue_url)
+        poller.before_request do |stats|
+          block_on_maintenance_window
+        end
 
         loop do
-          @logger.info "Polling queue #{@queue_name} for messages"
-          begin
-            msgs = @sqs.receive_message :queue_url => queue_url
-          rescue Exception => e
-            @logger.info "Error receiving messages from queue #{@queue_name}: #{e.message}"
-            next
-          end
-          msgs.messages.each do |received_message|
+          @logger.info "  Polling queue #{queue_name} for messages"
+          poller.poll do |received_message|
             begin
-              @logger.info "Received message #{@queue_name} : #{received_message.message_id}"
-              @task_delegator.process self, received_message, @queue_name
+              @task_delegator.process self, received_message, queue_name
             rescue Exception => e
-              @logger.info "Encountered error #{e.message} while submitting message from queue #{queue_url}"
+              @logger.info "  Encountered error #{e.message} while submitting message to worker from queue #{queue_name}"
+              throw :skip_delete
             end
           end
         end
@@ -56,5 +66,18 @@ module Sqspoller
                           receipt_handle: receipt_handle
     end
 
+    def block_on_maintenance_window
+      if @maintenance_window
+        loop do
+          window_open = REDIS.get @cache_key
+          if window_open
+            @logger.info "    Maintenance Window is open for #{@window_identifier}, sleeping for 5 minutes"
+            sleep 300
+          else
+            break
+          end
+        end
+      end
+    end
   end
 end

data/lib/sqspoller/sns_forwarder.rb

@@ -0,0 +1,21 @@
+require 'aws-sdk'
+
+module Sqspoller
+  class SnsForwarder
+    def initialize args
+      @sns = Aws::SNS::Client.new access_key_id: args[:access_key_id],
+                                  secret_access_key: args[:secret_access_key],
+                                  region: args[:region]
+      @topic_arn = args[:sns_topic_arn]
+      @logger = args[:logger]
+      @logger.info "     Inializing SnsForwarder with SNS topic arn: #{@topic_arn}"
+    end
+
+    def process message, message_id
+      @logger.info "      Processing message"
+      @logger.info "        Publishing to #{@topic_arn}"
+      response = @sns.publish topic_arn: @topic_arn, message: message
+      @logger.info "        SNS response message id: #{response.message_id}"
+    end
+  end
+end

data/lib/sqspoller/sqs_poller.rb

@@ -3,11 +3,17 @@ require "concurrent"
 require "net/http"
 require "yaml"
 require "erb"
+require 'redis'
 require "sqspoller/worker_task"
+require "sqspoller/sns_forwarder"
 require "sqspoller/message_delegator"
 require "sqspoller/queue_controller"
 
 module Sqspoller
+  REDIS = Redis.new host: ENV['REDIS_HOST'], port: ENV['REDIS_PORT']
+  HEADERS = { 'Content-Type' => 'application/json',
+              'Accept' => 'application/json'
+            }
 
   class SqsPoller
     class << self
@@ -42,70 +48,148 @@ module Sqspoller
         STDERR.reopen STDOUT
       end
 
-      def start_poller_with_config(config, queue_config_name, access_key_id, secret_access_key, region, logger_file)
-        puts "Started poller method"
-        @logger = Logger.new(logger_file)
-
-        qcs = []
-        queues_config = config[queue_config_name] || config[queue_config_name.to_sym]
-        total_poller_threads = queues_config.keys.reduce(0) do |sum, queue|
-                                 sum += queues_config[queue][:polling_threads]
-                               end
-        message_delegator = initialize_worker config[:worker_configuration], total_poller_threads, logger_file
-        queues_config.keys.each do |queue|
-          if queues_config[queue][:polling_threads] == 0
-            @logger.info "Polling disabled for queue: #{queue}"
+      def start_poller content_name, queue_config_name, access_key_id, secret_access_key, region, log_filename=nil, redis_or_file=false
+        puts "StartPoller"
+
+        poller_args = {content_name:      content_name,
+                       queue_config_name: queue_config_name.to_sym,
+                       access_key_id:     access_key_id,
+                       secret_access_key: secret_access_key,
+                       region:            region,
+                       log_filename:      log_filename,
+                       redis_or_file:     redis_or_file
+                      }
+
+        if redis_or_file
+          if REDIS.get('random_key') == nil
+            puts "Able to connect to Redis"
           else
-            @logger.info "Creating QueueController object for queue: #{queue}"
-            qc = QueueController.new queue,
-                                     queues_config[queue][:polling_threads],
-                                     message_delegator,
-                                     access_key_id,
-                                     secret_access_key,
-                                     region,
-                                     logger_file
-            qc.start
-            qc.threads.each do |thread|
-              thread.join
-            end
+            puts "*** Unable to connect to Redis"
+            exit -1
           end
         end
+        if log_filename.nil? || log_filename.empty?
+          poller_args[:log_filename] = STDOUT
+          puts "Did not receive log file name"
+          fork do
+            Process.daemon nil, :noclose
+            start_queues_with_config poller_args
+          end
+        else
+          puts "Did receive log file name"
+          daemonize log_filename
+          start_queues_with_config poller_args
+        end
       end
 
-      def start_poller(filename, queue_config_name, access_key_id, secret_access_key, region, log_filename=nil)
+      def load_config_from_file filename
         content = IO.read filename
-        start_poller_from_content content, queue_config_name, access_key_id, secret_access_key, region, log_filename
+        symbolize_from_content content
+      end
+
+      def load_config_from_redis redis_key
+        content = REDIS.get redis_key
+        symbolize_from_content content
       end
 
-      def start_poller_from_content(content, queue_config_name, access_key_id, secret_access_key, region, log_filename)
-        puts "Starting poller"
+      def symbolize_from_content content
         erb = ERB.new content
-        config = YAML.load erb.result
-        config = symbolize config
+        yaml = YAML.load erb.result
+        symbolize yaml
+      end
 
-        if log_filename.nil? || log_filename.empty?
-          puts "Did not receive log file name"
-          fork do
-            Process.daemon
-            start_poller_with_config config, queue_config_name, access_key_id, secret_access_key, region, STDOUT
+      def start_queues_with_config poller_args
+        @logger = Logger.new(poller_args[:log_filename])
+        poller_args[:logger] = @logger
+        @logger.info "Get config"
+        config = if poller_args[:redis_or_file]
+                   load_config_from_redis poller_args[:content_name]
+                 else
+                   load_config_from_file poller_args[:content_name]
+                 end
+        @logger.info "Config: #{config.inspect}"
+        queues_config = config[poller_args[:queue_config_name]]
+        if queues_config
+          @logger.info "QueuesConfig: #{queues_config.inspect}"
+        else
+          @logger.error "Unable to fetch Queue Config"
+        end
+
+        @logger.info "Started poller method"
+
+        message_delegator = worker_pool_init config[:worker_configuration].merge(poller_args), queues_config, poller_args[:log_filename]
+
+        if poller_args[:redis_or_file]
+          start_all_queues_with_refresh queues_config, message_delegator, poller_args, config[:worker_configuration][:refresh_interval_in_seconds] || 3600
+        else
+          start_all_queues  queues_config, message_delegator, poller_args
+        end
+      end
+
+      def start_all_queues_with_refresh queues_config, message_delegator, poller_args, refresh_interval
+        @logger.info "Start all queues with refresh"
+        queues = {}
+        loop do
+          Thread.pass
+          queue_names = queues_config.keys
+          queues_config.keys.each do |queue|
+            @logger.info "    Checking queue #{queue}"
+            if queues[queue]
+              @logger.info "      Queue: #{queue} not created, already initialized and running"
+            else
+              queues[queue] = start_queue_controller queues_config, queue, message_delegator, poller_args
+            end
           end
+          @logger.info "  Done creating queues, sleeping for #{refresh_interval} seconds"
+          sleep refresh_interval
+          @logger.info "  Refreshing config"
+          config = load_config_from_redis poller_args[:content_name]
+          queues_config = config[poller_args[:queue_config_name]]
+        end
+      end
+
+      def start_all_queues queues_config, message_delegator, poller_args
+        @logger.info "Start all queues " 
+        queues_config.keys.each do |queue|
+          start_queue_controller queues_config, queue, message_delegator, poller_args
+        end
+      end
+
+      def worker_pool_init worker_config, queues_config, logger_file
+        total_poller_threads = queues_config.keys.reduce(0) do |sum, queue|
+                                 sum += queues_config[queue][:polling_threads]
+                               end
+        initialize_worker worker_config, total_poller_threads, logger_file
+      end
+
+      def start_queue_controller queues_config, queue, message_delegator, poller_args
+        if queues_config[queue][:polling_threads] == 0
+          @logger.info "  Polling disabled for queue: #{queue}"
+          nil
         else
-          puts "Did receive log file name"
-          daemonize log_filename
-          start_poller_with_config config, queue_config_name, access_key_id, secret_access_key, region, log_filename
+          @logger.info "  Creating QueueController object for queue: #{queue}"
+          qc = QueueController.new queue_name: queue,
+                                   polling_threads_count: queues_config[queue][:polling_threads],
+                                   task_delegator: message_delegator,
+                                   access_key_id: poller_args[:access_key_id],
+                                   secret_access_key: poller_args[:secret_access_key],
+                                   region: poller_args[:region],
+                                   logger_file: poller_args[:log_filename],
+                                   logger: poller_args[:logger]
+          qc.start
+          qc
         end
       end
 
-      def initialize_worker(worker_configuration, total_poller_threads, logger_file)
+      def initialize_worker worker_configuration, total_poller_threads, logger_file
         worker_thread_count = worker_configuration[:concurrency] || total_poller_threads
         waiting_tasks_ratio = worker_configuration[:waiting_tasks_ratio] || 1
 
-        klass = worker_configuration[:worker_class].split('::').inject(Object) {|o,c| o.const_get c}
-        worker_task = klass.new(worker_configuration, logger_file)
+        klass = worker_configuration[:worker_class].split('::').reduce(Object, :const_get)
+        worker_task = klass.new worker_configuration
 
         MessageDelegator.new worker_thread_count, waiting_tasks_ratio, worker_task, logger_file
       end
     end
   end
-
 end

data/lib/sqspoller/worker_task.rb

@@ -4,14 +4,11 @@ require "rest-client"
 module Sqspoller
   class WorkerTask
 
-    HEADERS = { 'Content-Type' => 'application/json',
-                'Accept' => 'application/json'
-              }
     ALLOWED_METHODS = { 'post' => :post,
                         'get'  => :get
                       }
 
-    def initialize(worker_configuration, logger_file)
+    def initialize worker_configuration
       @http_method = ALLOWED_METHODS[worker_configuration[:http_method].downcase]
       @http_url = worker_configuration[:http_url]
       @timeout = worker_configuration[:timeout] && worker_configuration[:timeout].to_i || 450

data/sqspoller.gemspec

@@ -5,7 +5,7 @@ require 'sqspoller/version'
 
 Gem::Specification.new do |spec|
   spec.name          = "fk-redis-sqspoller"
-  spec.version       = '0.1.1'
+  spec.version       = '0.1.2'
   spec.authors       = ["FourKites"]
   spec.email         = ["developer@fourkites.com "]
 
@@ -24,5 +24,7 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "concurrent-ruby"
   spec.add_runtime_dependency "rest-client"  
   spec.add_runtime_dependency 'aws-sdk', '~> 2'
+  spec.add_runtime_dependency 'redis'
+
 
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fk-redis-sqspoller
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - FourKites
@@ -80,6 +80,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2'
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: ''
 email:
 - 'developer@fourkites.com '
@@ -99,6 +113,7 @@ files:
 - lib/sqspoller.rb
 - lib/sqspoller/message_delegator.rb
 - lib/sqspoller/queue_controller.rb
+- lib/sqspoller/sns_forwarder.rb
 - lib/sqspoller/sqs_poller.rb
 - lib/sqspoller/version.rb
 - lib/sqspoller/worker_task.rb