fourkites-sqspoller-v2 1.0.0

data/lib/sqspoller/process/message_handler.rb

@@ -0,0 +1,53 @@
+require "concurrent"
+require "net/http"
+require "rest-client"
+require "json"
+require "sqspoller/logger/logger"
+
+# This handles the SQS message and forward to local puma server and get the response.
+module SqsPoller
+  module Process
+    class MessageHandler
+
+      HEADERS = {
+        'Content-Type' => 'application/json',
+        'Accept' => 'application/json'
+      }
+
+      def initialize(worker_configuration)
+        @logger = SqsPoller::Logger.get_new_logger(self.class.name)
+        @http_method = worker_configuration[:http_method]
+        @http_url = worker_configuration[:http_url]
+        @timeout = worker_configuration[:timeout] ? worker_configuration[:timeout].to_i : 450
+        @uri = URI(@http_url)
+      end
+
+      def handle(message, message_id)
+        parsed_message = JSON.parse(message)
+        # @logger.info parsed_message
+        if @http_method.downcase == "post"
+          RestClient::Request.execute(:method => :post, :url => @http_url, :payload => parsed_message.to_json, :headers => HEADERS, :timeout => @timeout, :open_timeout => 5) do |response, request, result|
+            process_http_response response
+          end
+        elsif @http_method.downcase == "get"
+          RestClient::Request.execute(:method => :get, :url => @http_url, :payload => parsed_message.to_json, :headers => HEADERS, :timeout => @timeout, :open_timeout => 5) do |response, request, result|
+            process_http_response response
+          end
+        else
+          raise "Invalid http_method provided. #{http_method}"
+        end
+        parsed_message["MessageType"]
+      end
+
+      def process_http_response(response)
+        case response.code
+        when 200
+          return "OK"
+        else
+          raise "Service did not return 200 OK response. #{response.code}"
+        end
+      end
+
+    end
+  end
+end

data/lib/sqspoller/process/task_finalizer.rb

@@ -0,0 +1,88 @@
+require "logger"
+require "concurrent"
+require "net/http"
+require "rest-client"
+require "json"
+require "sqspoller/logger/logger"
+require "sqspoller/common/ring_buffer"
+require "sqspoller/poll/queue_controller"
+
+# TaskFinalizer will delete the batch of messages from SQS.
+# It used the RingBuffer as the cache (Not persisted. We will lose all the messages if process killed or stopped.) to hold all completed tasks.
+# When buffer filled the half the sizes it will try to delete the messages using sqs batch delete (max 10 per batch)  from SQS and with max of 5 retries
+module SqsPoller
+  module Process
+    class TaskFinalizer
+
+      DEFAULT_FINALIZE_TIMER_DELAY = 5.0
+      MAX_SQS_DELETE_BATCH_SIZE = 10
+      MAX_DELETE_RETRY = 5
+
+      def initialize(buffer_size)
+        @logger = SqsPoller::Logger.get_new_logger(self.class.name)
+        @buffer_size = buffer_size
+        @completed_tasks = SqsPoller::Common::RingBuffer.new(buffer_size * 2)
+        schedule_timer
+      end
+
+      def buffer_size
+        @buffer_size
+      end
+
+      def finalize(completed_task)
+        @completed_tasks.push completed_task
+        if @completed_tasks.count >= @buffer_size
+          @timer.reset
+          finalize_tasks
+        end
+      end
+
+      private
+
+      def schedule_timer
+        @timer = Concurrent::ScheduledTask.new(DEFAULT_FINALIZE_TIMER_DELAY) {
+          finalize_tasks true
+          schedule_timer
+        }
+        @timer.execute
+      end
+
+      def finalize_tasks(scheduled = false)
+        return if !scheduled && @completed_tasks.count < @buffer_size
+        completed_tasks = @completed_tasks.flush
+        begin
+          task_group = completed_tasks.group_by { |task|
+            task[:queue_name]
+          }
+          task_group.each { |queue, tasks|
+            messages = tasks.map { |task|
+              task[:message]
+            }
+            messages.each_slice(MAX_SQS_DELETE_BATCH_SIZE) do |msgs|
+              SqsPoller::Poller::QueueController.delete_messages(queue, msgs)
+            end
+            @logger.debug "Deleted #{messages.size} messages from #{queue}"
+          }
+        rescue Exception => e
+          @logger.error "TaskFinalizer Caught error: #{e.message}, #{e.backtrace.join("\n")}"
+          finalize_task_individually(completed_tasks)
+        end
+      end
+
+      def finalize_task_individually(completed_tasks)
+        completed_tasks.each { |task|
+          begin
+            SqsPoller::Poller::QueueController.delete_message(task[:queue_name], task[:message])
+          rescue Exception => e
+            retry_delete = task[:retry_delete].present? ? task[:retry_delete] : 0
+            if retry_delete < MAX_DELETE_RETRY
+              task[:delete_retry] = retry_delete + 1
+              @completed_tasks.push task
+            end
+          end
+        }
+      end
+
+    end
+  end
+end

data/lib/sqspoller/process/task_worker.rb

@@ -0,0 +1,50 @@
+require "logger"
+require "concurrent"
+require "net/http"
+require "sqspoller/worker_task"
+require "sqspoller/common/ring_buffer"
+require "sqspoller/process/task_finalizer"
+require "sqspoller/common/utils"
+require "sqspoller/metrics/sqs_poller_metrics"
+
+# This Worker keep polls the task queue (SizedQueue with blocking) and delegate the message the message handler.
+# Once message is process based on its status will send to task finalizer if the message is successfully handled.
+# else it will ignore the message. Based on message visibility timeout in SQS it will reappear and processed.
+module SqsPoller
+  module Process
+    class Worker
+
+      def initialize(worker_name, task_queue, task_finalizer, message_handler)
+        @worker_name = worker_name
+        @task_queue = task_queue
+        @message_handler = message_handler
+        @task_finalizer = task_finalizer
+        @logger = SqsPoller::Logger.get_new_logger("#{self.class.name}-#{@worker_name}")
+      end
+
+      def run
+        loop do
+          task = @task_queue.pop
+          message = task[:message]
+          success = false
+          timer = SqsPoller::Common::Utils.start_timer
+          message_type= "UNKNOWN"
+          begin
+            @logger.debug "Starting worker task for message: #{message.message_id}"
+            message_type = @message_handler.handle(message.body, message.message_id)
+            @logger.debug "Finished worker task for message: #{message.message_id}"
+            success = true
+          rescue Exception => e
+            @logger.error "Caught error: #{e.message}, #{e.backtrace.join("\n")} for message id: #{message.message_id}, body: #{message.body}"
+          end
+          elapsed_time = timer.stop
+          queue_wait_time = timer.start_time - task[:queue_time]
+          @logger.info "Task Completed queue_name: #{task[:queue_name]}, message_id: #{message.message_id}, message_type: #{message_type}, elapsed_time: #{elapsed_time}, queue_wait_time: #{queue_wait_time} message_count: #{task[:index]}"
+          SqsPoller::Metrics.record(task, success, timer, elapsed_time)
+          @task_finalizer.finalize(task) if success
+        end
+      end
+
+    end
+  end
+end

data/lib/sqspoller/process/worker_controller.rb

@@ -0,0 +1,81 @@
+require "logger"
+require "concurrent"
+require "net/http"
+require "sqspoller/worker_task"
+require "sqspoller/common/ring_buffer"
+require "sqspoller/process/task_finalizer"
+require "sqspoller/common/utils"
+require "sqspoller/metrics/sqs_poller_metrics"
+require "sqspoller/process/task_worker"
+
+# This manages all process task workers threads using threadpools.
+# The number of thread can be configured in yaml config using concurrency:
+# Refer test.yaml
+module SqsPoller
+  module Process
+    class WorkerController
+
+      private_class_method :new
+
+      def initialize(worker_count, task_queue, worker_task, auto_tuning = true)
+        @task_queue = task_queue
+        @message_handler = worker_task
+        @dynamic_scheduling = auto_tuning
+        @worker_count = worker_count
+        @task_workers = Concurrent::RubyThreadPoolExecutor.new(min_threads: @worker_count)
+        @task_finalizer = TaskFinalizer.new(task_queue.max)
+        @logger = SqsPoller::Logger.get_new_logger(self.class.name)
+        @started = false
+      end
+
+      def self.get
+        return @instance if @instance
+        raise "WorkerController not yet started"
+      end
+
+      def self.start (worker_count, task_queue, worker_task, auto_tuning = true)
+        return @instance if @instance
+        @instance = new(worker_count, task_queue, worker_task, auto_tuning)
+        @instance.start
+        @instance
+      end
+
+      def started?
+        @started
+      end
+
+      def start
+        @logger.info "Starting #{@worker_count} workers"
+        begin
+          start_workers
+          @started = true
+        rescue Exception => e
+          @logger.error "Task Worker killed for and restarted. Caught error: #{e.message}, #{e.backtrace.join("\n")}"
+        end
+      end
+
+      def shutdown
+        @logger.info "WorkerController Terminated"
+      end
+
+      private
+
+      def start_workers
+        @worker_count.times do |index|
+          @task_workers.post do
+            @logger.info "Starting worker #{index}"
+            worker = Worker.new("Worker-#{index}", @task_queue, @task_finalizer, @message_handler)
+            loop do
+              begin
+                worker.run
+              rescue Exception => e
+                @logger.error "Task Worker killed for and restarted. Caught error: #{e.message}, #{e.backtrace.join("\n")}"
+              end
+            end
+          end
+        end
+      end
+
+    end
+  end
+end

data/lib/sqspoller/sqs_poller.rb

@@ -0,0 +1,147 @@
+require "logger"
+require "concurrent"
+require "net/http"
+require "yaml"
+require "erb"
+require "sqspoller"
+require "sqspoller/worker_task"
+require "sqspoller/poll/queue_controller"
+require "sqspoller/process/worker_controller"
+require "sqspoller/process/task_finalizer"
+require "sqspoller/process/message_handler"
+require 'new_relic/agent'
+
+# This is the starting point of SQS polling. Load the Yaml config and starts QueueController and WorkerController.
+# Waits until process get kill or stop.
+module SqsPoller
+  class << self
+
+    def sym(map)
+      if map.class == Hash
+        map = map.inject({}) { |memo, (k, v)| memo[k.to_sym] = sym(v); memo }
+      end
+      map
+    end
+
+    def daemonize(filename)
+      raise 'Must run as root' if Process.euid != 0
+
+      raise 'First fork failed' if (pid = fork) == -1
+      exit unless pid.nil?
+
+      Process.setsid
+      raise 'Second fork failed' if (pid = fork) == -1
+      exit unless pid.nil?
+      puts "Daemon pid: #{Process.pid}" # Or save it somewhere, etc.
+
+      Dir.chdir '/'
+      File.umask 0000
+
+      STDIN.reopen filename
+      STDOUT.reopen '/dev/null', 'a'
+      STDERR.reopen STDOUT
+    end
+
+    def start_poller_with_config(config, queue_config_name, access_key_id, secret_access_key, region, logger_file, log_level = ::Logger::ERROR)
+      SqsPoller::Logger.set_log_level(log_level)
+      SqsPoller::Logger.set_logger_file(logger_file)
+
+      ::NewRelic::Agent.manual_start if ENV['START_NEW_RELIC_AGENT']
+      SqsPoller::Metrics.start_metrics_agent if ENV['START_LOCAL_METRICS_AGENT']
+      @logger = SqsPoller::Logger.get_new_logger("SqsPoller")
+      @logger.info "Started poller method"
+      queues_config = config[queue_config_name] || config[queue_config_name.to_sym]
+      total_poller_threads = get_total_poller_threads(queues_config)
+      worker_configuration = config[:worker_configuration]
+      total_worker_threads = get_total_worker_threads(worker_configuration, total_poller_threads)
+      waiting_tasks_ratio = get_waiting_tasks_ratio(worker_configuration)
+      aws_config = {
+        :access_key_id => access_key_id,
+        :secret_access_key => secret_access_key,
+        :region => region
+      }
+
+      task_queue = SizedQueue.new(total_worker_threads * waiting_tasks_ratio)
+
+      qc = SqsPoller::Poller::QueueController.start queues_config, task_queue, aws_config
+      unless qc.started?
+        @logger.error("Unable to start Queue Pollers.")
+        return
+      end
+
+      worker_task = worker_configuration[:worker_class].split('::').inject(Object) { |o, c| o.const_get c }.new(worker_configuration)
+
+      wc = SqsPoller::Process::WorkerController.start total_worker_threads, task_queue, worker_task
+
+      unless wc.started?
+        @logger.error("Unable to start Workers.")
+        return
+      end
+
+      wait
+
+    end
+
+    def start_poller(filename, queue_config_name, access_key_id, secret_access_key, region, log_filename = nil)
+      begin
+        puts "Starting poller"
+        config = YAML.load(ERB.new(IO.read(filename)).result)
+        config = sym(config)
+
+        if log_filename.nil? || log_filename.empty?
+          puts "Did not receive log file name"
+          fork do
+            Process.daemon
+            start_poller_with_config config, queue_config_name, access_key_id, secret_access_key, region, STDOUT, ::Logger::DEBUG
+          end
+        else
+          puts "Did receive log file name #{log_filename}"
+          #daemonize log_filename
+          puts "Daemonize log file name #{log_filename}"
+          start_poller_with_config config, queue_config_name, access_key_id, secret_access_key, region, STDOUT, ::Logger::INFO
+        end
+      rescue Exception => e
+        puts "#{e}"
+      end
+    end
+
+    def get_total_poller_threads(queues_config)
+      total_poller_threads = 0
+      queues_config.keys.each { |queue|
+        total_poller_threads += queues_config[queue][:polling_threads]
+      }
+      total_poller_threads
+    end
+
+    def get_waiting_tasks_ratio(worker_configuration)
+      waiting_tasks_ratio = worker_configuration[:waiting_tasks_ratio]
+      waiting_tasks_ratio = 1 if waiting_tasks_ratio.nil?
+      waiting_tasks_ratio
+    end
+
+    def get_total_worker_threads(worker_configuration, total_poller_threads)
+      worker_thread_count = worker_configuration[:concurrency]
+      worker_thread_count = total_poller_threads if worker_thread_count.nil?
+      worker_thread_count
+    end
+
+    def timeout
+      n_bytes = [42].pack('i').size
+      n_bits = n_bytes * 8
+      2 ** (n_bits - 2) - 1
+    end
+
+    private
+
+    def wait
+      t = Thread.new do
+        loop do
+          sleep(60 * 60)
+        end
+      end
+      t.join
+    end
+
+  end
+
+end

data/lib/sqspoller/version.rb

@@ -0,0 +1,3 @@
+module SqsPoller
+  VERSION = "2.0.0"
+end

data/lib/sqspoller/worker_task.rb

@@ -0,0 +1,17 @@
+require 'sqspoller/process/message_handler'
+# DEPRECATED: This class is to support backward compatibility. This class will be removed once we updated all the config files updated.
+module Sqspoller
+
+  class WorkerTask
+
+    def initialize(worker_configuration)
+      @message_handler = SqsPoller::Process::MessageHandler.new(worker_configuration)
+    end
+
+    def handle(message, message_id)
+      @message_handler.handle(message, message_id)
+    end
+
+  end
+
+end

data/lib/sqspoller.rb

@@ -0,0 +1,7 @@
+require "sqspoller/version"
+require "seahorse/stringio"
+require "sqspoller/sqs_poller"
+
+module SqsPoller
+
+end

data/lib/test.yaml

@@ -0,0 +1,15 @@
+worker_configuration:
+  worker_class: "Sqspoller::WorkerTask"
+  http_method: "post"
+  http_url: "http://localhost:5001/process"
+  concurrency: 50 # => Total Number of service threads - threads occupied by cron jobs - thread for heartbeat => 2*32 - 8 - 1 = 55
+  waiting_tasks_ratio: 2
+production:
+  my-queue:
+    polling_threads: 4
+    max_number_of_messages: 10
+    message_types: [ ]
+  my-queue1:
+    polling_threads: 2
+    max_number_of_messages: 10
+    message_types: [ ]

data/sqspoller.gemspec

@@ -0,0 +1,30 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'sqspoller/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "fourkites-sqspoller-v2"
+  spec.version       = '1.0.0'
+  spec.authors       = %w[anshul-fk nagarajan]
+  spec.email         = %w[anshul@fourkites.com nagarajan.shanmugam@fourkites.com]
+
+  spec.summary       = 'Gem to poll messages from SQS queue and process them'
+  spec.description   = ''
+  spec.homepage      = ''
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "bin"
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.10"
+  spec.add_development_dependency "rake", "~> 10.0"
+
+  spec.add_runtime_dependency "concurrent-ruby"
+  spec.add_runtime_dependency "rest-client"  
+  spec.add_runtime_dependency 'aws-sdk', '~> 3'
+  spec.add_runtime_dependency 'ruby-metrics'
+  spec.add_runtime_dependency 'newrelic_rpm'
+
+end

metadata

@@ -0,0 +1,180 @@
+--- !ruby/object:Gem::Specification
+name: fourkites-sqspoller-v2
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- anshul-fk
+- nagarajan
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2022-11-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: concurrent-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rest-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3'
+- !ruby/object:Gem::Dependency
+  name: ruby-metrics
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: newrelic_rpm
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: ''
+email:
+- anshul@fourkites.com
+- nagarajan.shanmugam@fourkites.com
+executables:
+- run_sqs_poller
+- sqs_poller.service
+- sqs_poller_v2
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".ruby-gemset"
+- ".ruby-version"
+- CODE_OF_CONDUCT.md
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- bin/run_sqs_poller
+- bin/sqs_poller.service
+- bin/sqs_poller_v2
+- images/SQS_Poller_V2.png
+- install.sh
+- lib/config/newrelic.yml
+- lib/seahorse/stringio.rb
+- lib/sqs-ruby-example-consume-queue.rb
+- lib/sqs-ruby-example-create-queue.rb
+- lib/sqspoller.rb
+- lib/sqspoller/common/ring_buffer.rb
+- lib/sqspoller/common/utils.rb
+- lib/sqspoller/logger/logger.rb
+- lib/sqspoller/metrics/log_reporter.rb
+- lib/sqspoller/metrics/queue_stats_reporter.rb
+- lib/sqspoller/metrics/sqs_poller_metrics.rb
+- lib/sqspoller/poll/queue_controller.rb
+- lib/sqspoller/poll/queue_poller.rb
+- lib/sqspoller/process/message_handler.rb
+- lib/sqspoller/process/task_finalizer.rb
+- lib/sqspoller/process/task_worker.rb
+- lib/sqspoller/process/worker_controller.rb
+- lib/sqspoller/sqs_poller.rb
+- lib/sqspoller/version.rb
+- lib/sqspoller/worker_task.rb
+- lib/test.yaml
+- sqspoller.gemspec
+homepage: ''
+licenses: []
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.3.14
+signing_key:
+specification_version: 4
+summary: Gem to poll messages from SQS queue and process them
+test_files: []