txgh-queue 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ff7b50a82408cdac666a7921a2f9f011bc8d5226
+  data.tar.gz: dbe3735109b47773c7bb8175544273f2edffd86c
+SHA512:
+  metadata.gz: 8c0b6b954cf71c340094acf5de1f34d33796b3eb92bcd6ac4b13ec5758f3c0f38b97f2a35617e1d224e2fd93881cd554d5cb3ce8bb26e3326b22f47480bd5af8
+  data.tar.gz: d3113158ce321050b04ebe1c6abdb8260039a8cfe00ac6764d45f46d7d4cd2c1da7be04b101674aa9bc4bd234b96465f5e59c5a2c78e096b661784d4fadaed6d

data/lib/txgh-queue/application.rb

@@ -0,0 +1,35 @@
+require 'sinatra'
+require 'sinatra/json'
+
+module TxghQueue
+  module RespondWith
+    def respond_with(resp)
+      env['txgh.response'] = resp
+      status resp.status
+      json resp.body
+    end
+  end
+
+  class WebhookEndpoints < Sinatra::Base
+    include TxghQueue::Webhooks
+    helpers RespondWith
+
+    configure do
+      set :logging, nil
+      logger = Txgh::TxLogger.logger
+      set :logger, logger
+    end
+
+    post '/transifex/enqueue' do
+      respond_with(
+        Transifex::RequestHandler.handle_request(request, settings.logger)
+      )
+    end
+
+    post '/github/enqueue' do
+      respond_with(
+        Github::RequestHandler.handle_request(request, settings.logger)
+      )
+    end
+  end
+end

data/lib/txgh-queue/backends/null.rb

@@ -0,0 +1,19 @@
+require 'txgh'
+
+module TxghQueue
+  module Backends
+    module Null
+
+      class << self
+        def producer_for(*args)
+          raise BackendNotConfiguredError, 'No queue backend has been configured'
+        end
+
+        def consumer_for(*args)
+          raise BackendNotConfiguredError, 'No queue backend has been configured'
+        end
+      end
+
+    end
+  end
+end

data/lib/txgh-queue/backends/sqs/config.rb

@@ -0,0 +1,29 @@
+module TxghQueue
+  module Backends
+    module Sqs
+      class Config
+        class << self
+          def queues
+            @queues ||= TxghQueue::Config.options[:queues].map do |queue_options|
+              Queue.new(queue_options)
+            end
+          end
+
+          def failure_queue
+            @failure_queue ||= Queue.new(
+              TxghQueue::Config.options[:failure_queue]
+            )
+          end
+
+          def get_queue(queue_name)
+            queues.find { |q| q.name == queue_name }
+          end
+
+          def reset!
+            @queues = nil
+          end
+        end
+      end
+    end
+  end
+end

data/lib/txgh-queue/backends/sqs/consumer.rb

@@ -0,0 +1,30 @@
+require 'json'
+require 'txgh'
+
+module TxghQueue
+  module Backends
+    module Sqs
+      class Consumer
+        RECEIVE_PARAMS = { message_attribute_names: %w(history_sequence) }
+
+        attr_reader :queues, :logger
+
+        def initialize(queues, logger)
+          @queues = queues
+          @logger = logger
+        end
+
+        def work
+          queues.each do |queue|
+            queue.receive_message(RECEIVE_PARAMS).messages.each do |message|
+              logger.info("Received message from #{queue.name}, id: #{message.message_id}")
+              Job.new(message, queue, logger).complete
+            end
+          end
+        rescue => e
+          Txgh.events.publish_error!(e)
+        end
+      end
+    end
+  end
+end

data/lib/txgh-queue/backends/sqs/history_sequence.rb

@@ -0,0 +1,62 @@
+require 'json'
+require 'txgh'
+
+module TxghQueue
+  module Backends
+    module Sqs
+      class HistorySequence
+        extend Forwardable
+        include Enumerable
+
+        class << self
+          def from_message(message)
+            if attribute = message.message_attributes['history_sequence']
+              new(JSON.parse(attribute.string_value))
+            else
+              new([])
+            end
+          end
+
+          def from_h(hash)
+            new(
+              JSON.parse(
+                Txgh::Utils.deep_symbolize_keys(hash)
+                  .fetch(:history_sequence, {})
+                  .fetch(:string_value, nil)
+              )
+            )
+          end
+        end
+
+        attr_reader :sequence
+
+        def_delegators :sequence, :[], :<<, :first, :last, :size, :length
+
+        def initialize(sequence)
+          @sequence = Txgh::Utils.deep_symbolize_keys(sequence)
+        end
+
+        def add(obj)
+          sequence << obj
+        end
+
+        def to_h
+          { string_value: sequence.to_json, data_type: 'String' }
+        end
+
+        def dup
+          # use marshal serialization to deep copy the sequence
+          self.class.new(Marshal.load(Marshal.dump(sequence)))
+        end
+
+        def current
+          sequence.last
+        end
+
+        def each(&block)
+          sequence.each(&block)
+        end
+      end
+    end
+  end
+end

data/lib/txgh-queue/backends/sqs/job.rb

@@ -0,0 +1,115 @@
+require 'json'
+
+module TxghQueue
+  module Backends
+    module Sqs
+      class Job < TxghQueue::Job
+        attr_reader :message, :message_attributes, :queue, :logger
+
+        def initialize(message, queue, logger)
+          @message = message
+          @queue = queue
+          @message_attributes = MessageAttributes.from_message(message)
+
+          # add empty retry attributes hash to sequence - will be populated when
+          # the complete method is called
+          message_attributes.history_sequence.add({})
+
+          super(logger)
+        end
+
+        def complete
+          result = process(payload)
+          logger.info("Finished processing #{message.message_id}, result: #{result.status}")
+
+          message_attributes.history_sequence.current.merge!(attributes_for(result))
+
+          return do_ok(result) if result.status.ok?
+          return do_retry(result) if result.status.retry?
+          return do_fail(result) if result.status.fail?
+        end
+
+        private
+
+        def attributes_for(result)
+          { status: result.status.to_s }.merge(
+            if result.has_error?
+              error_attributes_for(result)
+            else
+              response_attributes_for(result)
+            end
+          )
+        end
+
+        def error_attributes_for(result)
+          {
+            response_type: 'error',
+            class: result.error.class,
+            message: result.error.message,
+            backtrace: (result.error.backtrace || []).first,
+            error_tracking: publish_error_for(result)
+          }
+        end
+
+        def response_attributes_for(result)
+          {
+            response_type: 'response',
+            code: result.response.status,
+            body: result.response.body
+          }
+        end
+
+        def payload
+          @payload ||= begin
+            logger.info("Processing #{message.message_id}")
+
+            JSON.parse(message.body).tap do |payload|
+              logger.info("Payload: #{payload.inspect}")
+            end
+          end
+        end
+
+        def do_ok(result)
+          delete(message)
+          logger.info("Deleted #{message.message_id}")
+        end
+
+        def do_retry(result)
+          retry_logic = RetryLogic.new(message_attributes, result.status)
+
+          if retry_logic.retry?
+            logger.info("Retrying #{message.message_id} with #{retry_logic.next_delay_seconds} second delay")
+            new_message = queue.send_message(message.body, retry_logic.sqs_retry_params)
+            logger.info("Re-enqueued as #{new_message.message_id}")
+            delete(message)
+            logger.info("Deleted original #{message.message_id}")
+          elsif retry_logic.retries_exceeded?
+            logger.info("Message #{message.message_id} has exceeded allowed retries.")
+            do_fail(result)
+          end
+        end
+
+        def do_fail(result)
+          # send to failure queue
+          new_message = Config.failure_queue.send_message(
+            message.body, message_attributes: message_attributes.to_h
+          )
+
+          delete(message)
+        end
+
+        def publish_error_for(result)
+          Txgh.events.publish_error(result.error, {
+            payload: payload,
+            message_id: message.message_id,
+            queue: queue.name
+          })
+        end
+
+        def delete(message)
+          queue.delete_message(message.receipt_handle)
+        end
+      end
+    end
+  end
+end

data/lib/txgh-queue/backends/sqs/message_attributes.rb

@@ -0,0 +1,33 @@
+module TxghQueue
+  module Backends
+    module Sqs
+      class MessageAttributes
+        class << self
+          def from_message(message)
+            history_sequence = HistorySequence.from_message(message)
+            new(history_sequence)
+          end
+
+          def from_h(hash)
+            history_sequence = HistorySequence.from_h(hash)
+            new(history_sequence)
+          end
+        end
+
+        attr_reader :history_sequence
+
+        def initialize(history_sequence)
+          @history_sequence = history_sequence
+        end
+
+        def to_h
+          { history_sequence: history_sequence.to_h }
+        end
+
+        def dup
+          self.class.new(history_sequence.dup)
+        end
+      end
+    end
+  end
+end

data/lib/txgh-queue/backends/sqs/producer.rb

@@ -0,0 +1,33 @@
+require 'json'
+
+module TxghQueue
+  module Backends
+    module Sqs
+      class Producer
+        attr_reader :queues, :logger
+
+        def initialize(queues, logger)
+          @queues = queues
+          @logger = logger
+        end
+
+        def enqueue(payload, options = {})
+          payload_json = payload.to_json
+
+          message_ids = queues.map do |queue|
+            new_message = queue.send_message(payload_json, options)
+
+            logger.info(
+              "Enqueued new message with id #{new_message.message_id} and params "\
+                "#{payload_json}"
+            )
+
+            new_message.message_id
+          end
+
+          { message_ids: message_ids }
+        end
+      end
+    end
+  end
+end

data/lib/txgh-queue/backends/sqs/queue.rb

@@ -0,0 +1,45 @@
+require 'aws-sdk'
+
+module TxghQueue
+  module Backends
+    module Sqs
+      class Queue
+        attr_reader :name, :region, :events
+
+        def initialize(options = {})
+          @name = options.fetch(:name)
+          @region = options.fetch(:region)
+          @events = options.fetch(:events, [])
+        end
+
+        def client
+          @client ||= Aws::SQS::Client.new(region: region)
+        end
+
+        def url
+          @url ||= client.get_queue_url(queue_name: name).queue_url
+        end
+
+        def receive_message(options = {})
+          client.receive_message(options.merge(queue_url: url))
+        end
+
+        def send_message(body, options = {})
+          params = options.merge(message_body: body, queue_url: url)
+          client.send_message(params)
+        end
+
+        def delete_message(receipt_handle)
+          params = { queue_url: url, receipt_handle: receipt_handle }
+          client.delete_message(params)
+        end
+
+        private
+
+        def sqs_attributes
+          @sqs_attributes ||= client.get_queue_attributes(queue_url: url)
+        end
+      end
+    end
+  end
+end

data/lib/txgh-queue/backends/sqs/retry_logic.rb

@@ -0,0 +1,118 @@
+require 'json'
+
+module TxghQueue
+  module Backends
+    module Sqs
+      class RetriesExceededError < StandardError; end
+
+      class RetryLogic
+        SEQUENTIAL_MAX_RETRIES = 5
+        OVERALL_MAX_RETRIES = 15
+
+        # SQS max is 15 minutes, or 900 seconds
+        DELAY_INTERVALS = [16, 32, 64, 128, 256, 512, 900]
+
+        attr_reader :message_attributes, :current_status
+
+        def initialize(message_attributes, current_status)
+          @message_attributes = message_attributes.dup
+          @current_status = current_status
+        end
+
+        def retry?
+          retry_with_delay? || retry_without_delay?
+        end
+
+        def retries_exceeded?
+          max_overall_retries_exceeded? ||
+            max_sequential_retries_exceeded? ||
+            max_sequential_delays_exceeded?
+        end
+
+        def next_delay_seconds
+          raise RetriesExceededError unless retry?
+          return 0 unless retry_with_delay?
+          DELAY_INTERVALS[[current_sequence.size - 1, 0].max]
+        end
+
+        def sqs_retry_params
+          if retry_without_delay?
+            sqs_params_without_delay
+          elsif retry_with_delay?
+            sqs_params_with_delay
+          else
+            raise RetriesExceededError
+          end
+        end
+
+        private
+
+        def retry_with_delay?
+          return false if max_overall_retries_exceeded?
+          return false unless current_status.retry_with_delay?
+          !max_sequential_delays_exceeded?
+        end
+
+        def retry_without_delay?
+          return false if max_overall_retries_exceeded?
+          return false unless current_status.retry_without_delay?
+          !max_sequential_retries_exceeded?
+        end
+
+        def sqs_params_without_delay
+          { message_attributes: message_attributes.to_h }
+        end
+
+        def sqs_params_with_delay
+          sqs_params_without_delay.merge(delay_seconds: next_delay_seconds)
+        end
+
+        def max_overall_retries_exceeded?
+          history_sequence.size >= OVERALL_MAX_RETRIES
+        end
+
+        def max_sequential_delays_exceeded?
+          delay_sequence_will_continue? && current_sequence.size >= DELAY_INTERVALS.size
+        end
+
+        def max_sequential_retries_exceeded?
+          retry_sequence_will_continue? && current_sequence.size >= SEQUENTIAL_MAX_RETRIES
+        end
+
+        def retry_sequence_will_continue?
+          current_status.status == last_status &&
+            current_status.retry_without_delay?
+        end
+
+        def delay_sequence_will_continue?
+          current_status.status == last_status &&
+            current_status.retry_with_delay?
+        end
+
+        def last_status
+          rt = current_sequence.last
+          rt.to_sym if rt
+        end
+
+        def current_sequence
+          return [] unless sequence = chunked_history_sequence.last
+          return [] unless last_elem = sequence.last
+          return [] unless current_status.status.to_s == last_elem
+          sequence
+        end
+
+        def chunked_history_sequence
+          @chunked_history_sequence ||= history_sequence.lazy
+            .map { |elem| elem[:status] }
+            .chunk(&:itself)
+            .map(&:last)
+            .to_a
+        end
+
+        def history_sequence
+          message_attributes.history_sequence
+        end
+      end
+    end
+  end
+end

data/lib/txgh-queue/backends/sqs.rb

@@ -0,0 +1,36 @@
+require 'txgh'
+
+module TxghQueue
+  module Backends
+    module Sqs
+      autoload :Config,            'txgh-queue/backends/sqs/config'
+      autoload :Consumer,          'txgh-queue/backends/sqs/consumer'
+      autoload :HistorySequence,   'txgh-queue/backends/sqs/history_sequence'
+      autoload :Job,               'txgh-queue/backends/sqs/job'
+      autoload :MessageAttributes, 'txgh-queue/backends/sqs/message_attributes'
+      autoload :Producer,          'txgh-queue/backends/sqs/producer'
+      autoload :Queue,             'txgh-queue/backends/sqs/queue'
+      autoload :RetryLogic,        'txgh-queue/backends/sqs/retry_logic'
+
+      class << self
+        def producer_for(events, logger = Txgh::TxLogger.logger)
+          Producer.new(find_queues_for(Array(events)), logger)
+        end
+
+        def consumer_for(events, logger = Txgh::TxLogger.logger)
+          Consumer.new(find_queues_for(Array(events)), logger)
+        end
+
+        private
+
+        def find_queues_for(events)
+          queues = events.flat_map do |event|
+            Config.queues.select { |queue| queue.events.include?(event) }
+          end
+
+          queues.uniq
+        end
+      end
+    end
+  end
+end

data/lib/txgh-queue/backends.rb

@@ -0,0 +1,22 @@
+module TxghQueue
+  module Backends
+    autoload :Null, 'txgh-queue/backends/null'
+    autoload :Sqs,  'txgh-queue/backends/sqs'
+
+    class BackendNotConfiguredError < StandardError; end
+
+    class << self
+      def register(name, klass)
+        all[name] = klass
+      end
+
+      def get(name)
+        all[name]
+      end
+
+      def all
+        @all ||= {}
+      end
+    end
+  end
+end

data/lib/txgh-queue/config.rb

@@ -0,0 +1,48 @@
+require 'yaml'
+
+module TxghQueue
+  class Config
+    DEFAULT_BACKEND = 'null'
+
+    class << self
+      def backend
+        TxghQueue::Backends.get(
+          raw_config.fetch(:backend, DEFAULT_BACKEND)
+        )
+      end
+
+      def processing_enabled?
+        raw_config.fetch(:processing_enabled, true)
+      end
+
+      def options
+        raw_config.fetch(:options, {})
+      end
+
+      def reset!
+        @raw_config = nil
+      end
+
+      private
+
+      def raw_config
+        @raw_config ||= begin
+          if ENV['TXGH_QUEUE_CONFIG']
+            scheme, payload = ENV['TXGH_QUEUE_CONFIG'].split('://')
+            send(:"load_#{scheme}", payload)
+          else
+            {}
+          end
+        end
+      end
+
+      def load_file(payload)
+        Txgh::Utils.deep_symbolize_keys(YAML.load_file(payload))
+      end
+
+      def load_raw(payload)
+        Txgh::Utils.deep_symbolize_keys(YAML.load(payload))
+      end
+    end
+  end
+end

data/lib/txgh-queue/error_handlers/github.rb

@@ -0,0 +1,47 @@
+require 'octokit'
+
+module TxghQueue
+  module ErrorHandlers
+    class Github
+      ERROR_CLASSES = {
+        Octokit::AbuseDetected           => Status.fail,
+        Octokit::BadGateway              => Status.retry_without_delay,
+        Octokit::BadRequest              => Status.fail,
+        Octokit::ClientError             => Status.fail,
+        Octokit::Conflict                => Status.fail,
+        Octokit::Forbidden               => Status.fail,
+        Octokit::InternalServerError     => Status.retry_with_delay,
+        Octokit::MethodNotAllowed        => Status.fail,
+        Octokit::NotAcceptable           => Status.fail,
+        Octokit::NotFound                => Status.fail,
+        Octokit::NotImplemented          => Status.fail,
+        Octokit::OneTimePasswordRequired => Status.fail,
+        Octokit::RepositoryUnavailable   => Status.retry_with_delay,
+        Octokit::ServerError             => Status.retry_with_delay,
+        Octokit::ServiceUnavailable      => Status.retry_with_delay,
+        Octokit::TooManyLoginAttempts    => Status.retry_with_delay,
+        Octokit::TooManyRequests         => Status.retry_with_delay,
+        Octokit::Unauthorized            => Status.fail,
+        Octokit::UnprocessableEntity     => Status.fail,
+        Octokit::UnsupportedMediaType    => Status.fail,
+        Octokit::UnverifiedEmail         => Status.fail
+      }
+
+      class << self
+        def can_handle?(error_or_response)
+          error_or_response.is_a?(Octokit::Error)
+        end
+
+        def status_for(error)
+          ERROR_CLASSES.fetch(error.class) { handle_other(error) }
+        end
+
+        private
+
+        def handle_other(error)
+          Status.fail
+        end
+      end
+    end
+  end
+end