txgh-queue 1.0.0

data/lib/txgh-queue/error_handlers/server_response.rb

@@ -0,0 +1,22 @@
+require 'txgh-server'
+
+module TxghQueue
+  module ErrorHandlers
+    class ServerResponse
+      class << self
+        def can_handle?(error_or_response)
+          error_or_response.is_a?(TxghServer::Response)
+        end
+
+        def status_for(response)
+          case response.status.to_i / 100
+            when 2
+              Status.ok
+            else
+              Status.fail
+          end
+        end
+      end
+    end
+  end
+end

data/lib/txgh-queue/error_handlers/standard_errors.rb

@@ -0,0 +1,15 @@
+module TxghQueue
+  module ErrorHandlers
+    class StandardErrors
+      class << self
+        def can_handle?(error_or_response)
+          error_or_response.is_a?(StandardError)
+        end
+
+        def status_for(response)
+          Status.fail
+        end
+      end
+    end
+  end
+end

data/lib/txgh-queue/error_handlers/transifex.rb

@@ -0,0 +1,23 @@
+require 'txgh'
+
+module TxghQueue
+  module ErrorHandlers
+    class Transifex
+      ERROR_CLASSES = {
+        Txgh::TransifexApiError          => Status.retry_with_delay,
+        Txgh::TransifexNotFoundError     => Status.fail,
+        Txgh::TransifexUnauthorizedError => Status.fail
+      }
+
+      class << self
+        def can_handle?(error_or_response)
+          ERROR_CLASSES.any? { |klass, _| error_or_response.class <= klass }
+        end
+
+        def status_for(error)
+          ERROR_CLASSES[error.class]
+        end
+      end
+    end
+  end
+end

data/lib/txgh-queue/error_handlers/txgh_errors.rb

@@ -0,0 +1,27 @@
+require 'txgh'
+
+module TxghQueue
+  module ErrorHandlers
+    class TxghErrors
+      ERROR_CLASSES = {
+        Txgh::ConfigNotFoundError        => Status.fail,
+        Txgh::GitConfigNotFoundError     => Status.fail,
+        Txgh::InvalidProviderError       => Status.fail,
+        Txgh::ProjectConfigNotFoundError => Status.fail,
+        Txgh::RepoConfigNotFoundError    => Status.fail,
+        Txgh::TxghError                  => Status.fail,
+        Txgh::TxghInternalError          => Status.fail
+      }
+
+      class << self
+        def can_handle?(error_or_response)
+          ERROR_CLASSES.any? { |klass, _| error_or_response.class <= klass }
+        end
+
+        def status_for(error)
+          ERROR_CLASSES[error.class]
+        end
+      end
+    end
+  end
+end

data/lib/txgh-queue/error_handlers.rb

@@ -0,0 +1,9 @@
+module TxghQueue
+  module ErrorHandlers
+    autoload :Github,         'txgh-queue/error_handlers/github'
+    autoload :ServerResponse, 'txgh-queue/error_handlers/server_response'
+    autoload :StandardErrors, 'txgh-queue/error_handlers/standard_errors'
+    autoload :Transifex,      'txgh-queue/error_handlers/transifex'
+    autoload :TxghErrors,     'txgh-queue/error_handlers/txgh_errors'
+  end
+end

data/lib/txgh-queue/job.rb

@@ -0,0 +1,77 @@
+require 'txgh'
+require 'txgh-server'
+
+module TxghQueue
+  class Job
+    Github = TxghServer::Webhooks::Github
+    Transifex = TxghServer::Webhooks::Transifex
+    include TxghServer::ResponseHelpers
+
+    attr_reader :logger
+
+    def initialize(logger)
+      @logger = logger
+    end
+
+    def process(payload)
+      Supervisor.supervise do
+        config = config_from(payload)
+        project = config.transifex_project
+        repo = config.github_repo
+
+        case payload.fetch('txgh_event')
+          when 'github.push'
+            handle_github_push(project, repo, payload)
+          when 'github.delete'
+            handle_github_delete(project, repo, payload)
+          when 'transifex.hook'
+            handle_transifex_hook(project, repo, payload)
+          else
+            handle_unexpected
+        end
+      end
+    end
+
+    private
+
+    def config_from(payload)
+      Txgh::Config::KeyManager.config_from_repo(payload.fetch('repo_name'))
+    end
+
+    def handle_github_push(project, repo, payload)
+      attributes = Github::PushAttributes.new(payload)
+      handler = Github::PushHandler.new(project, repo, logger, attributes)
+      execute(handler)
+    end
+
+    def handle_github_delete(project, repo, payload)
+      attributes = Github::DeleteAttributes.new(payload)
+      handler = Github::DeleteHandler.new(project, repo, logger, attributes)
+      execute(handler)
+    end
+
+    def handle_transifex_hook(project, repo, payload)
+      handler = Transifex::HookHandler.new(
+        project: project,
+        repo: repo,
+        resource_slug: payload['resource'],
+        language: payload['language'],
+        logger: logger
+      )
+
+      execute(handler)
+    end
+
+    def execute(handler)
+      if TxghQueue::Config.processing_enabled?
+        handler.execute
+      else
+        respond_with(200, 'Ok')
+      end
+    end
+
+    def handle_unexpected
+      respond_with_error(400, 'Unexpected event type')
+    end
+  end
+end

data/lib/txgh-queue/result.rb

@@ -0,0 +1,26 @@
+module TxghQueue
+  class Result
+    attr_reader :status, :response_or_error
+
+    def initialize(status, response_or_error)
+      @status = status
+      @response_or_error = response_or_error
+    end
+
+    def has_response?
+      response_or_error.is_a?(TxghServer::Response)
+    end
+
+    def response
+      return response_or_error if has_response?
+    end
+
+    def has_error?
+      response_or_error.is_a?(Exception)
+    end
+
+    def error
+      return response_or_error if has_error?
+    end
+  end
+end

data/lib/txgh-queue/status.rb

@@ -0,0 +1,51 @@
+module TxghQueue
+  class Status
+    class << self
+      def retry_without_delay
+        @retry ||= new(status: :retry_without_delay)
+      end
+
+      def retry_with_delay
+        @retry_with_delay ||= new(status: :retry_with_delay)
+      end
+
+      def fail
+        @fail ||= new(status: :fail)
+      end
+
+      def ok
+        @ok ||= new(status: :ok)
+      end
+    end
+
+    attr_reader :status
+
+    def initialize(options = {})
+      @status = options.fetch(:status)
+    end
+
+    def retry?
+      retry_with_delay? || retry_without_delay?
+    end
+
+    def retry_with_delay?
+      status == :retry_with_delay
+    end
+
+    def retry_without_delay?
+      status == :retry_without_delay
+    end
+
+    def fail?
+      status == :fail
+    end
+
+    def ok?
+      status == :ok
+    end
+
+    def to_s
+      status.to_s
+    end
+  end
+end

data/lib/txgh-queue/supervisor.rb

@@ -0,0 +1,76 @@
+require 'txgh'
+
+module TxghQueue
+  class UnexpectedResponse < StandardError; end
+
+  class Supervisor
+    # ErrorHandlers::StandardErrors should always come last as a catch-all for
+    # unexpected errors. All errors handled by this supervisor inherit from
+    # StandardError, so putting it too early in the handler list may cause an
+    # error to be mis-handled.
+    ERROR_HANDLERS = [
+      ErrorHandlers::ServerResponse,
+      ErrorHandlers::Github,
+      ErrorHandlers::Transifex,
+      ErrorHandlers::TxghErrors,
+      ErrorHandlers::StandardErrors
+    ]
+
+    class << self
+      def supervise(&block)
+        new(&block).execute
+      end
+    end
+
+    attr_reader :block
+
+    def initialize(&block)
+      @block = block
+    end
+
+    def execute
+      response = block.call
+    rescue StandardError => e
+      status = status_for_error(e)
+      Result.new(status, e)
+    else
+      status = status_for_response(response)
+      Result.new(status, response)
+    end
+
+    private
+
+    def status_for_response(response)
+      klass = find_error_handler_class_for(response)
+
+      unless klass
+        message = unexpected_response_error_message(response)
+        raise UnexpectedResponse, message
+      end
+
+      klass.status_for(response)
+    end
+
+    def unexpected_response_error_message(response)
+      return response unless response.respond_to?(:status)
+      return response unless response.respond_to?(:body)
+      "#{response.status} #{response.body}"
+    end
+
+    def status_for_error(error)
+      # Don't bother handling the case where find_behavior_class returns nil
+      # since it will realistically never occur. The execute method above
+      # rescues all StandardErrors, which will always be matched by
+      # ErrorBehavior::StandardErrors. In cases where errors that don't inherit
+      # from StandardError are raised, execute won't catch them and therefore
+      # won't call this method.
+      find_error_handler_class_for(error).status_for(error)
+    end
+
+    def find_error_handler_class_for(response_or_error)
+      ERROR_HANDLERS.find do |klass|
+        klass.can_handle?(response_or_error)
+      end
+    end
+  end
+end

data/lib/txgh-queue/version.rb

@@ -0,0 +1,3 @@
+module TxghQueue
+  VERSION = '1.0.0'
+end

data/lib/txgh-queue/webhooks/github/request_handler.rb

@@ -0,0 +1,34 @@
+require 'txgh-server'
+
+module TxghQueue
+  module Webhooks
+    module Github
+      class RequestHandler < ::TxghServer::Webhooks::Github::RequestHandler
+        def handle_request
+          handle_safely do
+            case github_event
+              when 'push', 'delete'
+                txgh_event = "github.#{github_event}"
+
+                result = ::TxghQueue::Config.backend
+                  .producer_for(txgh_event, logger)
+                  .enqueue(attributes.to_h.merge(txgh_event: txgh_event))
+
+                respond_with(202, result.to_json)
+              when 'ping'
+                ping_handler.execute
+              else
+                respond_with_error(400, "Event '#{github_event}' cannot be enqueued")
+            end
+          end
+        end
+
+        private
+
+        def ping_handler
+          @ping_handler ||= TxghServer::Webhooks::Github::PingHandler.new(logger)
+        end
+      end
+    end
+  end
+end

data/lib/txgh-queue/webhooks/github.rb

@@ -0,0 +1,7 @@
+module TxghQueue
+  module Webhooks
+    module Github
+      autoload :RequestHandler, 'txgh-queue/webhooks/github/request_handler'
+    end
+  end
+end

data/lib/txgh-queue/webhooks/transifex/request_handler.rb

@@ -0,0 +1,21 @@
+require 'txgh-server'
+
+module TxghQueue
+  module Webhooks
+    module Transifex
+      class RequestHandler < ::TxghServer::Webhooks::Transifex::RequestHandler
+        TXGH_EVENT = 'transifex.hook'
+
+        def handle_request
+          handle_safely do
+            result = ::TxghQueue::Config.backend
+              .producer_for(TXGH_EVENT, logger)
+              .enqueue(payload.merge(txgh_event: TXGH_EVENT))
+
+            respond_with(202, result.to_json)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/txgh-queue/webhooks/transifex.rb

@@ -0,0 +1,7 @@
+module TxghQueue
+  module Webhooks
+    module Transifex
+      autoload :RequestHandler, 'txgh-queue/webhooks/transifex/request_handler'
+    end
+  end
+end

data/lib/txgh-queue/webhooks.rb

@@ -0,0 +1,6 @@
+module TxghQueue
+  module Webhooks
+    autoload :Github,    'txgh-queue/webhooks/github'
+    autoload :Transifex, 'txgh-queue/webhooks/transifex'
+  end
+end

data/lib/txgh-queue.rb

@@ -0,0 +1,14 @@
+module TxghQueue
+  autoload :Backends,         'txgh-queue/backends'
+  autoload :Config,           'txgh-queue/config'
+  autoload :ErrorHandlers,    'txgh-queue/error_handlers'
+  autoload :Job,              'txgh-queue/job'
+  autoload :Status,           'txgh-queue/status'
+  autoload :Supervisor,       'txgh-queue/supervisor'
+  autoload :Result,           'txgh-queue/result'
+  autoload :Webhooks,         'txgh-queue/webhooks'
+  autoload :WebhookEndpoints, 'txgh-queue/application'
+
+  Backends.register('null', TxghQueue::Backends::Null)
+  Backends.register('sqs',  TxghQueue::Backends::Sqs)
+end

data/spec/application_spec.rb

@@ -0,0 +1,97 @@
+require 'spec_helper'
+require 'rack/test'
+
+require 'helpers/github_payload_builder'
+require 'helpers/standard_txgh_setup'
+
+describe TxghQueue::WebhookEndpoints, auto_configure: true do
+  include Rack::Test::Methods
+  include StandardTxghSetup
+  include TxghServer::ResponseHelpers
+
+  def app
+    TxghQueue::WebhookEndpoints
+  end
+
+  let(:config) do
+    Txgh::Config::ConfigPair.new(project_config, repo_config)
+  end
+
+  let(:backend) { TxghQueue::Config.backend }
+
+  describe '/transifex/enqueue' do
+    def sign_with(body)
+      header(
+        TxghServer::TransifexRequestAuth::TRANSIFEX_HEADER,
+        TxghServer::TransifexRequestAuth.header_value(
+          body, config.transifex_project.webhook_secret
+        )
+      )
+    end
+
+    let(:producer) { backend.producer_for('transifex.hook') }
+    let(:params) do
+      {
+        'project' => project_name,
+        'resource' => resource_slug,
+        'language' => language,
+        'translated' => '100'
+      }
+    end
+
+    it 'enqueues a new job' do
+      payload = URI.encode_www_form(params.to_a)
+      sign_with payload
+
+      expect { post '/transifex/enqueue', payload }.to(
+        change { producer.enqueued_jobs.size }.from(0).to(1)
+      )
+
+      expect(last_response).to be_accepted
+
+      job = producer.enqueued_jobs.first
+      expect(job[:payload]).to include(
+        project: project_name,
+        resource: resource_slug,
+        language: language,
+        translated: '100',
+        txgh_event: 'transifex.hook'
+      )
+    end
+  end
+
+  describe '/github/enqueue' do
+    def sign_with(body)
+      header(
+        TxghServer::GithubRequestAuth::GITHUB_HEADER,
+        TxghServer::GithubRequestAuth.header_value(
+          body, config.github_repo.webhook_secret
+        )
+      )
+    end
+
+    let(:producer) { backend.producer_for('github.push') }
+
+    it 'enqueues a new job' do
+      payload = GithubPayloadBuilder.push_payload(repo_name, ref)
+      payload.add_commit
+
+      sign_with payload.to_json
+      header 'X-GitHub-Event', 'push'
+
+      expect { post '/github/enqueue', payload.to_json }.to(
+        change { producer.enqueued_jobs.size }.from(0).to(1)
+      )
+
+      expect(last_response).to be_accepted
+
+      job = producer.enqueued_jobs.first
+      expect(job[:payload]).to include(
+        event: 'push',
+        txgh_event: 'github.push',
+        repo_name: repo_name,
+        ref: "refs/#{ref}"
+      )
+    end
+  end
+end

data/spec/backends/sqs/config_spec.rb

@@ -0,0 +1,30 @@
+require 'spec_helper'
+
+include TxghQueue::Backends
+
+describe Sqs::Config, auto_configure: true do
+  let(:queue_config) { sqs_queue_config }
+
+  describe '.queues' do
+    it 'lists all queues' do
+      queues = described_class.queues
+      queues.each { |q| expect(q).to be_a(Sqs::Queue) }
+      expect(queues.map(&:name).sort).to eq(%w(test-queue test-queue-2))
+    end
+  end
+
+  describe '.failure_queue' do
+    it 'returns the failure queue' do
+      expect(described_class.failure_queue).to be_a(Sqs::Queue)
+      expect(described_class.failure_queue.name).to eq('test-failure-queue')
+    end
+  end
+
+  describe '.get_queue' do
+    it 'pulls out a single queue object' do
+      queue = described_class.get_queue('test-queue')
+      expect(queue).to be_a(Sqs::Queue)
+      expect(queue.name).to eq('test-queue')
+    end
+  end
+end

data/spec/backends/sqs/consumer_spec.rb

@@ -0,0 +1,34 @@
+require 'spec_helper'
+require 'helpers/sqs/sqs_test_message'
+
+include TxghQueue
+include TxghQueue::Backends
+
+describe Sqs::Consumer, auto_configure: true do
+  let(:queue_config) { sqs_queue_config }
+  let(:queues) { Sqs::Config.queues }
+  let(:logger) { NilLogger.new }
+  let(:message) { SqsTestMessage.new('abc123', '{}') }
+  let(:consumer) { described_class.new(queues, logger) }
+
+  it 'executes one job in each queue' do
+    queues.each do |queue|
+      job = double(:Job)
+      expect(queue).to receive(:receive_message).and_return(message.to_bundle)
+      expect(job).to receive(:complete)
+      expect(Sqs::Job).to(
+        receive(:new).with(message, queue, logger).and_return(job)
+      )
+    end
+
+    consumer.work
+  end
+
+  it 'reports errors' do
+    errors = []
+    Txgh.events.subscribe('errors') { |e| errors << e }
+    expect(queues.first).to receive(:receive_message).and_raise(StandardError, 'jelly beans')
+    expect { consumer.work }.to change { errors.size }.from(0).to(1)
+    expect(errors.first.message).to eq('jelly beans')
+  end
+end

data/spec/backends/sqs/history_sequence_spec.rb

@@ -0,0 +1,75 @@
+require 'spec_helper'
+require 'helpers/sqs/sqs_test_message'
+
+include TxghQueue
+include TxghQueue::Backends
+
+describe Sqs::HistorySequence do
+  let(:message) { SqsTestMessage.new('abc123', '{}', attributes_hash) }
+  let(:attributes_hash) do
+    {
+      'history_sequence' => {
+        'string_value' => [{
+          'status' => 'retry_without_delay'
+        }].to_json
+      }
+    }
+  end
+
+  describe '.from_message' do
+    it 'extracts the correct attributes from the given hash' do
+      sequence = described_class.from_message(message)
+      expect(sequence.sequence).to eq([
+        { status: 'retry_without_delay' }
+      ])
+    end
+  end
+
+  describe '.from_h' do
+    it 'extracts the correct attributes from the given hash' do
+      sequence = described_class.from_h(attributes_hash)
+      expect(sequence.sequence).to eq([
+        { status: 'retry_without_delay' }
+      ])
+    end
+  end
+
+  context 'with a sequence' do
+    let(:sequence) { described_class.from_message(message) }
+
+    describe '#add' do
+      it 'adds the given object to the sequence' do
+        expect { sequence.add('abc') }.to(
+          change { sequence.sequence.size }.by(1)
+        )
+
+        expect(sequence.sequence.last).to eq('abc')
+      end
+    end
+
+    describe '#to_h' do
+      it 'serializes the sequence into a hash' do
+        expect(sequence.to_h).to eq(
+          string_value: [{'status' => 'retry_without_delay'}].to_json,
+          data_type: 'String'
+        )
+      end
+    end
+
+    describe '#dup' do
+      it 'deep copies the sequence' do
+        copied_sequence = sequence.dup
+        expect(sequence.object_id).to_not eq(copied_sequence.object_id)
+        expect(sequence.sequence.object_id).to_not(
+          eq(copied_sequence.sequence.object_id)
+        )
+      end
+    end
+
+    describe '#current' do
+      it 'returns the last element in the sequence' do
+        expect(sequence.current).to eq(status: 'retry_without_delay')
+      end
+    end
+  end
+end