txgh-queue 1.0.0

data/spec/backends/sqs/job_spec.rb

@@ -0,0 +1,189 @@
+require 'spec_helper'
+require 'helpers/sqs/sqs_test_message'
+
+include TxghQueue
+include TxghQueue::Backends
+
+describe Sqs::Job, auto_configure: true do
+  let(:queue_config) { sqs_queue_config }
+  let(:queue) { Sqs::Config.queues.first }
+  let(:failure_queue) { Sqs::Config.failure_queue }
+  let(:logger) { NilLogger.new }
+  let(:body) { { 'foo' => 'bar' } }
+  let(:message) { SqsTestMessage.new('123abc', body.to_json) }
+  let(:new_message) { SqsTestMessage.new('456def', body.to_json) }
+  let(:job) { described_class.new(message, queue, logger) }
+
+  shared_examples "it updates the message's history sequence" do |status, queue_sym|
+    it 'updates the history sequence with failure details for an exception' do
+      error = StandardError.new('foobar')
+      error.set_backtrace('path/to/file.rb:10')
+      result = Result.new(status, error)
+      expect(job).to receive(:process).with(body).and_return(result)
+
+      # this call to send_message signifies a retry
+      expect(send(queue_sym)).to receive(:send_message) do |body, attributes|
+        message_attributes = Sqs::MessageAttributes.from_h(attributes[:message_attributes])
+        current_retry = message_attributes.history_sequence.current
+        expect(current_retry).to include(
+          response_type: 'error',
+          class: 'StandardError',
+          message: 'foobar',
+          backtrace: 'path/to/file.rb:10'
+        )
+
+        new_message
+      end
+
+      job.complete
+    end
+
+    it 'updates the history sequence with failure details for a txgh response' do
+      response = TxghServer::Response.new(502, 'Bad gateway')
+      result = Result.new(status, response)
+      expect(job).to receive(:process).with(body).and_return(result)
+
+      # this call to send_message signifies a retry
+      expect(send(queue_sym)).to receive(:send_message) do |body, attributes|
+        message_attributes = Sqs::MessageAttributes.from_h(attributes[:message_attributes])
+        current_retry = message_attributes.history_sequence.current
+        expect(current_retry).to include(
+          response_type: 'response',
+          code: 502,
+          body: 'Bad gateway'
+        )
+
+        new_message
+      end
+
+      job.complete
+    end
+  end
+
+  describe '#complete' do
+    it 'processes a single job and deletes the message' do
+      result = Result.new(Status.ok, TxghServer::Response.new(200, 'Ok'))
+      expect(queue).to receive(:delete_message).with(message.receipt_handle)
+      expect(job).to receive(:process).with(body).and_return(result)
+      job.complete
+    end
+
+    context 'error reporting' do
+      let(:error) { StandardError.new('jelly beans') }
+      let(:result) { Result.new(Status.fail, error) }
+
+      before(:each) do
+        expect(job).to receive(:process).and_return(result)
+        expect(queue).to receive(:delete_message).with(message.receipt_handle)
+      end
+
+      it 'reports errors to the event system' do
+        expect(failure_queue).to receive(:send_message)
+
+        expect(Txgh.events).to receive(:publish_error) do |e, params|
+          expect(e).to eq(error)
+          expect(params).to eq(
+            payload: body, message_id: message.message_id, queue: queue.name
+          )
+        end
+
+        job.complete
+      end
+
+      it 'includes error tracking details returned from publishing the event' do
+        expect(Txgh.events).to receive(:publish_error).and_return(foo: 'bar')
+        expect(failure_queue).to receive(:send_message) do |body, attributes|
+          message_attributes = Sqs::MessageAttributes.from_h(attributes[:message_attributes])
+          current_retry = message_attributes.history_sequence.current
+          expect(current_retry).to include(error_tracking: { foo: 'bar' })
+        end
+
+        job.complete
+      end
+    end
+
+    context 'retries' do
+      let(:response) { TxghServer::Response.new(502, 'Bad gateway') }
+
+      before(:each) do
+        # should delete the old message
+        expect(queue).to receive(:delete_message).with(message.receipt_handle)
+      end
+
+      it 're-enqueues the message if told to retry' do
+        response = TxghServer::Response.new(502, 'Bad gateway')
+        result = Result.new(Status.retry_without_delay, response)
+        expect(job).to receive(:process).with(body).and_return(result)
+
+        # this call to send_message signifies a retry
+        expect(queue).to receive(:send_message) do |body, attributes|
+          message_attributes = Sqs::MessageAttributes.from_h(attributes[:message_attributes])
+          history_sequence = message_attributes.history_sequence.sequence.map { |elem| elem[:status] }
+          expect(history_sequence).to eq(%w(retry_without_delay))
+          new_message
+        end
+
+        job.complete
+      end
+
+      it 're-enqueues with delay if told to do so' do
+        response = TxghServer::Response.new(502, 'Bad gateway')
+        result = Result.new(Status.retry_with_delay, response)
+        expect(job).to receive(:process).with(body).and_return(result)
+
+        # this call to send_message signifies a retry
+        expect(queue).to receive(:send_message) do |body, attributes|
+          message_attributes = Sqs::MessageAttributes.from_h(attributes[:message_attributes])
+          history_sequence = message_attributes.history_sequence.sequence.map { |elem| elem[:status] }
+          expect(history_sequence).to eq(%w(retry_with_delay))
+
+          expect(attributes[:delay_seconds]).to eq(
+            Sqs::RetryLogic::DELAY_INTERVALS.first
+          )
+
+          new_message
+        end
+
+        job.complete
+      end
+
+      context 'with all retries exceeded' do
+        let(:message) do
+          SqsTestMessage.new('123abc', body.to_json, {
+            'history_sequence' => {
+              'string_value' => (Sqs::RetryLogic::OVERALL_MAX_RETRIES - 1).times.map do
+                { status: 'retry_without_delay' }
+              end.to_json
+            }
+          })
+        end
+
+        it 'deletes the message and adds it to the failure queue' do
+          response = TxghServer::Response.new(502, 'Bad gateway')
+          result = Result.new(Status.retry_with_delay, response)
+          expect(job).to receive(:process).with(body).and_return(result)
+          expect(failure_queue).to receive(:send_message).and_return(new_message)
+          job.complete
+        end
+      end
+
+      it_behaves_like "it updates the message's history sequence", Status.retry_without_delay, :queue
+      it_behaves_like "it updates the message's history sequence", Status.retry_with_delay, :queue
+    end
+
+    context 'failures' do
+      before(:each) do
+        expect(queue).to receive(:delete_message).with(message.receipt_handle)
+      end
+
+      it 'sends the message to the failure queue' do
+        result = Result.new(Status.fail, TxghServer::Response.new(500, '💩'))
+        expect(failure_queue).to receive(:send_message).with(body.to_json, anything)
+        expect(job).to receive(:process).with(body).and_return(result)
+        job.complete
+      end
+
+      it_behaves_like "it updates the message's history sequence", Status.fail, :failure_queue
+    end
+  end
+end

data/spec/backends/sqs/message_attributes_spec.rb

@@ -0,0 +1,64 @@
+require 'spec_helper'
+require 'helpers/sqs/sqs_test_message'
+require 'json'
+
+include TxghQueue
+include TxghQueue::Backends
+
+describe Sqs::MessageAttributes do
+  let(:message) { SqsTestMessage.new('abc123', '{}', attributes_hash) }
+  let(:attributes_hash) do
+    {
+      'history_sequence' => {
+        'string_value' => [{
+          'status' => 'retry_without_delay'
+        }].to_json
+      }
+    }
+  end
+
+  describe '.from_message' do
+    it 'extracts the history sequence from the message attributes' do
+      attributes = described_class.from_message(message)
+      expect(attributes.history_sequence).to be_a(Sqs::HistorySequence)
+      expect(attributes.history_sequence.sequence.first[:status]).to(
+        eq('retry_without_delay')
+      )
+    end
+  end
+
+  describe '.from_h' do
+    it 'creates the history sequence from the hash elements' do
+      attributes = described_class.from_h(attributes_hash)
+      expect(attributes.history_sequence).to be_a(Sqs::HistorySequence)
+      expect(attributes.history_sequence.sequence.first[:status]).to(
+        eq('retry_without_delay')
+      )
+    end
+  end
+
+  describe '#to_h' do
+    it 'converts the history sequence into a hash' do
+      attributes = described_class.from_message(message).to_h
+      expect(attributes).to eq(
+        history_sequence: {
+          data_type: 'String',
+          string_value: [
+            status: 'retry_without_delay'
+          ].to_json
+        }
+      )
+    end
+  end
+
+  describe '#dup' do
+    it 'duplicates the attributes' do
+      original_attributes = described_class.from_message(message)
+      copied_attributes = original_attributes.dup
+      expect(original_attributes.object_id).to_not eq(copied_attributes.object_id)
+      expect(original_attributes.history_sequence.object_id).to_not(
+        eq(copied_attributes.object_id)
+      )
+    end
+  end
+end

data/spec/backends/sqs/producer_spec.rb

@@ -0,0 +1,32 @@
+require 'spec_helper'
+require 'securerandom'
+
+include TxghQueue::Backends
+
+describe Sqs::Producer, auto_configure: true do
+  let(:queue_config) { sqs_queue_config }
+  let(:logger) { NilLogger.new }
+  let(:queues) { TxghQueue::Backends::Sqs::Config.queues }
+  let(:producer) { described_class.new(queues, logger) }
+
+  describe '#enqueue' do
+    it 'sends a message to each SQS queue' do
+      payload = { abc: 'def' }
+      message_ids = []
+
+      queues.each do |queue|
+        message_ids << SecureRandom.hex
+        message = double(:Message, message_id: message_ids.last)
+
+        expect(queue).to(
+          receive(:send_message)
+            .with(payload.to_json, foo: 'bar')
+            .and_return(message)
+        )
+      end
+
+      result = producer.enqueue(payload, foo: 'bar')
+      expect(result).to eq(message_ids: message_ids)
+    end
+  end
+end

data/spec/backends/sqs/queue_spec.rb

@@ -0,0 +1,65 @@
+require 'spec_helper'
+
+include TxghQueue::Backends
+
+describe Sqs::Queue do
+  let(:options) { { name: 'test-queue', region: 'us-east-1', events: %w(a b c) } }
+  let(:queue) { described_class.new(options) }
+
+  describe '#client' do
+    it 'instantiates an AWS SQS client' do
+      expect(queue.client).to be_a(Aws::SQS::Client)
+    end
+  end
+
+  describe '#url' do
+    it 'grabs the queue URL from the SQS client' do
+      queue_url = double(:QueueUrl, queue_url: 'test-queue')
+
+      expect(queue.client).to(
+        receive(:get_queue_url)
+          .with(queue_name: 'test-queue')
+          .and_return(queue_url)
+      )
+
+      queue.url
+    end
+  end
+
+  describe '#receive_message' do
+    it 'proxies to the client' do
+      url = 'test://host'
+      allow(queue).to receive(:url).and_return(url)
+      expect(queue.client).to receive(:receive_message).with(queue_url: url, foo: 'bar')
+      queue.receive_message(foo: 'bar')
+    end
+  end
+
+  describe '#send_message' do
+    it 'proxies to the client' do
+      url = 'test://host'
+      body = 'All your base are belong to us'
+      allow(queue).to receive(:url).and_return(url)
+
+      expect(queue.client).to(
+        receive(:send_message).with(message_body: body, queue_url: url, foo: 'bar')
+      )
+
+      queue.send_message(body, foo: 'bar')
+    end
+  end
+
+  describe '#delete_message' do
+    it 'proxies to the client' do
+      url = 'test://host'
+      receipt_handle = 'abc123'
+      allow(queue).to receive(:url).and_return(url)
+
+      expect(queue.client).to(
+        receive(:delete_message).with(queue_url: url, receipt_handle: receipt_handle)
+      )
+
+      queue.delete_message(receipt_handle)
+    end
+  end
+end

data/spec/backends/sqs/retry_logic_spec.rb

@@ -0,0 +1,157 @@
+require 'spec_helper'
+require 'helpers/sqs/sqs_test_message'
+
+include TxghQueue
+include TxghQueue::Backends
+
+describe Sqs::RetryLogic do
+  context 'with overall retries exceeded' do
+    let(:logic) { described_class.new(message_attributes, current_status) }
+    let(:current_status) { Status.retry_without_delay }
+    let(:message) { SqsTestMessage.new('abc123', '{}', message_attributes.to_h) }
+    let(:message_attributes) do
+      Sqs::MessageAttributes.from_h(
+        history_sequence: {
+          string_value: described_class::OVERALL_MAX_RETRIES.times.map do
+            { status: 'retry_without_delay' }
+          end.to_json
+        }
+      )
+    end
+
+    describe '#retries_exceeded?' do
+      it 'indicates retries have been exceeded' do
+        expect(logic.retries_exceeded?).to eq(true)
+      end
+    end
+
+    describe '#retry?' do
+      it 'indicates another retry should not be attempted' do
+        expect(logic.retry?).to eq(false)
+      end
+    end
+
+    describe '#next_delay_seconds' do
+      it 'raises an error' do
+        expect { logic.next_delay_seconds }.to raise_error(Sqs::RetriesExceededError)
+      end
+    end
+
+    describe '#sqs_retry_params' do
+      it 'raises an error' do
+        expect { logic.sqs_retry_params }.to raise_error(Sqs::RetriesExceededError)
+      end
+    end
+  end
+
+  context 'with a run of no-delay retries' do
+    let(:logic) { described_class.new(message_attributes, current_status) }
+    let(:current_status) { Status.retry_without_delay }
+    let(:message) { SqsTestMessage.new('abc123', '{}', message_attributes.to_h) }
+    let(:message_attributes) do
+      Sqs::MessageAttributes.from_h(
+        history_sequence: {
+          string_value: [
+            { status: 'retry_without_delay' },
+            { status: 'retry_without_delay' },
+            { status: 'retry_without_delay' }
+          ].to_json
+        }
+      )
+    end
+
+    describe '#retries_exceeded?' do
+      it 'indicates retries have not been exceeded' do
+        expect(logic.retries_exceeded?).to eq(false)
+      end
+    end
+
+    describe '#retry?' do
+      it 'indicates another retry may be attempted' do
+        expect(logic.retry?).to eq(true)
+      end
+    end
+
+    describe '#next_delay_seconds' do
+      it 'indicates a delay of zero seconds' do
+        expect(logic.next_delay_seconds).to eq(0)
+      end
+    end
+
+    describe '#sqs_retry_params' do
+      it 'contains the correct parameters' do
+        expect(logic.sqs_retry_params[:message_attributes]).to(
+          eq(message_attributes.to_h)
+        )
+      end
+    end
+
+    context 'and a delayed current status' do
+      let(:current_status) { Status.retry_with_delay }
+
+      describe '#next_delay_seconds' do
+        it 'indicates a first-stage delay' do
+          expect(logic.next_delay_seconds).to(
+            eq(described_class::DELAY_INTERVALS.first)
+          )
+        end
+      end
+    end
+  end
+
+  context 'with a run of delayed retries' do
+    let(:logic) { described_class.new(message_attributes, current_status) }
+    let(:current_status) { Status.retry_with_delay }
+    let(:message) { SqsTestMessage.new('abc123', '{}', message_attributes.to_h) }
+    let(:message_attributes) do
+      Sqs::MessageAttributes.from_h(
+        history_sequence: {
+          string_value: [
+            { status: 'retry_with_delay' },
+            { status: 'retry_with_delay' },
+            { status: 'retry_with_delay' }
+          ].to_json
+        }
+      )
+    end
+
+    describe '#retries_exceeded?' do
+      it 'indicates retries have not been exceeded' do
+        expect(logic.retries_exceeded?).to eq(false)
+      end
+    end
+
+    describe '#retry?' do
+      it 'indicates another retry may be attempted' do
+        expect(logic.retry?).to eq(true)
+      end
+    end
+
+    describe '#next_delay_seconds' do
+      it 'indicates a third-stage delay' do
+        expect(logic.next_delay_seconds).to eq(
+          described_class::DELAY_INTERVALS[2]
+        )
+      end
+    end
+
+    describe '#sqs_retry_params' do
+      it 'contains the correct parameters, including the delay' do
+        expect(logic.sqs_retry_params).to eq(
+          message_attributes: message_attributes.to_h,
+          delay_seconds: described_class::DELAY_INTERVALS[2]
+        )
+      end
+    end
+
+    context 'and a non-delayed current status' do
+      let(:current_status) { Status.retry_without_delay }
+
+      describe '#next_delay_seconds' do
+        it 'indicates no delay' do
+          expect(logic.next_delay_seconds).to eq(0)
+        end
+      end
+    end
+  end
+end

data/spec/backends/sqs_spec.rb

@@ -0,0 +1,57 @@
+require 'spec_helper'
+
+include TxghQueue::Backends
+
+describe Sqs, auto_configure: true do
+  let(:queue_config) { sqs_queue_config }
+
+  describe '.producer_for' do
+    it 'looks up the queues for the given event and returns a producer object' do
+      producer = described_class.producer_for('a')
+      expect(producer).to be_a(Sqs::Producer)
+      expect(producer.queues.size).to eq(1)
+      expect(producer.queues.first).to be_a(Sqs::Queue)
+      expect(producer.queues.first.name).to eq('test-queue')
+    end
+
+    it 'looks up multiple events if given an array' do
+      producer = described_class.producer_for(%w(a d))
+      expect(producer.queues.size).to eq(2)
+      expect(producer.queues.map(&:name).sort).to eq(%w(test-queue test-queue-2))
+    end
+
+    it 'dedupes the list of matching queues' do
+      producer = described_class.producer_for(%w(a b c d e f))
+      expect(producer.queues.size).to eq(2)
+      expect(producer.queues.map(&:name).sort).to eq(%w(test-queue test-queue-2))
+    end
+  end
+
+  describe '.consumer_for' do
+    it 'looks up the queue for the given event and returns a consumer object' do
+      consumer = described_class.consumer_for('b')
+      expect(consumer).to be_a(Sqs::Consumer)
+      expect(consumer.queues.size).to eq(1)
+      expect(consumer.queues.first).to be_a(Sqs::Queue)
+      expect(consumer.queues.first.name).to eq('test-queue')
+    end
+
+    it 'handles the case if the event matches multiple queues' do
+      consumer = described_class.consumer_for('c')
+      expect(consumer.queues.size).to eq(2)
+      expect(consumer.queues.map(&:name).sort).to eq(%w(test-queue test-queue-2))
+    end
+
+    it 'looks up multiple events if given an array' do
+      consumer = described_class.consumer_for(%w(a d))
+      expect(consumer.queues.size).to eq(2)
+      expect(consumer.queues.map(&:name).sort).to eq(%w(test-queue test-queue-2))
+    end
+
+    it 'dedupes the list of matching queues' do
+      consumer = described_class.consumer_for(%w(a b c d e f))
+      expect(consumer.queues.size).to eq(2)
+      expect(consumer.queues.map(&:name).sort).to eq(%w(test-queue test-queue-2))
+    end
+  end
+end

data/spec/backends_spec.rb

@@ -0,0 +1,31 @@
+require 'spec_helper'
+
+include TxghQueue
+
+describe Backends do
+  describe '.register' do
+    it 'associates the name of a backend with a class' do
+      # use a string just for testing purposes; second argument should be a class
+      described_class.register(:foo, 'BarClass')
+      expect(described_class.all[:foo]).to eq('BarClass')
+    end
+  end
+
+  describe '.get' do
+    it 'retrieves the class for the given name' do
+      described_class.register(:foo, 'BarClass')
+      expect(described_class.get(:foo)).to eq('BarClass')
+    end
+  end
+
+  describe '.all' do
+    it 'returns a hash of all the name/class pairs' do
+      described_class.register(:foo, 'BarClass')
+      described_class.register(:baz, 'BooClass')
+
+      expect(described_class.all).to include(
+        foo: 'BarClass', baz: 'BooClass'
+      )
+    end
+  end
+end

data/spec/config_spec.rb

@@ -0,0 +1,15 @@
+require 'spec_helper'
+
+describe TxghQueue::Config, auto_configure: true do
+  describe '.backend' do
+    it 'identifies the class to use for the queue backend' do
+      expect(described_class.backend).to eq(TxghQueue::TestBackend)
+    end
+  end
+
+  describe '.options' do
+    it 'identifies the backend options' do
+      expect(described_class.options).to eq(queue_config[:options])
+    end
+  end
+end

data/spec/error_handlers/github_spec.rb

@@ -0,0 +1,30 @@
+require 'spec_helper'
+
+include TxghQueue
+
+describe ErrorHandlers::Github do
+  describe '.can_handle?' do
+    it 'can reply to all configured error classes' do
+      described_class::ERROR_CLASSES.keys.each do |klass|
+        expect(described_class.can_handle?(klass.new)).to eq(true)
+      end
+    end
+
+    it "can't reply to unsupported error classes" do
+      expect(described_class.can_handle?(StandardError.new)).to eq(false)
+    end
+  end
+
+  describe '.status_for' do
+    it 'replies to all configured errors correctly' do
+      described_class::ERROR_CLASSES.each_pair do |klass, expected_response|
+        expect(described_class.status_for(klass.new)).to eq(expected_response)
+      end
+    end
+
+    it 'replies to all unconfigured errors with fail' do
+      # i.e. if octokit raises an error we didn't account for
+      expect(described_class.status_for(StandardError.new)).to eq(Status.fail)
+    end
+  end
+end

data/spec/error_handlers/server_response_spec.rb

@@ -0,0 +1,36 @@
+require 'spec_helper'
+
+include TxghQueue
+
+describe ErrorHandlers::ServerResponse do
+  describe '.can_handle?' do
+    it 'can reply to TxghServer responses' do
+      server_response = TxghServer::Response.new(200, 'Ok')
+      expect(described_class.can_handle?(server_response)).to eq(true)
+    end
+
+    it "can't reply to anything else" do
+      expect(described_class.can_handle?('foo')).to eq(false)
+    end
+  end
+
+  describe '.status_for' do
+    it 'replies with ok if the status code is in the 200 range' do
+      server_response = TxghServer::Response.new(201, 'Created')
+      reply = described_class.status_for(server_response)
+      expect(reply).to eq(Status.ok)
+    end
+
+    it 'replies with fail if the status code is in the 400 range' do
+      server_response = TxghServer::Response.new(404, 'Not found')
+      reply = described_class.status_for(server_response)
+      expect(reply).to eq(Status.fail)
+    end
+
+    it 'replies with fail if the status code is in the 500 range' do
+      server_response = TxghServer::Response.new(502, 'Bad gateway')
+      reply = described_class.status_for(server_response)
+      expect(reply).to eq(Status.fail)
+    end
+  end
+end