alephant-publisher-queue 2.5.0 → 2.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: bf29d6e5e6cd44bc99c3d654a013e57afb4191a2
-  data.tar.gz: 019f8916b0859fb3e1eca573c08edff35ee8cd04
+  metadata.gz: 9d85acb1700dc11db1d701833055fc681c829de1
+  data.tar.gz: a409bff70bf7b7f3122d1eacf43dc0893c638ddc
 SHA512:
-  metadata.gz: 11bcdfdb4998721749b617dcfb86196cf3d780666ee11f2d93f230270a0192653d71fd9e187a1bb99e9684b09c0d9926120e11466ec642a85b0d35fef208c757
-  data.tar.gz: 37aa6ffdd3f8e356897d6f23184347c136ffd632211b75f35d1e8037d756582351df6e498bf4bbd72cddd6a4c287a462a73fee831f9a8750a5f405e03c784497
+  metadata.gz: 4ccf8e8b65b304ce50df32efa571ef3942d0bbd7675648aadb1d0c541665ebc7f4b2a135f4d3f2a1364404346b255d8ca66e25b231fbfe14909ea4c2cca53a7d
+  data.tar.gz: 24f6bc86e7397e5a0062a82357107d49ceef1dbb70a072015246399c4b9b901ed54e66e9a5a293e5a9b68108dcc9b699b242a26f02ea79a725d3a5afed6e235f

data/.ruby-version

@@ -1 +1 @@
-jruby-1.7.17
+2.3

data/.travis.yml

@@ -1,24 +1,10 @@
-sudo: false
-cache: bundler
 language: ruby
-
-# Ruby build matrix
 rvm:
-  - 2.0
-  - 2.1
-  - 2.2
-  - 2.3.0
-  - 2.3.1
-  - jruby
-
-# Environment variables
-env:
-  - RUBYOPT="-W0"
-
-# Ensure we don't build for *every* commit (doesn't apply to PR builds)
-branches:
-  only:
-  - master
-
-script:
-  - bundle exec rspec --format documentation
+  - "2.3"
+  - "jruby"
+notifications:
+  email:
+    recipients:
+      - D&ENewsFrameworksTeam@bbc.co.uk
+    on_failure: change
+    on_success: never

data/alephant-publisher-queue.gemspec

@@ -30,14 +30,14 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "simplecov"
 
   spec.add_runtime_dependency "rake"
-  spec.add_runtime_dependency "aws-sdk", "~> 1.0"
+  spec.add_runtime_dependency "aws-sdk-sqs"
   spec.add_runtime_dependency "crimp"
   spec.add_runtime_dependency "faraday"
   spec.add_runtime_dependency "dalli-elasticache"
   spec.add_runtime_dependency "alephant-support"
-  spec.add_runtime_dependency "alephant-sequencer", "~> 3"
+  spec.add_runtime_dependency "alephant-sequencer"
   spec.add_runtime_dependency "alephant-storage"
-  spec.add_runtime_dependency "alephant-logger", "~> 3.1"
-  spec.add_runtime_dependency "alephant-lookup", "~> 2"
+  spec.add_runtime_dependency "alephant-logger"
+  spec.add_runtime_dependency "alephant-lookup"
   spec.add_runtime_dependency "alephant-renderer"
 end

data/lib/alephant/publisher/env.rb

@@ -1,12 +1,10 @@
-require "aws-sdk"
+require "aws-sdk-sqs"
 require "yaml"
 require "alephant/logger"
 
 config_file = "config/aws.yaml"
 
-AWS.eager_autoload!
-
 if File.exist? config_file
   config = YAML.load(File.read(config_file))
-  AWS.config(config)
+  Aws.config.update(config)
 end

data/lib/alephant/publisher/queue/options.rb

@@ -1,4 +1,4 @@
-require "aws-sdk"
+require "aws-sdk-sqs"
 require "alephant/logger"
 
 module Alephant

data/lib/alephant/publisher/queue/processor.rb

@@ -10,10 +10,12 @@ module Alephant
           @opts = opts
         end
 
-        def consume(msg)
-          return if msg.nil?
-          write(msg)
-          msg.delete
+        def consume(message_collection)
+          return unless message_collection && message_collection.size > 0
+
+          message = message_collection.first
+          write(message)
+          message.delete
         end
 
         private

data/lib/alephant/publisher/queue/publisher.rb

@@ -54,34 +54,36 @@ module Alephant
         end
 
         def get_region
-          opts.queue[:sqs_account_region] || AWS.config.region
+          # @TODO: Where does region come from?
+          opts.queue[:sqs_account_region] || Aws.config[:region] || 'eu-west-1'
         end
 
         def sqs_client
-          @sqs_client ||= AWS::SQS.new(region: get_region)
+          @sqs_client ||= Aws::SQS::Client.new(sqs_queue_options)
         end
 
         def sqs_queue_options
-          (opts.queue[:aws_account_id].nil? ? {} : fallback).tap do |ops|
-            logger.info(
-              "event"   => "SQSQueueOptionsConfigured",
-              "options" => ops,
-              "method"  => "#{self.class}#sqs_queue_options"
-            )
-          end
-        end
+          options = {}
+          options[:endpoint] = ENV['AWS_SQS_ENDPOINT'] if ENV['AWS_SQS_ENDPOINT']
+          options[:region]   = get_region
 
-        def fallback
-          {
-            :queue_owner_aws_account_id => opts.queue[:aws_account_id]
-          }
+          logger.info(
+            "event"   => "SQSQueueOptionsConfigured",
+            "options" => options,
+            "method"  => "#{self.class}#sqs_queue_options"
+          )
+
+          options
         end
 
         def aws_queue
-          queue_url = sqs_client.queues.url_for(
-            opts.queue[:sqs_queue_name], sqs_queue_options
-          )
-          sqs_client.queues[queue_url]
+          options = { queue_name: opts.queue[:sqs_queue_name] }
+          options[:queue_owner_aws_account_id] = opts.queue[:aws_account_id] if opts.queue[:aws_account_id]
+
+          queue_url = sqs_client.get_queue_url(options).queue_url
+
+          resource = Aws::SQS::Resource.new(client: sqs_client)
+          resource.queue(queue_url)
         end
       end
     end

data/lib/alephant/publisher/queue/revalidate_processor.rb

@@ -1,10 +1,11 @@
 require 'faraday'
-require 'aws-sdk'
+require "aws-sdk-sqs"
 require 'crimp'
 require 'alephant/publisher/queue/processor'
 require 'alephant/publisher/queue/revalidate_writer'
 require 'json'
 require 'alephant/logger'
+require 'ostruct'
 
 module Alephant
   module Publisher
@@ -20,16 +21,19 @@ module Alephant
           @http_response_processor = http_response_processor
         end
 
-        def consume(message)
-          return if message.nil?
+        def consume(message_collection)
+          return unless message_collection && message_collection.size > 0
+
+          message = message_collection.first
 
           msg_body = message_content(message)
 
+          # @TODO: This is not a http response but a data struct. We should look at renaming this.
           http_response = {
             renderer_id:   msg_body.fetch(:id),
             http_options:  msg_body,
             http_response: get(message),
-            ttl:           http_response_processor.ttl(msg_body)
+            ttl:           http_response_processor.ttl(msg_body).to_s # @TODO: What happens if this is nil? Storage requires this to be a string.
           }
 
           http_message = build_http_message(message, ::JSON.generate(http_response))
@@ -76,11 +80,9 @@ module Alephant
         end
 
         def build_http_message(message, http_response)
-          # I feel dirty...
-          # FIXME: refactor `Writer` so it's not so tightly coupled to a AWS::SQS::ReceivedMessage object
-          http_message = message.dup
-          http_message.instance_variable_set(:@body, http_response)
-          http_message
+          OpenStruct.new({
+            body: http_response
+          })
         end
 
         def get(message)

data/lib/alephant/publisher/queue/revalidate_writer.rb

@@ -72,11 +72,13 @@ module Alephant
         end
 
         def storage_opts
-          { ttl: message_content[:ttl] }
+          { ttl: message_content[:ttl] } # @TODO: What happens if this is nil?
         end
 
         def config
-          @config.merge(renderer_id: message_content.fetch(:renderer_id))
+          # @TODO: This part needs refactoring
+          @config[:renderer_id] = message_content.fetch(:renderer_id)
+          @config
         end
 
         def http_data

data/lib/alephant/publisher/queue/sqs_helper/queue.rb

@@ -1,4 +1,4 @@
-require "aws-sdk"
+require "aws-sdk-sqs"
 require "alephant/logger"
 
 module Alephant
@@ -27,7 +27,7 @@ module Alephant
           end
 
           def message
-            receive.tap { |m| process(m) unless m.nil? }
+            receive.tap { |m| process(m) }
           end
 
           private
@@ -43,13 +43,16 @@ module Alephant
           end
 
           def process(m)
+            return unless m.size > 0
+
             logger.metric "MessagesReceived"
             logger.info(
               "event"     => "QueueMessageReceived",
-              "messageId" => m.id,
+              "messageId" => m.first.message_id,
               "method"    => "#{self.class}#process"
             )
-            archive m
+            # @TODO: Look at archiver as should support message from collection.
+            archive m.first
           end
 
           def archive(m)
@@ -66,9 +69,10 @@ module Alephant
           end
 
           def receive
-            queue.receive_message(
-              :visibility_timeout => timeout,
-              :wait_time_seconds  => wait_time
+            queue.receive_messages(
+              :visibility_timeout     => timeout,
+              :wait_time_seconds      => wait_time,
+              :max_number_of_messages => 1
             )
           end
         end

data/lib/alephant/publisher/queue/version.rb

@@ -1,7 +1,7 @@
 module Alephant
   module Publisher
     module Queue
-      VERSION = '2.5.0'.freeze
+      VERSION = '2.6.0'.freeze
     end
   end
 end

data/spec/alephant/publisher/queue/processor_spec.rb

@@ -11,11 +11,12 @@ describe Alephant::Publisher::Queue::Processor do
     ).to receive(:run!)
   end
 
-  describe "#consume(msg)" do
+  describe "#consume" do
     it "Consume the message and deletes it" do
-      msg = instance_double(AWS::SQS::ReceivedMessage, :delete => nil)
-      expect(msg).to receive(:delete)
-      subject.consume(msg)
+      message = instance_double(Aws::SQS::Message, :delete => nil)
+      message_collection = [message]
+      expect(message).to receive(:delete)
+      subject.consume(message_collection)
     end
   end
 end

data/spec/alephant/publisher/queue/revalidate_processor_spec.rb

@@ -32,7 +32,8 @@ RSpec.describe Alephant::Publisher::Queue::RevalidateProcessor do
   let(:cache_double)       { instance_double(Dalli::Client, delete: nil) }
   let(:elasticache_double) { instance_double(Dalli::ElastiCache, client: cache_double) }
 
-  let(:message)      { instance_double(AWS::SQS::ReceivedMessage, body: JSON.generate(message_body), delete: nil) }
+  let(:message)      { instance_double(Aws::SQS::Message, body: JSON.generate(message_body), delete: nil) }
+  let(:message_collection) { [message] }
   let(:message_body) { { id: '', batch_id: '', options: {} } }
 
   before do
@@ -64,7 +65,7 @@ RSpec.describe Alephant::Publisher::Queue::RevalidateProcessor do
 
           expect(writer_double).to receive(:run!)
 
-          subject.consume(message)
+          subject.consume(message_collection)
         end
 
         it 'passes the response to the http_response_processor' do
@@ -73,7 +74,7 @@ RSpec.describe Alephant::Publisher::Queue::RevalidateProcessor do
             .with(message_body, resp_status, resp_body)
             .and_call_original
 
-          subject.consume(message)
+          subject.consume(message_collection)
         end
 
         it "calls the 'ttl' method on the http_response_processor" do
@@ -82,19 +83,19 @@ RSpec.describe Alephant::Publisher::Queue::RevalidateProcessor do
             .with(message_body)
             .and_call_original
 
-          subject.consume(message)
+          subject.consume(message_collection)
         end
 
         it 'deletes the message from the queue' do
           expect(message).to receive(:delete)
 
-          subject.consume(message)
+          subject.consume(message_collection)
         end
 
         it "removes the 'inflight' cache message" do
           expect(cache_double).to receive(:delete)
 
-          subject.consume(message)
+          subject.consume(message_collection)
         end
       end
 
@@ -106,34 +107,34 @@ RSpec.describe Alephant::Publisher::Queue::RevalidateProcessor do
         it 'does not call #run! on the writer' do
           expect(writer_double).to_not receive(:run!)
 
-          expect { subject.consume(message) }
+          expect { subject.consume(message_collection) }
             .to raise_error(Faraday::TimeoutError)
         end
 
         it 'does NOT delele the message from the queue' do
           expect(message).to_not receive(:delete)
 
-          expect { subject.consume(message) }
+          expect { subject.consume(message_collection) }
             .to raise_error(Faraday::TimeoutError)
         end
 
         it "does not remove the 'inflight' cache message" do
           expect(cache_double).to_not receive(:delete)
 
-          expect { subject.consume(message) }
+          expect { subject.consume(message_collection) }
             .to raise_error(Faraday::TimeoutError)
         end
       end
     end
 
     context 'when there is no message passed through' do
-      let(:message) { nil }
+      let(:message_collection) { [] }
 
       it 'does nothing' do
         expect(writer_double).to_not receive(:run!)
         expect(cache_double).to_not receive(:delete)
 
-        subject.consume(message)
+        subject.consume(message_collection)
       end
     end
   end

data/spec/alephant/publisher/queue/sqs_helper/archiver_spec.rb

@@ -2,12 +2,12 @@ require "spec_helper"
 
 describe Alephant::Publisher::Queue::SQSHelper::Archiver do
   let(:storage) { instance_double(Alephant::Storage, :put => nil) }
-  let(:queue)    { instance_double(AWS::SQS::Queue, :url => nil) }
+  let(:queue)    { instance_double(Aws::SQS::Queue, :url => nil) }
   let(:msg_body) { { :Message => JSON.generate(msg_uri) } }
   let(:msg_uri)  { { :uri => "/content/asset/newsbeat" } }
   let(:message) do
     instance_double(
-      "AWS::SQS::ReceivedMessage",
+      "Aws::SQS::ReceivedMessage",
       :id    => "id",
       :md5   => "qux",
       :queue => queue,

data/spec/alephant/publisher/queue/sqs_helper/queue_spec.rb

@@ -6,7 +6,7 @@ describe Alephant::Publisher::Queue::SQSHelper::Queue do
       m = double("message").as_null_object
       q = double("queue").as_null_object
 
-      expect(q).to receive(:receive_message).and_return(m)
+      expect(q).to receive(:receive_messages).and_return(m)
 
       instance = Alephant::Publisher::Queue::SQSHelper::Queue.new(q)
 
@@ -18,7 +18,7 @@ describe Alephant::Publisher::Queue::SQSHelper::Queue do
       m = double("message").as_null_object
       q = double("queue").as_null_object
 
-      expect(q).to receive(:receive_message).and_return(m)
+      expect(q).to receive(:receive_messages).and_return(m)
       expect(a).to receive(:see).with(m)
 
       instance = Alephant::Publisher::Queue::SQSHelper::Queue.new(q, a)

data/spec/alephant/publisher/queue/writer_spec.rb

@@ -15,8 +15,6 @@ describe Alephant::Publisher::Queue::Writer do
   end
 
   before(:each) do
-    AWS.stub!
-
     allow_any_instance_of(Alephant::Storage).to receive(:initialize)
       .with(
         opts[:s3_bucket_id],

data/spec/alephant/publisher/queue_spec.rb

@@ -1,19 +1,20 @@
 require "spec_helper"
 
 describe Alephant::Publisher::Queue do
-  let(:options)       { Alephant::Publisher::Queue::Options.new }
-  let(:queue)         { double(AWS::SQS::Queue, :url => nil) }
-  let(:client_double) { double(AWS::SQS, :queues => queue_double) }
-  let(:queue_double) do
-    double("AWS::SQS::QueueCollection", :[] => queue, :url_for => nil)
-  end
+  let(:options)     { Alephant::Publisher::Queue::Options.new }
+  let(:fake_client) { Aws::SQS::Client.new(stub_responses: true) }
 
   before(:each) do
-    expect(AWS::SQS).to receive(:new).and_return(client_double)
+    allow_any_instance_of(Alephant::Publisher::Queue::Publisher).to receive(:sqs_client).and_return(fake_client)
+    fake_client.stub_responses(:get_queue_url, { queue_url: 'http://sqs.aws.myqueue/id' })
   end
 
   describe ".create" do
     it "sets parser, sequencer, queue and writer" do
+      options.add_queue(
+        :sqs_queue_name => "bar",
+        :aws_account_id => "foo"
+      )
       instance = Alephant::Publisher::Queue.create(options)
       expect(instance.queue)
         .to be_a Alephant::Publisher::Queue::SQSHelper::Queue
@@ -27,12 +28,9 @@ describe Alephant::Publisher::Queue do
           :aws_account_id => "foo"
         )
 
-        expect(queue_double).to receive(:url_for).with(
-          "bar",
-          :queue_owner_aws_account_id => "foo"
-        )
+        publisher = Alephant::Publisher::Queue.create(options)
 
-        Alephant::Publisher::Queue.create(options)
+        expect(publisher.queue.queue).to be_a(Aws::SQS::Queue)
       end
     end
 
@@ -41,9 +39,9 @@ describe Alephant::Publisher::Queue do
         options = Alephant::Publisher::Queue::Options.new
         options.add_queue(:sqs_queue_name => "bar")
 
-        expect(queue_double).to receive(:url_for).with("bar", {})
+        publisher = Alephant::Publisher::Queue.create(options)
 
-        Alephant::Publisher::Queue.create(options)
+        expect(publisher.queue.queue).to be_a(Aws::SQS::Queue)
       end
     end
   end

data/spec/spec_helper.rb

@@ -1,9 +1,11 @@
 $LOAD_PATH << File.join(File.dirname(__FILE__), "..", "lib")
 
 require "pry"
-require "aws-sdk"
+require "aws-sdk-sqs"
 require "simplecov"
 
+ENV['AWS_REGION'] = 'eu-west-1'
+
 SimpleCov.start do
   add_filter "/spec/"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: alephant-publisher-queue
 version: !ruby/object:Gem::Version
-  version: 2.5.0
+  version: 2.6.0
 platform: ruby
 authors:
 - BBC News
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-04-24 00:00:00.000000000 Z
+date: 2018-05-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -179,19 +179,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: aws-sdk
+  name: aws-sdk-sqs
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: crimp
   requirement: !ruby/object:Gem::Requirement
@@ -252,16 +252,16 @@ dependencies:
   name: alephant-sequencer
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '3'
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '3'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: alephant-storage
   requirement: !ruby/object:Gem::Requirement
@@ -280,30 +280,30 @@ dependencies:
   name: alephant-logger
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '3.1'
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '3.1'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: alephant-lookup
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '2'
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '2'
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: alephant-renderer
   requirement: !ruby/object:Gem::Requirement