alephant-publisher-queue 2.3.1 → 2.4.0

data/lib/alephant/publisher/queue/revalidate_processor.rb

@@ -0,0 +1,115 @@
+require 'faraday'
+require 'aws-sdk'
+require 'crimp'
+require 'alephant/publisher/queue/processor'
+require 'alephant/publisher/queue/revalidate_writer'
+require 'json'
+require 'alephant/logger'
+
+module Alephant
+  module Publisher
+    module Queue
+      class RevalidateProcessor < Processor
+        include Alephant::Logger
+
+        attr_reader :opts, :url_generator, :http_response_processor
+
+        def initialize(opts = nil, url_generator, http_response_processor)
+          @opts                    = opts
+          @url_generator           = url_generator
+          @http_response_processor = http_response_processor
+        end
+
+        def consume(message)
+          return if message.nil?
+
+          msg_body = message_content(message)
+
+          http_response = {
+            renderer_id:   msg_body.fetch(:id),
+            http_options:  msg_body,
+            http_response: get(message),
+            ttl:           http_response_processor.ttl(msg_body)
+          }
+
+          http_message = build_http_message(message, ::JSON.generate(http_response))
+
+          write(http_message)
+
+          message.delete
+          logger.info(event: 'SQSMessageDeleted', message_content: message_content(message), method: "#{self.class}#consume")
+
+          cache.delete(inflight_message_key(message))
+          logger.info(event: 'InFlightMessageDeleted', key: inflight_message_key(message), method: "#{self.class}#consume")
+        end
+
+        private
+
+        def write(message)
+          RevalidateWriter.new(writer_config, message).run!
+        end
+
+        # NOTE: If you change this, you'll need to change this in
+        #       `alephant-broker` also.
+        def inflight_message_key(message)
+          opts = ::JSON.parse(message.body)
+          version_cache_key(
+            "inflight-#{opts['id']}/#{build_inflight_opts_hash(opts)}"
+          )
+        end
+
+        def build_inflight_opts_hash(opts)
+          opts_hash = Hash[opts['options'].map { |k, v| [k.to_sym, v] }]
+          Crimp.signature(opts_hash)
+        end
+
+        def version_cache_key(key)
+          cache_version = opts.cache[:elasticache_cache_version]
+          [key, cache_version].compact.join('_')
+        end
+
+        def cache
+          @cache ||= proc do
+            endpoint = opts.cache.fetch(:elasticache_config_endpoint)
+            Dalli::ElastiCache.new(endpoint).client
+          end.call
+        end
+
+        def build_http_message(message, http_response)
+          # I feel dirty...
+          # FIXME: refactor `Writer` so it's not so tightly coupled to a AWS::SQS::ReceivedMessage object
+          http_message = message.dup
+          http_message.instance_variable_set(:@body, http_response)
+          http_message
+        end
+
+        def get(message)
+          msg_content = message_content(message)
+          url         = url_generator.generate(msg_content)
+
+          logger.info(
+            event:  'Sending HTTP GET request',
+            url:    url,
+            method: "#{self.class}#get"
+          )
+
+          res = Faraday.get(url)
+
+          logger.info(
+            event:  'HTTP request complete',
+            url:    url,
+            status: res.status,
+            body:   res.body,
+            method: "#{self.class}#get"
+          )
+
+          http_response_processor.process(msg_content, res.status, res.body)
+        end
+
+        def message_content(message)
+          ::JSON.parse(message.body, symbolize_names: true)
+        end
+      end
+    end
+  end
+end

data/lib/alephant/publisher/queue/revalidate_writer.rb

@@ -0,0 +1,96 @@
+module Alephant
+  module Publisher
+    module Queue
+      class RevalidateWriter
+        include Alephant::Logger
+
+        attr_reader :message
+
+        def initialize(config, message)
+          @config  = config
+          @message = message
+        end
+
+        def run!
+          renderer.views.each do |view_id, view|
+            store(view_id, view)
+            write_lookup_record(view_id)
+          end
+        end
+
+        def renderer
+          @renderer ||= Alephant::Renderer.create(config, http_data)
+        end
+
+        def storage
+          @storage ||= Alephant::Cache.new(config.fetch(:s3_bucket_id), config.fetch(:s3_object_path))
+        end
+
+        def lookup
+          @lookup ||= Alephant::Lookup.create(config.fetch(:lookup_table_name), config)
+        end
+
+        private
+
+        def write_lookup_record(view_id)
+          lookup.write(view_id, http_options.fetch(:options), seq_id, storage_location(view_id))
+
+          logger.info(event:       'LookupLocationUpdated',
+                      view_id:     view_id,
+                      options:     http_options.fetch(:options),
+                      seq_id:      seq_id,
+                      location:    storage_location(view_id),
+                      renderer_id: config.fetch(:renderer_id),
+                      method:      "#{self.class}#write_lookup_record")
+        end
+
+        def store(view_id, view)
+          storage.put(storage_location(view_id), view.render, view.content_type, storage_opts)
+
+          logger.info(event:        'MessageStored',
+                      location:     storage_location(view_id),
+                      view_id:      view_id,
+                      view:         view,
+                      content:      view.render,
+                      content_type: view.content_type,
+                      storage_opts: storage_opts,
+                      renderer_id:  config.fetch(:renderer_id),
+                      method:       "#{self.class}#store")
+        end
+
+        # NOTE: we _really_ don't care about sequence here - we just _have_ to pass something through
+        def seq_id
+          1
+        end
+
+        def storage_location(view_id)
+          [
+            config.fetch(:renderer_id),
+            view_id,
+            Crimp.signature(http_options)
+          ].join('/')
+        end
+
+        def storage_opts
+          { ttl: message_content[:ttl] }
+        end
+
+        def config
+          @config.merge(renderer_id: message_content.fetch(:renderer_id))
+        end
+
+        def http_data
+          @http_data ||= ::JSON.parse(message_content.fetch(:http_response), symbolize_names: true)
+        end
+
+        def http_options
+          @http_options ||= message_content.fetch(:http_options)
+        end
+
+        def message_content
+          @message_content ||= ::JSON.parse(message.body, symbolize_names: true)
+        end
+      end
+    end
+  end
+end

data/lib/alephant/publisher/queue/version.rb

@@ -1,7 +1,7 @@
 module Alephant
   module Publisher
     module Queue
-      VERSION = "2.3.1"
+      VERSION = '2.4.0'.freeze
     end
   end
 end

data/lib/alephant/publisher/queue/writer.rb

@@ -13,12 +13,15 @@ module Alephant
       class Writer
         include Alephant::Logger
 
-        attr_reader :config, :message, :cache, :parser, :renderer
+        attr_reader :config, :message, :cache, :parser
 
         def initialize(config, message)
           @config   = config
           @message  = message
-          @renderer = Alephant::Renderer.create(config, data)
+        end
+
+        def renderer
+          @renderer ||= Alephant::Renderer.create(config, data)
         end
 
         def cache
@@ -65,7 +68,24 @@ module Alephant
         end
 
         def store(component, view, location, storage_opts = {})
+          logger.info(
+            event:        'StoreBeforeRender',
+            component:    component,
+            view:         view,
+            location:     location,
+            storage_opts: storage_opts
+          )
+
           render = view.render
+
+          logger.info(
+            event:        'StoreAfterRender',
+            component:    component,
+            view:         view,
+            location:     location,
+            storage_opts: storage_opts
+          )
+
           cache.put(location, render, view.content_type, storage_opts).tap do
             logger.info(
               "event"          => "MessageStored",
@@ -78,6 +98,7 @@ module Alephant
               "method"         => "#{self.class}#store"
             )
           end
+
           lookup.write(component, options, seq_id, location).tap do
             logger.info(
               "event"      => "LookupLocationUpdated",

data/spec/alephant/publisher/queue/revalidate_processor_spec.rb

@@ -0,0 +1,140 @@
+require 'spec_helper'
+
+RSpec.describe Alephant::Publisher::Queue::RevalidateProcessor do
+  class TestUrlGenerator
+    def self.generate(_opts = {})
+      'http://example.com'
+    end
+  end
+
+  class TestHttpResponseProcessor
+    def self.process(_opts, _response_status, response_body)
+      response_body
+    end
+
+    def self.ttl(_opts)
+      nil
+    end
+  end
+
+  subject { described_class.new(opts, url_generator, http_response_processor) }
+
+  let(:url_generator)           { TestUrlGenerator }
+  let(:http_response_processor) { TestHttpResponseProcessor }
+
+  let(:opts) do
+    instance_double(Alephant::Publisher::Queue::Options,
+      writer: {},
+      cache:  { elasticache_config_endpoint: 'wibble' })
+  end
+
+  let(:writer_double)      { instance_double(Alephant::Publisher::Queue::RevalidateWriter, run!: nil) }
+  let(:cache_double)       { instance_double(Dalli::Client, delete: nil) }
+  let(:elasticache_double) { instance_double(Dalli::ElastiCache, client: cache_double) }
+
+  let(:message)      { instance_double(AWS::SQS::ReceivedMessage, body: JSON.generate(message_body), delete: nil) }
+  let(:message_body) { { id: '', batch_id: '', options: {} } }
+
+  before do
+    allow(Alephant::Publisher::Queue::RevalidateWriter)
+      .to receive(:new)
+      .and_return(writer_double)
+
+    allow(Dalli::ElastiCache)
+      .to receive(:new)
+      .and_return(elasticache_double)
+  end
+
+  describe '#consume' do
+    context 'when there is a message passed through' do
+      context 'when the HTTP request is successful' do
+        let(:resp_double) { double(body: resp_body, status: resp_status) }
+        let(:resp_body)   { JSON.generate(id: 'foo') }
+        let(:resp_status) { 200 }
+
+        before do
+          allow(Faraday).to receive(:get).and_return(resp_double)
+        end
+
+        it 'calls #run! on the writer with the http request result' do
+          expect(Alephant::Publisher::Queue::RevalidateWriter)
+            .to receive(:new)
+            .with(opts.writer, anything)
+            .and_return(writer_double)
+
+          expect(writer_double).to receive(:run!)
+
+          subject.consume(message)
+        end
+
+        it 'passes the response to the http_response_processor' do
+          expect(http_response_processor)
+            .to receive(:process)
+            .with(message_body, resp_status, resp_body)
+            .and_call_original
+
+          subject.consume(message)
+        end
+
+        it "calls the 'ttl' method on the http_response_processor" do
+          expect(http_response_processor)
+            .to receive(:ttl)
+            .with(message_body)
+            .and_call_original
+
+          subject.consume(message)
+        end
+
+        it 'deletes the message from the queue' do
+          expect(message).to receive(:delete)
+
+          subject.consume(message)
+        end
+
+        it "removes the 'inflight' cache message" do
+          expect(cache_double).to receive(:delete)
+
+          subject.consume(message)
+        end
+      end
+
+      context 'when the HTTP request is unsuccessful' do
+        before do
+          allow(Faraday).to receive(:get).and_raise(Faraday::TimeoutError)
+        end
+
+        it 'does not call #run! on the writer' do
+          expect(writer_double).to_not receive(:run!)
+
+          expect { subject.consume(message) }
+            .to raise_error(Faraday::TimeoutError)
+        end
+
+        it 'does NOT delele the message from the queue' do
+          expect(message).to_not receive(:delete)
+
+          expect { subject.consume(message) }
+            .to raise_error(Faraday::TimeoutError)
+        end
+
+        it "does not remove the 'inflight' cache message" do
+          expect(cache_double).to_not receive(:delete)
+
+          expect { subject.consume(message) }
+            .to raise_error(Faraday::TimeoutError)
+        end
+      end
+    end
+
+    context 'when there is no message passed through' do
+      let(:message) { nil }
+
+      it 'does nothing' do
+        expect(writer_double).to_not receive(:run!)
+        expect(cache_double).to_not receive(:delete)
+
+        subject.consume(message)
+      end
+    end
+  end
+end

data/spec/alephant/publisher/queue/revalidate_writer_spec.rb

@@ -0,0 +1,96 @@
+require 'spec_helper'
+
+RSpec.describe Alephant::Publisher::Queue::RevalidateWriter do
+  subject { described_class.new(config, message) }
+
+  let(:config) do
+    {
+      s3_bucket_id:      'qwerty',
+      s3_object_path:    'hello/int',
+      lookup_table_name: 'lookup_table'
+    }
+  end
+
+  let(:message)        { double(body: JSON.generate(message_body)) }
+  let(:http_response)  { { ticker: 'WMT', val: 180.00 } }
+  let(:ttl)            { 45 }
+
+  let(:message_body) do
+    {
+      renderer_id:   'hello_world',
+      http_options:  { id: 'hello_world', options: { ticker: 'WMT', duration: '1_day' } },
+      http_response: JSON.generate(http_response),
+      ttl:           ttl
+    }
+  end
+
+  describe '#run!' do
+    let(:storage_double)  { instance_double(Alephant::Cache, put: nil) }
+    let(:lookup_double)   { instance_double(Alephant::Lookup::LookupHelper, write: nil) }
+    let(:renderer_double) do
+      instance_double(Alephant::Renderer::Renderer, views: { hello_world_view: hello_world_view })
+    end
+
+    let(:hello_world_view)      { double(render: rendered_content, content_type: rendered_content_type) }
+    let(:rendered_content)      { '<h1>Hello, world!</h1>' }
+    let(:rendered_content_type) { 'text/html' }
+
+    let(:storage_location) { "hello_world/hello_world_view/#{Crimp.signature(message_body[:http_options])}" }
+
+    before do
+      allow(Alephant::Renderer).to receive(:create).and_return(renderer_double)
+      allow(Alephant::Cache).to receive(:new).and_return(storage_double)
+      allow(Alephant::Lookup).to receive(:create).and_return(lookup_double)
+    end
+
+    it 'renders the http_response in the Renderer' do
+      expect(renderer_double).to receive(:views).and_return(foo: hello_world_view)
+      expect(hello_world_view).to receive(:render)
+      expect(hello_world_view).to receive(:content_type)
+
+      subject.run!
+    end
+
+    it 'stores the HTTP content in S3 with Alephant::Cache' do
+      expect(storage_double)
+        .to receive(:put)
+        .with(storage_location, rendered_content, rendered_content_type, ttl: ttl)
+
+      subject.run!
+    end
+
+    it 'writes the S3 location with Alephant::Lookup' do
+      expect(lookup_double)
+        .to receive(:write)
+        .with(:hello_world_view, message_body[:http_options][:options], 1, storage_location)
+
+      subject.run!
+    end
+  end
+
+  describe '#renderer' do
+    it 'builds an Alephant::Renderer::Renderer object' do
+      expect(subject.renderer).to be_a(Alephant::Renderer::Renderer)
+    end
+
+    it 'passes through the `renderer_id` from the message' do
+      expect(Alephant::Renderer)
+        .to receive(:create)
+        .with(hash_including(renderer_id: 'hello_world'), http_response)
+
+      subject.renderer
+    end
+  end
+
+  describe '#storage' do
+    it 'builds an Alephant::Cache object' do
+      expect(subject.storage).to be_a(Alephant::Cache)
+    end
+  end
+
+  describe '#lookup' do
+    it 'builds an Alephant::Lookup object' do
+      expect(subject.lookup).to be_a(Alephant::Lookup::LookupHelper)
+    end
+  end
+end