alephant-publisher-queue 2.3.1 → 2.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,115 @@
1
+ require 'faraday'
2
+ require 'aws-sdk'
3
+ require 'crimp'
4
+ require 'alephant/publisher/queue/processor'
5
+ require 'alephant/publisher/queue/revalidate_writer'
6
+ require 'json'
7
+ require 'alephant/logger'
8
+
9
+ module Alephant
10
+ module Publisher
11
+ module Queue
12
+ class RevalidateProcessor < Processor
13
+ include Alephant::Logger
14
+
15
+ attr_reader :opts, :url_generator, :http_response_processor
16
+
17
+ def initialize(opts = nil, url_generator, http_response_processor)
18
+ @opts = opts
19
+ @url_generator = url_generator
20
+ @http_response_processor = http_response_processor
21
+ end
22
+
23
+ def consume(message)
24
+ return if message.nil?
25
+
26
+ msg_body = message_content(message)
27
+
28
+ http_response = {
29
+ renderer_id: msg_body.fetch(:id),
30
+ http_options: msg_body,
31
+ http_response: get(message),
32
+ ttl: http_response_processor.ttl(msg_body)
33
+ }
34
+
35
+ http_message = build_http_message(message, ::JSON.generate(http_response))
36
+
37
+ write(http_message)
38
+
39
+ message.delete
40
+ logger.info(event: 'SQSMessageDeleted', message_content: message_content(message), method: "#{self.class}#consume")
41
+
42
+ cache.delete(inflight_message_key(message))
43
+ logger.info(event: 'InFlightMessageDeleted', key: inflight_message_key(message), method: "#{self.class}#consume")
44
+ end
45
+
46
+ private
47
+
48
+ def write(message)
49
+ RevalidateWriter.new(writer_config, message).run!
50
+ end
51
+
52
+ # NOTE: If you change this, you'll need to change this in
53
+ # `alephant-broker` also.
54
+ def inflight_message_key(message)
55
+ opts = ::JSON.parse(message.body)
56
+ version_cache_key(
57
+ "inflight-#{opts['id']}/#{build_inflight_opts_hash(opts)}"
58
+ )
59
+ end
60
+
61
+ def build_inflight_opts_hash(opts)
62
+ opts_hash = Hash[opts['options'].map { |k, v| [k.to_sym, v] }]
63
+ Crimp.signature(opts_hash)
64
+ end
65
+
66
+ def version_cache_key(key)
67
+ cache_version = opts.cache[:elasticache_cache_version]
68
+ [key, cache_version].compact.join('_')
69
+ end
70
+
71
+ def cache
72
+ @cache ||= proc do
73
+ endpoint = opts.cache.fetch(:elasticache_config_endpoint)
74
+ Dalli::ElastiCache.new(endpoint).client
75
+ end.call
76
+ end
77
+
78
+ def build_http_message(message, http_response)
79
+ # I feel dirty...
80
+ # FIXME: refactor `Writer` so it's not so tightly coupled to a AWS::SQS::ReceivedMessage object
81
+ http_message = message.dup
82
+ http_message.instance_variable_set(:@body, http_response)
83
+ http_message
84
+ end
85
+
86
+ def get(message)
87
+ msg_content = message_content(message)
88
+ url = url_generator.generate(msg_content)
89
+
90
+ logger.info(
91
+ event: 'Sending HTTP GET request',
92
+ url: url,
93
+ method: "#{self.class}#get"
94
+ )
95
+
96
+ res = Faraday.get(url)
97
+
98
+ logger.info(
99
+ event: 'HTTP request complete',
100
+ url: url,
101
+ status: res.status,
102
+ body: res.body,
103
+ method: "#{self.class}#get"
104
+ )
105
+
106
+ http_response_processor.process(msg_content, res.status, res.body)
107
+ end
108
+
109
+ def message_content(message)
110
+ ::JSON.parse(message.body, symbolize_names: true)
111
+ end
112
+ end
113
+ end
114
+ end
115
+ end
@@ -0,0 +1,96 @@
1
+ module Alephant
2
+ module Publisher
3
+ module Queue
4
+ class RevalidateWriter
5
+ include Alephant::Logger
6
+
7
+ attr_reader :message
8
+
9
+ def initialize(config, message)
10
+ @config = config
11
+ @message = message
12
+ end
13
+
14
+ def run!
15
+ renderer.views.each do |view_id, view|
16
+ store(view_id, view)
17
+ write_lookup_record(view_id)
18
+ end
19
+ end
20
+
21
+ def renderer
22
+ @renderer ||= Alephant::Renderer.create(config, http_data)
23
+ end
24
+
25
+ def storage
26
+ @storage ||= Alephant::Cache.new(config.fetch(:s3_bucket_id), config.fetch(:s3_object_path))
27
+ end
28
+
29
+ def lookup
30
+ @lookup ||= Alephant::Lookup.create(config.fetch(:lookup_table_name), config)
31
+ end
32
+
33
+ private
34
+
35
+ def write_lookup_record(view_id)
36
+ lookup.write(view_id, http_options.fetch(:options), seq_id, storage_location(view_id))
37
+
38
+ logger.info(event: 'LookupLocationUpdated',
39
+ view_id: view_id,
40
+ options: http_options.fetch(:options),
41
+ seq_id: seq_id,
42
+ location: storage_location(view_id),
43
+ renderer_id: config.fetch(:renderer_id),
44
+ method: "#{self.class}#write_lookup_record")
45
+ end
46
+
47
+ def store(view_id, view)
48
+ storage.put(storage_location(view_id), view.render, view.content_type, storage_opts)
49
+
50
+ logger.info(event: 'MessageStored',
51
+ location: storage_location(view_id),
52
+ view_id: view_id,
53
+ view: view,
54
+ content: view.render,
55
+ content_type: view.content_type,
56
+ storage_opts: storage_opts,
57
+ renderer_id: config.fetch(:renderer_id),
58
+ method: "#{self.class}#store")
59
+ end
60
+
61
+ # NOTE: we _really_ don't care about sequence here - we just _have_ to pass something through
62
+ def seq_id
63
+ 1
64
+ end
65
+
66
+ def storage_location(view_id)
67
+ [
68
+ config.fetch(:renderer_id),
69
+ view_id,
70
+ Crimp.signature(http_options)
71
+ ].join('/')
72
+ end
73
+
74
+ def storage_opts
75
+ { ttl: message_content[:ttl] }
76
+ end
77
+
78
+ def config
79
+ @config.merge(renderer_id: message_content.fetch(:renderer_id))
80
+ end
81
+
82
+ def http_data
83
+ @http_data ||= ::JSON.parse(message_content.fetch(:http_response), symbolize_names: true)
84
+ end
85
+
86
+ def http_options
87
+ @http_options ||= message_content.fetch(:http_options)
88
+ end
89
+
90
+ def message_content
91
+ @message_content ||= ::JSON.parse(message.body, symbolize_names: true)
92
+ end
93
+ end
94
+ end
95
+ end
96
+ end
@@ -1,7 +1,7 @@
1
1
  module Alephant
2
2
  module Publisher
3
3
  module Queue
4
- VERSION = "2.3.1"
4
+ VERSION = '2.4.0'.freeze
5
5
  end
6
6
  end
7
7
  end
@@ -13,12 +13,15 @@ module Alephant
13
13
  class Writer
14
14
  include Alephant::Logger
15
15
 
16
- attr_reader :config, :message, :cache, :parser, :renderer
16
+ attr_reader :config, :message, :cache, :parser
17
17
 
18
18
  def initialize(config, message)
19
19
  @config = config
20
20
  @message = message
21
- @renderer = Alephant::Renderer.create(config, data)
21
+ end
22
+
23
+ def renderer
24
+ @renderer ||= Alephant::Renderer.create(config, data)
22
25
  end
23
26
 
24
27
  def cache
@@ -65,7 +68,24 @@ module Alephant
65
68
  end
66
69
 
67
70
  def store(component, view, location, storage_opts = {})
71
+ logger.info(
72
+ event: 'StoreBeforeRender',
73
+ component: component,
74
+ view: view,
75
+ location: location,
76
+ storage_opts: storage_opts
77
+ )
78
+
68
79
  render = view.render
80
+
81
+ logger.info(
82
+ event: 'StoreAfterRender',
83
+ component: component,
84
+ view: view,
85
+ location: location,
86
+ storage_opts: storage_opts
87
+ )
88
+
69
89
  cache.put(location, render, view.content_type, storage_opts).tap do
70
90
  logger.info(
71
91
  "event" => "MessageStored",
@@ -78,6 +98,7 @@ module Alephant
78
98
  "method" => "#{self.class}#store"
79
99
  )
80
100
  end
101
+
81
102
  lookup.write(component, options, seq_id, location).tap do
82
103
  logger.info(
83
104
  "event" => "LookupLocationUpdated",
@@ -0,0 +1,140 @@
1
+ require 'spec_helper'
2
+
3
+ RSpec.describe Alephant::Publisher::Queue::RevalidateProcessor do
4
+ class TestUrlGenerator
5
+ def self.generate(_opts = {})
6
+ 'http://example.com'
7
+ end
8
+ end
9
+
10
+ class TestHttpResponseProcessor
11
+ def self.process(_opts, _response_status, response_body)
12
+ response_body
13
+ end
14
+
15
+ def self.ttl(_opts)
16
+ nil
17
+ end
18
+ end
19
+
20
+ subject { described_class.new(opts, url_generator, http_response_processor) }
21
+
22
+ let(:url_generator) { TestUrlGenerator }
23
+ let(:http_response_processor) { TestHttpResponseProcessor }
24
+
25
+ let(:opts) do
26
+ instance_double(Alephant::Publisher::Queue::Options,
27
+ writer: {},
28
+ cache: { elasticache_config_endpoint: 'wibble' })
29
+ end
30
+
31
+ let(:writer_double) { instance_double(Alephant::Publisher::Queue::RevalidateWriter, run!: nil) }
32
+ let(:cache_double) { instance_double(Dalli::Client, delete: nil) }
33
+ let(:elasticache_double) { instance_double(Dalli::ElastiCache, client: cache_double) }
34
+
35
+ let(:message) { instance_double(AWS::SQS::ReceivedMessage, body: JSON.generate(message_body), delete: nil) }
36
+ let(:message_body) { { id: '', batch_id: '', options: {} } }
37
+
38
+ before do
39
+ allow(Alephant::Publisher::Queue::RevalidateWriter)
40
+ .to receive(:new)
41
+ .and_return(writer_double)
42
+
43
+ allow(Dalli::ElastiCache)
44
+ .to receive(:new)
45
+ .and_return(elasticache_double)
46
+ end
47
+
48
+ describe '#consume' do
49
+ context 'when there is a message passed through' do
50
+ context 'when the HTTP request is successful' do
51
+ let(:resp_double) { double(body: resp_body, status: resp_status) }
52
+ let(:resp_body) { JSON.generate(id: 'foo') }
53
+ let(:resp_status) { 200 }
54
+
55
+ before do
56
+ allow(Faraday).to receive(:get).and_return(resp_double)
57
+ end
58
+
59
+ it 'calls #run! on the writer with the http request result' do
60
+ expect(Alephant::Publisher::Queue::RevalidateWriter)
61
+ .to receive(:new)
62
+ .with(opts.writer, anything)
63
+ .and_return(writer_double)
64
+
65
+ expect(writer_double).to receive(:run!)
66
+
67
+ subject.consume(message)
68
+ end
69
+
70
+ it 'passes the response to the http_response_processor' do
71
+ expect(http_response_processor)
72
+ .to receive(:process)
73
+ .with(message_body, resp_status, resp_body)
74
+ .and_call_original
75
+
76
+ subject.consume(message)
77
+ end
78
+
79
+ it "calls the 'ttl' method on the http_response_processor" do
80
+ expect(http_response_processor)
81
+ .to receive(:ttl)
82
+ .with(message_body)
83
+ .and_call_original
84
+
85
+ subject.consume(message)
86
+ end
87
+
88
+ it 'deletes the message from the queue' do
89
+ expect(message).to receive(:delete)
90
+
91
+ subject.consume(message)
92
+ end
93
+
94
+ it "removes the 'inflight' cache message" do
95
+ expect(cache_double).to receive(:delete)
96
+
97
+ subject.consume(message)
98
+ end
99
+ end
100
+
101
+ context 'when the HTTP request is unsuccessful' do
102
+ before do
103
+ allow(Faraday).to receive(:get).and_raise(Faraday::TimeoutError)
104
+ end
105
+
106
+ it 'does not call #run! on the writer' do
107
+ expect(writer_double).to_not receive(:run!)
108
+
109
+ expect { subject.consume(message) }
110
+ .to raise_error(Faraday::TimeoutError)
111
+ end
112
+
113
+ it 'does NOT delele the message from the queue' do
114
+ expect(message).to_not receive(:delete)
115
+
116
+ expect { subject.consume(message) }
117
+ .to raise_error(Faraday::TimeoutError)
118
+ end
119
+
120
+ it "does not remove the 'inflight' cache message" do
121
+ expect(cache_double).to_not receive(:delete)
122
+
123
+ expect { subject.consume(message) }
124
+ .to raise_error(Faraday::TimeoutError)
125
+ end
126
+ end
127
+ end
128
+
129
+ context 'when there is no message passed through' do
130
+ let(:message) { nil }
131
+
132
+ it 'does nothing' do
133
+ expect(writer_double).to_not receive(:run!)
134
+ expect(cache_double).to_not receive(:delete)
135
+
136
+ subject.consume(message)
137
+ end
138
+ end
139
+ end
140
+ end
@@ -0,0 +1,96 @@
1
+ require 'spec_helper'
2
+
3
+ RSpec.describe Alephant::Publisher::Queue::RevalidateWriter do
4
+ subject { described_class.new(config, message) }
5
+
6
+ let(:config) do
7
+ {
8
+ s3_bucket_id: 'qwerty',
9
+ s3_object_path: 'hello/int',
10
+ lookup_table_name: 'lookup_table'
11
+ }
12
+ end
13
+
14
+ let(:message) { double(body: JSON.generate(message_body)) }
15
+ let(:http_response) { { ticker: 'WMT', val: 180.00 } }
16
+ let(:ttl) { 45 }
17
+
18
+ let(:message_body) do
19
+ {
20
+ renderer_id: 'hello_world',
21
+ http_options: { id: 'hello_world', options: { ticker: 'WMT', duration: '1_day' } },
22
+ http_response: JSON.generate(http_response),
23
+ ttl: ttl
24
+ }
25
+ end
26
+
27
+ describe '#run!' do
28
+ let(:storage_double) { instance_double(Alephant::Cache, put: nil) }
29
+ let(:lookup_double) { instance_double(Alephant::Lookup::LookupHelper, write: nil) }
30
+ let(:renderer_double) do
31
+ instance_double(Alephant::Renderer::Renderer, views: { hello_world_view: hello_world_view })
32
+ end
33
+
34
+ let(:hello_world_view) { double(render: rendered_content, content_type: rendered_content_type) }
35
+ let(:rendered_content) { '<h1>Hello, world!</h1>' }
36
+ let(:rendered_content_type) { 'text/html' }
37
+
38
+ let(:storage_location) { "hello_world/hello_world_view/#{Crimp.signature(message_body[:http_options])}" }
39
+
40
+ before do
41
+ allow(Alephant::Renderer).to receive(:create).and_return(renderer_double)
42
+ allow(Alephant::Cache).to receive(:new).and_return(storage_double)
43
+ allow(Alephant::Lookup).to receive(:create).and_return(lookup_double)
44
+ end
45
+
46
+ it 'renders the http_response in the Renderer' do
47
+ expect(renderer_double).to receive(:views).and_return(foo: hello_world_view)
48
+ expect(hello_world_view).to receive(:render)
49
+ expect(hello_world_view).to receive(:content_type)
50
+
51
+ subject.run!
52
+ end
53
+
54
+ it 'stores the HTTP content in S3 with Alephant::Cache' do
55
+ expect(storage_double)
56
+ .to receive(:put)
57
+ .with(storage_location, rendered_content, rendered_content_type, ttl: ttl)
58
+
59
+ subject.run!
60
+ end
61
+
62
+ it 'writes the S3 location with Alephant::Lookup' do
63
+ expect(lookup_double)
64
+ .to receive(:write)
65
+ .with(:hello_world_view, message_body[:http_options][:options], 1, storage_location)
66
+
67
+ subject.run!
68
+ end
69
+ end
70
+
71
+ describe '#renderer' do
72
+ it 'builds an Alephant::Renderer::Renderer object' do
73
+ expect(subject.renderer).to be_a(Alephant::Renderer::Renderer)
74
+ end
75
+
76
+ it 'passes through the `renderer_id` from the message' do
77
+ expect(Alephant::Renderer)
78
+ .to receive(:create)
79
+ .with(hash_including(renderer_id: 'hello_world'), http_response)
80
+
81
+ subject.renderer
82
+ end
83
+ end
84
+
85
+ describe '#storage' do
86
+ it 'builds an Alephant::Cache object' do
87
+ expect(subject.storage).to be_a(Alephant::Cache)
88
+ end
89
+ end
90
+
91
+ describe '#lookup' do
92
+ it 'builds an Alephant::Lookup object' do
93
+ expect(subject.lookup).to be_a(Alephant::Lookup::LookupHelper)
94
+ end
95
+ end
96
+ end