alephant-publisher-queue 2.3.1 → 2.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +22 -7
- data/README.md +109 -3
- data/alephant-publisher-queue.gemspec +2 -0
- data/lib/alephant/publisher/queue.rb +4 -86
- data/lib/alephant/publisher/queue/options.rb +28 -19
- data/lib/alephant/publisher/queue/processor.rb +11 -9
- data/lib/alephant/publisher/queue/publisher.rb +89 -0
- data/lib/alephant/publisher/queue/revalidate_processor.rb +115 -0
- data/lib/alephant/publisher/queue/revalidate_writer.rb +96 -0
- data/lib/alephant/publisher/queue/version.rb +1 -1
- data/lib/alephant/publisher/queue/writer.rb +23 -2
- data/spec/alephant/publisher/queue/revalidate_processor_spec.rb +140 -0
- data/spec/alephant/publisher/queue/revalidate_writer_spec.rb +96 -0
- metadata +94 -60
- data/lib/alephant/publisher/queue/base_processor.rb +0 -13
@@ -0,0 +1,115 @@
|
|
1
|
+
require 'faraday'
|
2
|
+
require 'aws-sdk'
|
3
|
+
require 'crimp'
|
4
|
+
require 'alephant/publisher/queue/processor'
|
5
|
+
require 'alephant/publisher/queue/revalidate_writer'
|
6
|
+
require 'json'
|
7
|
+
require 'alephant/logger'
|
8
|
+
|
9
|
+
module Alephant
|
10
|
+
module Publisher
|
11
|
+
module Queue
|
12
|
+
class RevalidateProcessor < Processor
|
13
|
+
include Alephant::Logger
|
14
|
+
|
15
|
+
attr_reader :opts, :url_generator, :http_response_processor
|
16
|
+
|
17
|
+
def initialize(opts = nil, url_generator, http_response_processor)
|
18
|
+
@opts = opts
|
19
|
+
@url_generator = url_generator
|
20
|
+
@http_response_processor = http_response_processor
|
21
|
+
end
|
22
|
+
|
23
|
+
def consume(message)
|
24
|
+
return if message.nil?
|
25
|
+
|
26
|
+
msg_body = message_content(message)
|
27
|
+
|
28
|
+
http_response = {
|
29
|
+
renderer_id: msg_body.fetch(:id),
|
30
|
+
http_options: msg_body,
|
31
|
+
http_response: get(message),
|
32
|
+
ttl: http_response_processor.ttl(msg_body)
|
33
|
+
}
|
34
|
+
|
35
|
+
http_message = build_http_message(message, ::JSON.generate(http_response))
|
36
|
+
|
37
|
+
write(http_message)
|
38
|
+
|
39
|
+
message.delete
|
40
|
+
logger.info(event: 'SQSMessageDeleted', message_content: message_content(message), method: "#{self.class}#consume")
|
41
|
+
|
42
|
+
cache.delete(inflight_message_key(message))
|
43
|
+
logger.info(event: 'InFlightMessageDeleted', key: inflight_message_key(message), method: "#{self.class}#consume")
|
44
|
+
end
|
45
|
+
|
46
|
+
private
|
47
|
+
|
48
|
+
def write(message)
|
49
|
+
RevalidateWriter.new(writer_config, message).run!
|
50
|
+
end
|
51
|
+
|
52
|
+
# NOTE: If you change this, you'll need to change this in
|
53
|
+
# `alephant-broker` also.
|
54
|
+
def inflight_message_key(message)
|
55
|
+
opts = ::JSON.parse(message.body)
|
56
|
+
version_cache_key(
|
57
|
+
"inflight-#{opts['id']}/#{build_inflight_opts_hash(opts)}"
|
58
|
+
)
|
59
|
+
end
|
60
|
+
|
61
|
+
def build_inflight_opts_hash(opts)
|
62
|
+
opts_hash = Hash[opts['options'].map { |k, v| [k.to_sym, v] }]
|
63
|
+
Crimp.signature(opts_hash)
|
64
|
+
end
|
65
|
+
|
66
|
+
def version_cache_key(key)
|
67
|
+
cache_version = opts.cache[:elasticache_cache_version]
|
68
|
+
[key, cache_version].compact.join('_')
|
69
|
+
end
|
70
|
+
|
71
|
+
def cache
|
72
|
+
@cache ||= proc do
|
73
|
+
endpoint = opts.cache.fetch(:elasticache_config_endpoint)
|
74
|
+
Dalli::ElastiCache.new(endpoint).client
|
75
|
+
end.call
|
76
|
+
end
|
77
|
+
|
78
|
+
def build_http_message(message, http_response)
|
79
|
+
# I feel dirty...
|
80
|
+
# FIXME: refactor `Writer` so it's not so tightly coupled to a AWS::SQS::ReceivedMessage object
|
81
|
+
http_message = message.dup
|
82
|
+
http_message.instance_variable_set(:@body, http_response)
|
83
|
+
http_message
|
84
|
+
end
|
85
|
+
|
86
|
+
def get(message)
|
87
|
+
msg_content = message_content(message)
|
88
|
+
url = url_generator.generate(msg_content)
|
89
|
+
|
90
|
+
logger.info(
|
91
|
+
event: 'Sending HTTP GET request',
|
92
|
+
url: url,
|
93
|
+
method: "#{self.class}#get"
|
94
|
+
)
|
95
|
+
|
96
|
+
res = Faraday.get(url)
|
97
|
+
|
98
|
+
logger.info(
|
99
|
+
event: 'HTTP request complete',
|
100
|
+
url: url,
|
101
|
+
status: res.status,
|
102
|
+
body: res.body,
|
103
|
+
method: "#{self.class}#get"
|
104
|
+
)
|
105
|
+
|
106
|
+
http_response_processor.process(msg_content, res.status, res.body)
|
107
|
+
end
|
108
|
+
|
109
|
+
def message_content(message)
|
110
|
+
::JSON.parse(message.body, symbolize_names: true)
|
111
|
+
end
|
112
|
+
end
|
113
|
+
end
|
114
|
+
end
|
115
|
+
end
|
@@ -0,0 +1,96 @@
|
|
1
|
+
module Alephant
|
2
|
+
module Publisher
|
3
|
+
module Queue
|
4
|
+
class RevalidateWriter
|
5
|
+
include Alephant::Logger
|
6
|
+
|
7
|
+
attr_reader :message
|
8
|
+
|
9
|
+
def initialize(config, message)
|
10
|
+
@config = config
|
11
|
+
@message = message
|
12
|
+
end
|
13
|
+
|
14
|
+
def run!
|
15
|
+
renderer.views.each do |view_id, view|
|
16
|
+
store(view_id, view)
|
17
|
+
write_lookup_record(view_id)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
def renderer
|
22
|
+
@renderer ||= Alephant::Renderer.create(config, http_data)
|
23
|
+
end
|
24
|
+
|
25
|
+
def storage
|
26
|
+
@storage ||= Alephant::Cache.new(config.fetch(:s3_bucket_id), config.fetch(:s3_object_path))
|
27
|
+
end
|
28
|
+
|
29
|
+
def lookup
|
30
|
+
@lookup ||= Alephant::Lookup.create(config.fetch(:lookup_table_name), config)
|
31
|
+
end
|
32
|
+
|
33
|
+
private
|
34
|
+
|
35
|
+
def write_lookup_record(view_id)
|
36
|
+
lookup.write(view_id, http_options.fetch(:options), seq_id, storage_location(view_id))
|
37
|
+
|
38
|
+
logger.info(event: 'LookupLocationUpdated',
|
39
|
+
view_id: view_id,
|
40
|
+
options: http_options.fetch(:options),
|
41
|
+
seq_id: seq_id,
|
42
|
+
location: storage_location(view_id),
|
43
|
+
renderer_id: config.fetch(:renderer_id),
|
44
|
+
method: "#{self.class}#write_lookup_record")
|
45
|
+
end
|
46
|
+
|
47
|
+
def store(view_id, view)
|
48
|
+
storage.put(storage_location(view_id), view.render, view.content_type, storage_opts)
|
49
|
+
|
50
|
+
logger.info(event: 'MessageStored',
|
51
|
+
location: storage_location(view_id),
|
52
|
+
view_id: view_id,
|
53
|
+
view: view,
|
54
|
+
content: view.render,
|
55
|
+
content_type: view.content_type,
|
56
|
+
storage_opts: storage_opts,
|
57
|
+
renderer_id: config.fetch(:renderer_id),
|
58
|
+
method: "#{self.class}#store")
|
59
|
+
end
|
60
|
+
|
61
|
+
# NOTE: we _really_ don't care about sequence here - we just _have_ to pass something through
|
62
|
+
def seq_id
|
63
|
+
1
|
64
|
+
end
|
65
|
+
|
66
|
+
def storage_location(view_id)
|
67
|
+
[
|
68
|
+
config.fetch(:renderer_id),
|
69
|
+
view_id,
|
70
|
+
Crimp.signature(http_options)
|
71
|
+
].join('/')
|
72
|
+
end
|
73
|
+
|
74
|
+
def storage_opts
|
75
|
+
{ ttl: message_content[:ttl] }
|
76
|
+
end
|
77
|
+
|
78
|
+
def config
|
79
|
+
@config.merge(renderer_id: message_content.fetch(:renderer_id))
|
80
|
+
end
|
81
|
+
|
82
|
+
def http_data
|
83
|
+
@http_data ||= ::JSON.parse(message_content.fetch(:http_response), symbolize_names: true)
|
84
|
+
end
|
85
|
+
|
86
|
+
def http_options
|
87
|
+
@http_options ||= message_content.fetch(:http_options)
|
88
|
+
end
|
89
|
+
|
90
|
+
def message_content
|
91
|
+
@message_content ||= ::JSON.parse(message.body, symbolize_names: true)
|
92
|
+
end
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|
@@ -13,12 +13,15 @@ module Alephant
|
|
13
13
|
class Writer
|
14
14
|
include Alephant::Logger
|
15
15
|
|
16
|
-
attr_reader :config, :message, :cache, :parser
|
16
|
+
attr_reader :config, :message, :cache, :parser
|
17
17
|
|
18
18
|
def initialize(config, message)
|
19
19
|
@config = config
|
20
20
|
@message = message
|
21
|
-
|
21
|
+
end
|
22
|
+
|
23
|
+
def renderer
|
24
|
+
@renderer ||= Alephant::Renderer.create(config, data)
|
22
25
|
end
|
23
26
|
|
24
27
|
def cache
|
@@ -65,7 +68,24 @@ module Alephant
|
|
65
68
|
end
|
66
69
|
|
67
70
|
def store(component, view, location, storage_opts = {})
|
71
|
+
logger.info(
|
72
|
+
event: 'StoreBeforeRender',
|
73
|
+
component: component,
|
74
|
+
view: view,
|
75
|
+
location: location,
|
76
|
+
storage_opts: storage_opts
|
77
|
+
)
|
78
|
+
|
68
79
|
render = view.render
|
80
|
+
|
81
|
+
logger.info(
|
82
|
+
event: 'StoreAfterRender',
|
83
|
+
component: component,
|
84
|
+
view: view,
|
85
|
+
location: location,
|
86
|
+
storage_opts: storage_opts
|
87
|
+
)
|
88
|
+
|
69
89
|
cache.put(location, render, view.content_type, storage_opts).tap do
|
70
90
|
logger.info(
|
71
91
|
"event" => "MessageStored",
|
@@ -78,6 +98,7 @@ module Alephant
|
|
78
98
|
"method" => "#{self.class}#store"
|
79
99
|
)
|
80
100
|
end
|
101
|
+
|
81
102
|
lookup.write(component, options, seq_id, location).tap do
|
82
103
|
logger.info(
|
83
104
|
"event" => "LookupLocationUpdated",
|
@@ -0,0 +1,140 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
RSpec.describe Alephant::Publisher::Queue::RevalidateProcessor do
|
4
|
+
class TestUrlGenerator
|
5
|
+
def self.generate(_opts = {})
|
6
|
+
'http://example.com'
|
7
|
+
end
|
8
|
+
end
|
9
|
+
|
10
|
+
class TestHttpResponseProcessor
|
11
|
+
def self.process(_opts, _response_status, response_body)
|
12
|
+
response_body
|
13
|
+
end
|
14
|
+
|
15
|
+
def self.ttl(_opts)
|
16
|
+
nil
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
subject { described_class.new(opts, url_generator, http_response_processor) }
|
21
|
+
|
22
|
+
let(:url_generator) { TestUrlGenerator }
|
23
|
+
let(:http_response_processor) { TestHttpResponseProcessor }
|
24
|
+
|
25
|
+
let(:opts) do
|
26
|
+
instance_double(Alephant::Publisher::Queue::Options,
|
27
|
+
writer: {},
|
28
|
+
cache: { elasticache_config_endpoint: 'wibble' })
|
29
|
+
end
|
30
|
+
|
31
|
+
let(:writer_double) { instance_double(Alephant::Publisher::Queue::RevalidateWriter, run!: nil) }
|
32
|
+
let(:cache_double) { instance_double(Dalli::Client, delete: nil) }
|
33
|
+
let(:elasticache_double) { instance_double(Dalli::ElastiCache, client: cache_double) }
|
34
|
+
|
35
|
+
let(:message) { instance_double(AWS::SQS::ReceivedMessage, body: JSON.generate(message_body), delete: nil) }
|
36
|
+
let(:message_body) { { id: '', batch_id: '', options: {} } }
|
37
|
+
|
38
|
+
before do
|
39
|
+
allow(Alephant::Publisher::Queue::RevalidateWriter)
|
40
|
+
.to receive(:new)
|
41
|
+
.and_return(writer_double)
|
42
|
+
|
43
|
+
allow(Dalli::ElastiCache)
|
44
|
+
.to receive(:new)
|
45
|
+
.and_return(elasticache_double)
|
46
|
+
end
|
47
|
+
|
48
|
+
describe '#consume' do
|
49
|
+
context 'when there is a message passed through' do
|
50
|
+
context 'when the HTTP request is successful' do
|
51
|
+
let(:resp_double) { double(body: resp_body, status: resp_status) }
|
52
|
+
let(:resp_body) { JSON.generate(id: 'foo') }
|
53
|
+
let(:resp_status) { 200 }
|
54
|
+
|
55
|
+
before do
|
56
|
+
allow(Faraday).to receive(:get).and_return(resp_double)
|
57
|
+
end
|
58
|
+
|
59
|
+
it 'calls #run! on the writer with the http request result' do
|
60
|
+
expect(Alephant::Publisher::Queue::RevalidateWriter)
|
61
|
+
.to receive(:new)
|
62
|
+
.with(opts.writer, anything)
|
63
|
+
.and_return(writer_double)
|
64
|
+
|
65
|
+
expect(writer_double).to receive(:run!)
|
66
|
+
|
67
|
+
subject.consume(message)
|
68
|
+
end
|
69
|
+
|
70
|
+
it 'passes the response to the http_response_processor' do
|
71
|
+
expect(http_response_processor)
|
72
|
+
.to receive(:process)
|
73
|
+
.with(message_body, resp_status, resp_body)
|
74
|
+
.and_call_original
|
75
|
+
|
76
|
+
subject.consume(message)
|
77
|
+
end
|
78
|
+
|
79
|
+
it "calls the 'ttl' method on the http_response_processor" do
|
80
|
+
expect(http_response_processor)
|
81
|
+
.to receive(:ttl)
|
82
|
+
.with(message_body)
|
83
|
+
.and_call_original
|
84
|
+
|
85
|
+
subject.consume(message)
|
86
|
+
end
|
87
|
+
|
88
|
+
it 'deletes the message from the queue' do
|
89
|
+
expect(message).to receive(:delete)
|
90
|
+
|
91
|
+
subject.consume(message)
|
92
|
+
end
|
93
|
+
|
94
|
+
it "removes the 'inflight' cache message" do
|
95
|
+
expect(cache_double).to receive(:delete)
|
96
|
+
|
97
|
+
subject.consume(message)
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
context 'when the HTTP request is unsuccessful' do
|
102
|
+
before do
|
103
|
+
allow(Faraday).to receive(:get).and_raise(Faraday::TimeoutError)
|
104
|
+
end
|
105
|
+
|
106
|
+
it 'does not call #run! on the writer' do
|
107
|
+
expect(writer_double).to_not receive(:run!)
|
108
|
+
|
109
|
+
expect { subject.consume(message) }
|
110
|
+
.to raise_error(Faraday::TimeoutError)
|
111
|
+
end
|
112
|
+
|
113
|
+
it 'does NOT delele the message from the queue' do
|
114
|
+
expect(message).to_not receive(:delete)
|
115
|
+
|
116
|
+
expect { subject.consume(message) }
|
117
|
+
.to raise_error(Faraday::TimeoutError)
|
118
|
+
end
|
119
|
+
|
120
|
+
it "does not remove the 'inflight' cache message" do
|
121
|
+
expect(cache_double).to_not receive(:delete)
|
122
|
+
|
123
|
+
expect { subject.consume(message) }
|
124
|
+
.to raise_error(Faraday::TimeoutError)
|
125
|
+
end
|
126
|
+
end
|
127
|
+
end
|
128
|
+
|
129
|
+
context 'when there is no message passed through' do
|
130
|
+
let(:message) { nil }
|
131
|
+
|
132
|
+
it 'does nothing' do
|
133
|
+
expect(writer_double).to_not receive(:run!)
|
134
|
+
expect(cache_double).to_not receive(:delete)
|
135
|
+
|
136
|
+
subject.consume(message)
|
137
|
+
end
|
138
|
+
end
|
139
|
+
end
|
140
|
+
end
|
@@ -0,0 +1,96 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
RSpec.describe Alephant::Publisher::Queue::RevalidateWriter do
|
4
|
+
subject { described_class.new(config, message) }
|
5
|
+
|
6
|
+
let(:config) do
|
7
|
+
{
|
8
|
+
s3_bucket_id: 'qwerty',
|
9
|
+
s3_object_path: 'hello/int',
|
10
|
+
lookup_table_name: 'lookup_table'
|
11
|
+
}
|
12
|
+
end
|
13
|
+
|
14
|
+
let(:message) { double(body: JSON.generate(message_body)) }
|
15
|
+
let(:http_response) { { ticker: 'WMT', val: 180.00 } }
|
16
|
+
let(:ttl) { 45 }
|
17
|
+
|
18
|
+
let(:message_body) do
|
19
|
+
{
|
20
|
+
renderer_id: 'hello_world',
|
21
|
+
http_options: { id: 'hello_world', options: { ticker: 'WMT', duration: '1_day' } },
|
22
|
+
http_response: JSON.generate(http_response),
|
23
|
+
ttl: ttl
|
24
|
+
}
|
25
|
+
end
|
26
|
+
|
27
|
+
describe '#run!' do
|
28
|
+
let(:storage_double) { instance_double(Alephant::Cache, put: nil) }
|
29
|
+
let(:lookup_double) { instance_double(Alephant::Lookup::LookupHelper, write: nil) }
|
30
|
+
let(:renderer_double) do
|
31
|
+
instance_double(Alephant::Renderer::Renderer, views: { hello_world_view: hello_world_view })
|
32
|
+
end
|
33
|
+
|
34
|
+
let(:hello_world_view) { double(render: rendered_content, content_type: rendered_content_type) }
|
35
|
+
let(:rendered_content) { '<h1>Hello, world!</h1>' }
|
36
|
+
let(:rendered_content_type) { 'text/html' }
|
37
|
+
|
38
|
+
let(:storage_location) { "hello_world/hello_world_view/#{Crimp.signature(message_body[:http_options])}" }
|
39
|
+
|
40
|
+
before do
|
41
|
+
allow(Alephant::Renderer).to receive(:create).and_return(renderer_double)
|
42
|
+
allow(Alephant::Cache).to receive(:new).and_return(storage_double)
|
43
|
+
allow(Alephant::Lookup).to receive(:create).and_return(lookup_double)
|
44
|
+
end
|
45
|
+
|
46
|
+
it 'renders the http_response in the Renderer' do
|
47
|
+
expect(renderer_double).to receive(:views).and_return(foo: hello_world_view)
|
48
|
+
expect(hello_world_view).to receive(:render)
|
49
|
+
expect(hello_world_view).to receive(:content_type)
|
50
|
+
|
51
|
+
subject.run!
|
52
|
+
end
|
53
|
+
|
54
|
+
it 'stores the HTTP content in S3 with Alephant::Cache' do
|
55
|
+
expect(storage_double)
|
56
|
+
.to receive(:put)
|
57
|
+
.with(storage_location, rendered_content, rendered_content_type, ttl: ttl)
|
58
|
+
|
59
|
+
subject.run!
|
60
|
+
end
|
61
|
+
|
62
|
+
it 'writes the S3 location with Alephant::Lookup' do
|
63
|
+
expect(lookup_double)
|
64
|
+
.to receive(:write)
|
65
|
+
.with(:hello_world_view, message_body[:http_options][:options], 1, storage_location)
|
66
|
+
|
67
|
+
subject.run!
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
describe '#renderer' do
|
72
|
+
it 'builds an Alephant::Renderer::Renderer object' do
|
73
|
+
expect(subject.renderer).to be_a(Alephant::Renderer::Renderer)
|
74
|
+
end
|
75
|
+
|
76
|
+
it 'passes through the `renderer_id` from the message' do
|
77
|
+
expect(Alephant::Renderer)
|
78
|
+
.to receive(:create)
|
79
|
+
.with(hash_including(renderer_id: 'hello_world'), http_response)
|
80
|
+
|
81
|
+
subject.renderer
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
describe '#storage' do
|
86
|
+
it 'builds an Alephant::Cache object' do
|
87
|
+
expect(subject.storage).to be_a(Alephant::Cache)
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
describe '#lookup' do
|
92
|
+
it 'builds an Alephant::Lookup object' do
|
93
|
+
expect(subject.lookup).to be_a(Alephant::Lookup::LookupHelper)
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|