alephant-broker 3.14.0 → 3.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +22 -9
- data/README.md +35 -6
- data/alephant-broker.gemspec +3 -2
- data/lib/alephant/broker/cache.rb +1 -0
- data/lib/alephant/broker/cache/cached_object.rb +71 -0
- data/lib/alephant/broker/cache/client.rb +29 -17
- data/lib/alephant/broker/cache/null_client.rb +12 -4
- data/lib/alephant/broker/component.rb +3 -1
- data/lib/alephant/broker/component_meta.rb +2 -0
- data/lib/alephant/broker/error_component.rb +1 -1
- data/lib/alephant/broker/load_strategy.rb +1 -0
- data/lib/alephant/broker/load_strategy/http.rb +2 -3
- data/lib/alephant/broker/load_strategy/revalidate/fetcher.rb +51 -0
- data/lib/alephant/broker/load_strategy/revalidate/refresher.rb +75 -0
- data/lib/alephant/broker/load_strategy/revalidate/strategy.rb +86 -0
- data/lib/alephant/broker/load_strategy/s3/base.rb +2 -2
- data/lib/alephant/broker/response/asset.rb +1 -1
- data/lib/alephant/broker/response/base.rb +7 -8
- data/lib/alephant/broker/response/batch.rb +1 -1
- data/lib/alephant/broker/version.rb +1 -1
- data/spec/alephant/broker/cache/cached_object_spec.rb +107 -0
- data/spec/alephant/broker/load_strategy/http_spec.rb +5 -7
- data/spec/alephant/broker/load_strategy/revalidate/fetcher_spec.rb +88 -0
- data/spec/alephant/broker/load_strategy/revalidate/refresher_spec.rb +69 -0
- data/spec/alephant/broker/load_strategy/revalidate/strategy_spec.rb +185 -0
- data/spec/fixtures/json/batch_compiled_no_sequence.json +1 -0
- data/spec/integration/not_modified_response_spec.rb +4 -6
- data/spec/integration/rack_spec.rb +0 -3
- data/spec/integration/revalidate_spec.rb +174 -0
- data/spec/spec_helper.rb +1 -0
- metadata +99 -55
@@ -0,0 +1,86 @@
|
|
1
|
+
require 'alephant/broker/cache'
|
2
|
+
require 'alephant/broker/errors'
|
3
|
+
require 'alephant/logger'
|
4
|
+
require 'alephant/broker/load_strategy/revalidate/refresher'
|
5
|
+
require 'alephant/broker/load_strategy/revalidate/fetcher'
|
6
|
+
require 'faraday'
|
7
|
+
|
8
|
+
module Alephant
|
9
|
+
module Broker
|
10
|
+
module LoadStrategy
|
11
|
+
module Revalidate
|
12
|
+
class Strategy
|
13
|
+
include Logger
|
14
|
+
|
15
|
+
STORAGE_ERRORS = [Alephant::Broker::Errors::ContentNotFound].freeze
|
16
|
+
|
17
|
+
def load(component_meta)
|
18
|
+
loaded_content = cached_object(component_meta)
|
19
|
+
|
20
|
+
update_content(component_meta) if loaded_content.expired?
|
21
|
+
|
22
|
+
data = loaded_content.to_h
|
23
|
+
data.fetch(:meta, {})[:status] = 200
|
24
|
+
data
|
25
|
+
rescue *STORAGE_ERRORS
|
26
|
+
update_content(component_meta)
|
27
|
+
|
28
|
+
{
|
29
|
+
content: '',
|
30
|
+
content_type: 'text/html',
|
31
|
+
meta: { status: 202 }
|
32
|
+
}
|
33
|
+
end
|
34
|
+
|
35
|
+
private
|
36
|
+
|
37
|
+
attr_reader :cache
|
38
|
+
|
39
|
+
def cache
|
40
|
+
@cache ||= Cache::Client.new
|
41
|
+
end
|
42
|
+
|
43
|
+
def cached_object(component_meta)
|
44
|
+
cache.get(component_meta.component_key) do
|
45
|
+
logger.info(msg: "#{self.class}#cached_object - No cache so loading and adding cache object")
|
46
|
+
Fetcher.new(component_meta).fetch
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
def update_content(component_meta)
|
51
|
+
Thread.new do
|
52
|
+
stored_content = fetch_stored_content(component_meta)
|
53
|
+
|
54
|
+
if stored_content && !stored_content.expired?
|
55
|
+
cache_new_content(component_meta, stored_content)
|
56
|
+
else
|
57
|
+
refresh_content(component_meta)
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
def fetch_stored_content(component_meta)
|
63
|
+
Fetcher.new(component_meta).fetch
|
64
|
+
rescue *STORAGE_ERRORS
|
65
|
+
nil
|
66
|
+
end
|
67
|
+
|
68
|
+
def cache_new_content(component_meta, new_content)
|
69
|
+
logger.info(event: 'NewContentFromS3',
|
70
|
+
key: component_meta.component_key,
|
71
|
+
val: new_content,
|
72
|
+
method: "#{self.class}#refresh_content")
|
73
|
+
|
74
|
+
cache.set(component_meta.component_key, new_content)
|
75
|
+
end
|
76
|
+
|
77
|
+
def refresh_content(component_meta)
|
78
|
+
logger.info(msg: "#{self.class}#refresh_content - Loading new content from thread")
|
79
|
+
|
80
|
+
Refresher.new(component_meta).refresh
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
@@ -58,7 +58,7 @@ module Alephant
|
|
58
58
|
end
|
59
59
|
|
60
60
|
def retrieve_object(component_meta)
|
61
|
-
cached = false
|
61
|
+
@cached = false
|
62
62
|
s3.get s3_path(component_meta)
|
63
63
|
rescue AWS::S3::Errors::NoSuchKey, InvalidCacheKey
|
64
64
|
logger.metric "S3InvalidCacheKey"
|
@@ -87,7 +87,7 @@ module Alephant
|
|
87
87
|
|
88
88
|
def headers(_component_meta)
|
89
89
|
{
|
90
|
-
"X-Cache-Version" => Broker.config["elasticache_cache_version"].to_s,
|
90
|
+
"X-Cache-Version" => (Broker.config[:elasticache_cache_version] || Broker.config["elasticache_cache_version"]).to_s,
|
91
91
|
"X-Cached" => cached.to_s
|
92
92
|
}
|
93
93
|
end
|
@@ -9,7 +9,7 @@ module Alephant
|
|
9
9
|
def initialize(component, request_env)
|
10
10
|
@component = component
|
11
11
|
|
12
|
-
@status = self.class.component_not_modified(@component.headers, request_env) ?
|
12
|
+
@status = self.class.component_not_modified(@component.headers, request_env) ? 304 : component.status
|
13
13
|
|
14
14
|
super(@status, component.content_type, request_env)
|
15
15
|
|
@@ -12,13 +12,12 @@ module Alephant
|
|
12
12
|
|
13
13
|
attr_reader :content, :headers, :status
|
14
14
|
|
15
|
-
NOT_MODIFIED_STATUS_CODE = 304
|
16
|
-
|
17
15
|
STATUS_CODE_MAPPING = {
|
18
|
-
200
|
19
|
-
|
20
|
-
|
21
|
-
|
16
|
+
200 => "ok",
|
17
|
+
202 => "Accepted",
|
18
|
+
304 => "",
|
19
|
+
404 => "Not found",
|
20
|
+
500 => "Error retrieving content"
|
22
21
|
}.freeze
|
23
22
|
|
24
23
|
def initialize(status = 200, content_type = "text/html", request_env = nil)
|
@@ -28,7 +27,7 @@ module Alephant
|
|
28
27
|
"Access-Control-Allow-Headers" => "If-None-Match, If-Modified-Since",
|
29
28
|
"Access-Control-Allow-Origin" => "*"
|
30
29
|
}
|
31
|
-
headers.merge!(Broker.config[:headers]) if Broker.config.key?(:headers)
|
30
|
+
@headers.merge!(Broker.config[:headers]) if Broker.config.key?(:headers)
|
32
31
|
@status = status
|
33
32
|
|
34
33
|
add_no_cache_headers if should_add_no_cache_headers?(status)
|
@@ -45,7 +44,7 @@ module Alephant
|
|
45
44
|
private
|
46
45
|
|
47
46
|
def should_add_no_cache_headers?(status)
|
48
|
-
status != 200 && status !=
|
47
|
+
status != 200 && status != 304
|
49
48
|
end
|
50
49
|
|
51
50
|
def add_no_cache_headers
|
@@ -12,7 +12,7 @@ module Alephant
|
|
12
12
|
def initialize(components, batch_id, request_env)
|
13
13
|
@components = components
|
14
14
|
@batch_id = batch_id
|
15
|
-
@status = self.class.component_not_modified(batch_response_headers, request_env) ?
|
15
|
+
@status = self.class.component_not_modified(batch_response_headers, request_env) ? 304 : 200
|
16
16
|
|
17
17
|
super(@status, "application/json", request_env)
|
18
18
|
|
@@ -0,0 +1,107 @@
|
|
1
|
+
require 'spec_helper'
|
2
|
+
|
3
|
+
RSpec.describe Alephant::Broker::Cache::CachedObject do
|
4
|
+
subject { described_class.new(s3_obj) }
|
5
|
+
|
6
|
+
let(:config) { {} }
|
7
|
+
let(:last_modified) { Time.parse('Mon, 11 Apr 2016 10:39:57 GMT') }
|
8
|
+
let(:ttl) { 15 }
|
9
|
+
|
10
|
+
let(:s3_obj) do
|
11
|
+
{
|
12
|
+
content: 'Test',
|
13
|
+
content_type: 'test/content',
|
14
|
+
meta: {
|
15
|
+
:ttl => ttl,
|
16
|
+
:head_ETag => '123',
|
17
|
+
:'head_Last-Modified' => last_modified.to_s
|
18
|
+
}
|
19
|
+
}
|
20
|
+
end
|
21
|
+
|
22
|
+
before do
|
23
|
+
allow_any_instance_of(Logger).to receive(:info)
|
24
|
+
allow(Alephant::Broker).to receive(:config).and_return(config)
|
25
|
+
end
|
26
|
+
|
27
|
+
describe '#updated' do
|
28
|
+
it 'extracts the #updated time from the S3 object' do
|
29
|
+
Timecop.freeze do
|
30
|
+
expect(subject.updated).to eq(last_modified)
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
34
|
+
context 'when there is no Last-Modified on the S3 object' do
|
35
|
+
let(:last_modified) { nil }
|
36
|
+
|
37
|
+
it 'sets #updated to now' do
|
38
|
+
now = Time.parse('Mon, 31 May 2016 12:00:00 GMT')
|
39
|
+
|
40
|
+
Timecop.freeze(now) do
|
41
|
+
expect(subject.updated).to eq(now)
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
describe '#ttl' do
|
48
|
+
it 'extracts the #ttl from the S3 object' do
|
49
|
+
expect(subject.ttl).to eq(ttl)
|
50
|
+
end
|
51
|
+
|
52
|
+
context 'when there is no TTL on the S3 object' do
|
53
|
+
let(:ttl) { nil }
|
54
|
+
|
55
|
+
context 'and a default cache TTL has been configured' do
|
56
|
+
let(:config) { { revalidate_cache_ttl: 100 } }
|
57
|
+
|
58
|
+
it 'sets the #ttl to the configured value' do
|
59
|
+
expect(subject.ttl).to eq(100)
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
context 'and a default cache TTL has NOT been configured' do
|
64
|
+
it 'sets the #ttl to a default (in code) value' do
|
65
|
+
expect(subject.ttl).to eq(described_class::DEFAULT_TTL)
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
describe '#update' do
|
72
|
+
let(:new_content) do
|
73
|
+
{
|
74
|
+
content: 'Test - NEW',
|
75
|
+
content_type: 'test/content',
|
76
|
+
meta: {}
|
77
|
+
}
|
78
|
+
end
|
79
|
+
|
80
|
+
it 'updates #s3_obj' do
|
81
|
+
expect { subject.update(new_content) }
|
82
|
+
.to change { subject.s3_obj }
|
83
|
+
.from(s3_obj)
|
84
|
+
.to(new_content)
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
88
|
+
describe '#expired?' do
|
89
|
+
context 'when the object is "young"' do
|
90
|
+
it 'should be false' do
|
91
|
+
Timecop.freeze(last_modified) do
|
92
|
+
expect(subject.expired?).to be false
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
96
|
+
|
97
|
+
context 'when the object is "old"' do
|
98
|
+
it 'shoule be true' do
|
99
|
+
new_time = last_modified + subject.ttl + 100
|
100
|
+
|
101
|
+
Timecop.freeze(new_time) do
|
102
|
+
expect(subject.expired?).to be true
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
106
|
+
end
|
107
|
+
end
|
@@ -4,15 +4,14 @@ describe Alephant::Broker::LoadStrategy::HTTP do
|
|
4
4
|
subject { described_class.new(url_generator) }
|
5
5
|
|
6
6
|
let(:component_meta) do
|
7
|
-
|
8
|
-
"
|
9
|
-
:
|
10
|
-
:
|
11
|
-
:options => {}
|
7
|
+
instance_double(Alephant::Broker::ComponentMeta,
|
8
|
+
:id => "test",
|
9
|
+
:options => {},
|
10
|
+
:component_key => "cache_key"
|
12
11
|
)
|
13
12
|
end
|
14
13
|
let(:url_generator) { double(:generate => "http://foo.bar") }
|
15
|
-
let(:cache) {
|
14
|
+
let(:cache) { instance_double(Alephant::Broker::Cache::Client) }
|
16
15
|
let(:body) { "<h1>Batman!</h1>" }
|
17
16
|
let(:content) do
|
18
17
|
{
|
@@ -39,7 +38,6 @@ describe Alephant::Broker::LoadStrategy::HTTP do
|
|
39
38
|
context "content not in cache" do
|
40
39
|
before :each do
|
41
40
|
allow(cache).to receive(:get).and_yield
|
42
|
-
allow(component_meta).to receive(:'cached=').with(false) { false }
|
43
41
|
end
|
44
42
|
|
45
43
|
context "and available over HTTP" do
|
@@ -0,0 +1,88 @@
|
|
1
|
+
require "spec_helper"
|
2
|
+
|
3
|
+
RSpec.describe Alephant::Broker::LoadStrategy::Revalidate::Fetcher do
|
4
|
+
subject { described_class.new(component_meta) }
|
5
|
+
|
6
|
+
let(:component_meta) do
|
7
|
+
Alephant::Broker::ComponentMeta.new("test", "test_batch", {})
|
8
|
+
end
|
9
|
+
|
10
|
+
let(:lookup_double) { instance_double(Alephant::Lookup::LookupHelper) }
|
11
|
+
let(:storage_double) { instance_double(Alephant::Storage) }
|
12
|
+
|
13
|
+
before do
|
14
|
+
allow(Alephant::Lookup).to receive(:create).and_return(lookup_double)
|
15
|
+
allow(Alephant::Storage).to receive(:new).and_return(storage_double)
|
16
|
+
allow(Alephant::Broker).to receive(:config).and_return({})
|
17
|
+
allow_any_instance_of(Logger).to receive(:info)
|
18
|
+
end
|
19
|
+
|
20
|
+
describe "#fetch" do
|
21
|
+
context "when there is something in DynamoDB & S3" do
|
22
|
+
let(:content_body) { "<h1>w00t</h1>" }
|
23
|
+
let(:content_type) { "text/html" }
|
24
|
+
|
25
|
+
let(:content) do
|
26
|
+
instance_double(AWS::S3::S3Object,
|
27
|
+
:content_type => "test/content",
|
28
|
+
:read => "Test",
|
29
|
+
:metadata => {
|
30
|
+
"ttl" => 30,
|
31
|
+
"head_ETag" => "123",
|
32
|
+
"head_Last-Modified" => "Mon, 11 Apr 2016 10:39:57 GMT"
|
33
|
+
}
|
34
|
+
)
|
35
|
+
end
|
36
|
+
|
37
|
+
before do
|
38
|
+
allow(lookup_double)
|
39
|
+
.to receive(:read)
|
40
|
+
.and_return(spy(:location => "/foo/bar"))
|
41
|
+
|
42
|
+
allow(storage_double)
|
43
|
+
.to receive(:get)
|
44
|
+
.with("/foo/bar")
|
45
|
+
.and_return(content)
|
46
|
+
end
|
47
|
+
|
48
|
+
it "fetches and returns the content as a CachedObject" do
|
49
|
+
returned_data = subject.fetch
|
50
|
+
expect(returned_data.s3_obj).to eq(content)
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
context "when there is NO entry in DynamoDB" do
|
55
|
+
before do
|
56
|
+
allow(lookup_double)
|
57
|
+
.to receive(:read)
|
58
|
+
.and_return(spy(:location => nil))
|
59
|
+
end
|
60
|
+
|
61
|
+
it "raises an Alephant::Broker::Errors::ContentNotFound error" do
|
62
|
+
expect { subject.fetch }
|
63
|
+
.to raise_error(Alephant::Broker::Errors::ContentNotFound)
|
64
|
+
end
|
65
|
+
end
|
66
|
+
|
67
|
+
context "when there is an entry in DynamoDB" do
|
68
|
+
before do
|
69
|
+
allow(lookup_double)
|
70
|
+
.to receive(:read)
|
71
|
+
.and_return(spy(:location => "/foo/bar"))
|
72
|
+
end
|
73
|
+
|
74
|
+
context "but there is NO content in S3" do
|
75
|
+
before do
|
76
|
+
allow(storage_double)
|
77
|
+
.to receive(:get)
|
78
|
+
.and_raise(AWS::S3::Errors::NoSuchKey.new(nil, nil))
|
79
|
+
end
|
80
|
+
|
81
|
+
it "raises an Alephant::Broker::Errors::ContentNotFound error" do
|
82
|
+
expect { subject.fetch }
|
83
|
+
.to raise_error(Alephant::Broker::Errors::ContentNotFound)
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
@@ -0,0 +1,69 @@
|
|
1
|
+
require "spec_helper"
|
2
|
+
|
3
|
+
RSpec.describe Alephant::Broker::LoadStrategy::Revalidate::Refresher do
|
4
|
+
subject { described_class.new(component_meta) }
|
5
|
+
|
6
|
+
let(:component_meta) do
|
7
|
+
Alephant::Broker::ComponentMeta.new("test", "test_batch", {})
|
8
|
+
end
|
9
|
+
|
10
|
+
let(:sqs_double) { instance_double(AWS::SQS, :queues => sqs_queues_double) }
|
11
|
+
let(:sqs_queue_double) { instance_double(AWS::SQS::Queue) }
|
12
|
+
let(:sqs_queues_double) { instance_double(AWS::SQS::QueueCollection, :url_for => "example.com", :[] => sqs_queue_double) }
|
13
|
+
|
14
|
+
let(:config) { { :aws_account_id => "12345", :sqs_queue_name => "bob" } }
|
15
|
+
|
16
|
+
let(:cache) { subject.send(:cache) }
|
17
|
+
let(:cache_key) { subject.send(:cache_key) }
|
18
|
+
let(:inflight_cache_key) { subject.send(:inflight_cache_key) }
|
19
|
+
|
20
|
+
before do
|
21
|
+
allow_any_instance_of(Logger).to receive(:info)
|
22
|
+
allow_any_instance_of(Logger).to receive(:debug)
|
23
|
+
allow(AWS::SQS).to receive(:new).and_return(sqs_double)
|
24
|
+
allow(Alephant::Broker).to receive(:config).and_return(config)
|
25
|
+
end
|
26
|
+
|
27
|
+
describe "#refresh" do
|
28
|
+
context "when there is already a 'inflight' message in the cache" do
|
29
|
+
before do
|
30
|
+
expect(cache)
|
31
|
+
.to receive(:get)
|
32
|
+
.with(subject.send(:inflight_cache_key))
|
33
|
+
.and_return("true")
|
34
|
+
end
|
35
|
+
|
36
|
+
it "does nothing" do
|
37
|
+
expect(cache).to_not receive(:set)
|
38
|
+
|
39
|
+
subject.refresh
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
context "when there is NOT already a 'inflight' message in the cache" do
|
44
|
+
before do
|
45
|
+
expect(cache)
|
46
|
+
.to receive(:get)
|
47
|
+
.with(inflight_cache_key)
|
48
|
+
.and_return(nil)
|
49
|
+
end
|
50
|
+
|
51
|
+
it "adds a message to the SQS queue, ",
|
52
|
+
"and puts a 'inflight' message in the cache" do
|
53
|
+
expect(cache).to receive(:set).with(inflight_cache_key, true, described_class::INFLIGHT_CACHE_TTL)
|
54
|
+
expect(sqs_queue_double).to receive(:send_message)
|
55
|
+
|
56
|
+
subject.refresh
|
57
|
+
end
|
58
|
+
|
59
|
+
context "there was a problem pushing a message onto the queue" do
|
60
|
+
it "does NOT put an 'inflight' message in the cache" do
|
61
|
+
expect(cache).to_not receive(:set)
|
62
|
+
expect(sqs_queue_double).to receive(:send_message).and_raise
|
63
|
+
|
64
|
+
expect { subject.refresh }.to raise_error
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|