alephant-broker 3.14.0 → 3.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. checksums.yaml +4 -4
  2. data/.travis.yml +22 -9
  3. data/README.md +35 -6
  4. data/alephant-broker.gemspec +3 -2
  5. data/lib/alephant/broker/cache.rb +1 -0
  6. data/lib/alephant/broker/cache/cached_object.rb +71 -0
  7. data/lib/alephant/broker/cache/client.rb +29 -17
  8. data/lib/alephant/broker/cache/null_client.rb +12 -4
  9. data/lib/alephant/broker/component.rb +3 -1
  10. data/lib/alephant/broker/component_meta.rb +2 -0
  11. data/lib/alephant/broker/error_component.rb +1 -1
  12. data/lib/alephant/broker/load_strategy.rb +1 -0
  13. data/lib/alephant/broker/load_strategy/http.rb +2 -3
  14. data/lib/alephant/broker/load_strategy/revalidate/fetcher.rb +51 -0
  15. data/lib/alephant/broker/load_strategy/revalidate/refresher.rb +75 -0
  16. data/lib/alephant/broker/load_strategy/revalidate/strategy.rb +86 -0
  17. data/lib/alephant/broker/load_strategy/s3/base.rb +2 -2
  18. data/lib/alephant/broker/response/asset.rb +1 -1
  19. data/lib/alephant/broker/response/base.rb +7 -8
  20. data/lib/alephant/broker/response/batch.rb +1 -1
  21. data/lib/alephant/broker/version.rb +1 -1
  22. data/spec/alephant/broker/cache/cached_object_spec.rb +107 -0
  23. data/spec/alephant/broker/load_strategy/http_spec.rb +5 -7
  24. data/spec/alephant/broker/load_strategy/revalidate/fetcher_spec.rb +88 -0
  25. data/spec/alephant/broker/load_strategy/revalidate/refresher_spec.rb +69 -0
  26. data/spec/alephant/broker/load_strategy/revalidate/strategy_spec.rb +185 -0
  27. data/spec/fixtures/json/batch_compiled_no_sequence.json +1 -0
  28. data/spec/integration/not_modified_response_spec.rb +4 -6
  29. data/spec/integration/rack_spec.rb +0 -3
  30. data/spec/integration/revalidate_spec.rb +174 -0
  31. data/spec/spec_helper.rb +1 -0
  32. metadata +99 -55
@@ -0,0 +1,86 @@
1
+ require 'alephant/broker/cache'
2
+ require 'alephant/broker/errors'
3
+ require 'alephant/logger'
4
+ require 'alephant/broker/load_strategy/revalidate/refresher'
5
+ require 'alephant/broker/load_strategy/revalidate/fetcher'
6
+ require 'faraday'
7
+
8
+ module Alephant
9
+ module Broker
10
+ module LoadStrategy
11
+ module Revalidate
12
+ class Strategy
13
+ include Logger
14
+
15
+ STORAGE_ERRORS = [Alephant::Broker::Errors::ContentNotFound].freeze
16
+
17
+ def load(component_meta)
18
+ loaded_content = cached_object(component_meta)
19
+
20
+ update_content(component_meta) if loaded_content.expired?
21
+
22
+ data = loaded_content.to_h
23
+ data.fetch(:meta, {})[:status] = 200
24
+ data
25
+ rescue *STORAGE_ERRORS
26
+ update_content(component_meta)
27
+
28
+ {
29
+ content: '',
30
+ content_type: 'text/html',
31
+ meta: { status: 202 }
32
+ }
33
+ end
34
+
35
+ private
36
+
37
+ attr_reader :cache
38
+
39
+ def cache
40
+ @cache ||= Cache::Client.new
41
+ end
42
+
43
+ def cached_object(component_meta)
44
+ cache.get(component_meta.component_key) do
45
+ logger.info(msg: "#{self.class}#cached_object - No cache so loading and adding cache object")
46
+ Fetcher.new(component_meta).fetch
47
+ end
48
+ end
49
+
50
+ def update_content(component_meta)
51
+ Thread.new do
52
+ stored_content = fetch_stored_content(component_meta)
53
+
54
+ if stored_content && !stored_content.expired?
55
+ cache_new_content(component_meta, stored_content)
56
+ else
57
+ refresh_content(component_meta)
58
+ end
59
+ end
60
+ end
61
+
62
+ def fetch_stored_content(component_meta)
63
+ Fetcher.new(component_meta).fetch
64
+ rescue *STORAGE_ERRORS
65
+ nil
66
+ end
67
+
68
+ def cache_new_content(component_meta, new_content)
69
+ logger.info(event: 'NewContentFromS3',
70
+ key: component_meta.component_key,
71
+ val: new_content,
72
+ method: "#{self.class}#refresh_content")
73
+
74
+ cache.set(component_meta.component_key, new_content)
75
+ end
76
+
77
+ def refresh_content(component_meta)
78
+ logger.info(msg: "#{self.class}#refresh_content - Loading new content from thread")
79
+
80
+ Refresher.new(component_meta).refresh
81
+ end
82
+ end
83
+ end
84
+ end
85
+ end
86
+ end
@@ -58,7 +58,7 @@ module Alephant
58
58
  end
59
59
 
60
60
  def retrieve_object(component_meta)
61
- cached = false
61
+ @cached = false
62
62
  s3.get s3_path(component_meta)
63
63
  rescue AWS::S3::Errors::NoSuchKey, InvalidCacheKey
64
64
  logger.metric "S3InvalidCacheKey"
@@ -87,7 +87,7 @@ module Alephant
87
87
 
88
88
  def headers(_component_meta)
89
89
  {
90
- "X-Cache-Version" => Broker.config["elasticache_cache_version"].to_s,
90
+ "X-Cache-Version" => (Broker.config[:elasticache_cache_version] || Broker.config["elasticache_cache_version"]).to_s,
91
91
  "X-Cached" => cached.to_s
92
92
  }
93
93
  end
@@ -9,7 +9,7 @@ module Alephant
9
9
  def initialize(component, request_env)
10
10
  @component = component
11
11
 
12
- @status = self.class.component_not_modified(@component.headers, request_env) ? NOT_MODIFIED_STATUS_CODE : component.status
12
+ @status = self.class.component_not_modified(@component.headers, request_env) ? 304 : component.status
13
13
 
14
14
  super(@status, component.content_type, request_env)
15
15
 
@@ -12,13 +12,12 @@ module Alephant
12
12
 
13
13
  attr_reader :content, :headers, :status
14
14
 
15
- NOT_MODIFIED_STATUS_CODE = 304
16
-
17
15
  STATUS_CODE_MAPPING = {
18
- 200 => "ok",
19
- NOT_MODIFIED_STATUS_CODE => "",
20
- 404 => "Not found",
21
- 500 => "Error retrieving content"
16
+ 200 => "ok",
17
+ 202 => "Accepted",
18
+ 304 => "",
19
+ 404 => "Not found",
20
+ 500 => "Error retrieving content"
22
21
  }.freeze
23
22
 
24
23
  def initialize(status = 200, content_type = "text/html", request_env = nil)
@@ -28,7 +27,7 @@ module Alephant
28
27
  "Access-Control-Allow-Headers" => "If-None-Match, If-Modified-Since",
29
28
  "Access-Control-Allow-Origin" => "*"
30
29
  }
31
- headers.merge!(Broker.config[:headers]) if Broker.config.key?(:headers)
30
+ @headers.merge!(Broker.config[:headers]) if Broker.config.key?(:headers)
32
31
  @status = status
33
32
 
34
33
  add_no_cache_headers if should_add_no_cache_headers?(status)
@@ -45,7 +44,7 @@ module Alephant
45
44
  private
46
45
 
47
46
  def should_add_no_cache_headers?(status)
48
- status != 200 && status != NOT_MODIFIED_STATUS_CODE
47
+ status != 200 && status != 304
49
48
  end
50
49
 
51
50
  def add_no_cache_headers
@@ -12,7 +12,7 @@ module Alephant
12
12
  def initialize(components, batch_id, request_env)
13
13
  @components = components
14
14
  @batch_id = batch_id
15
- @status = self.class.component_not_modified(batch_response_headers, request_env) ? NOT_MODIFIED_STATUS_CODE : 200
15
+ @status = self.class.component_not_modified(batch_response_headers, request_env) ? 304 : 200
16
16
 
17
17
  super(@status, "application/json", request_env)
18
18
 
@@ -1,5 +1,5 @@
1
1
  module Alephant
2
2
  module Broker
3
- VERSION = "3.14.0".freeze
3
+ VERSION = "3.15.0".freeze
4
4
  end
5
5
  end
@@ -0,0 +1,107 @@
1
+ require 'spec_helper'
2
+
3
+ RSpec.describe Alephant::Broker::Cache::CachedObject do
4
+ subject { described_class.new(s3_obj) }
5
+
6
+ let(:config) { {} }
7
+ let(:last_modified) { Time.parse('Mon, 11 Apr 2016 10:39:57 GMT') }
8
+ let(:ttl) { 15 }
9
+
10
+ let(:s3_obj) do
11
+ {
12
+ content: 'Test',
13
+ content_type: 'test/content',
14
+ meta: {
15
+ :ttl => ttl,
16
+ :head_ETag => '123',
17
+ :'head_Last-Modified' => last_modified.to_s
18
+ }
19
+ }
20
+ end
21
+
22
+ before do
23
+ allow_any_instance_of(Logger).to receive(:info)
24
+ allow(Alephant::Broker).to receive(:config).and_return(config)
25
+ end
26
+
27
+ describe '#updated' do
28
+ it 'extracts the #updated time from the S3 object' do
29
+ Timecop.freeze do
30
+ expect(subject.updated).to eq(last_modified)
31
+ end
32
+ end
33
+
34
+ context 'when there is no Last-Modified on the S3 object' do
35
+ let(:last_modified) { nil }
36
+
37
+ it 'sets #updated to now' do
38
+ now = Time.parse('Mon, 31 May 2016 12:00:00 GMT')
39
+
40
+ Timecop.freeze(now) do
41
+ expect(subject.updated).to eq(now)
42
+ end
43
+ end
44
+ end
45
+ end
46
+
47
+ describe '#ttl' do
48
+ it 'extracts the #ttl from the S3 object' do
49
+ expect(subject.ttl).to eq(ttl)
50
+ end
51
+
52
+ context 'when there is no TTL on the S3 object' do
53
+ let(:ttl) { nil }
54
+
55
+ context 'and a default cache TTL has been configured' do
56
+ let(:config) { { revalidate_cache_ttl: 100 } }
57
+
58
+ it 'sets the #ttl to the configured value' do
59
+ expect(subject.ttl).to eq(100)
60
+ end
61
+ end
62
+
63
+ context 'and a default cache TTL has NOT been configured' do
64
+ it 'sets the #ttl to a default (in code) value' do
65
+ expect(subject.ttl).to eq(described_class::DEFAULT_TTL)
66
+ end
67
+ end
68
+ end
69
+ end
70
+
71
+ describe '#update' do
72
+ let(:new_content) do
73
+ {
74
+ content: 'Test - NEW',
75
+ content_type: 'test/content',
76
+ meta: {}
77
+ }
78
+ end
79
+
80
+ it 'updates #s3_obj' do
81
+ expect { subject.update(new_content) }
82
+ .to change { subject.s3_obj }
83
+ .from(s3_obj)
84
+ .to(new_content)
85
+ end
86
+ end
87
+
88
+ describe '#expired?' do
89
+ context 'when the object is "young"' do
90
+ it 'should be false' do
91
+ Timecop.freeze(last_modified) do
92
+ expect(subject.expired?).to be false
93
+ end
94
+ end
95
+ end
96
+
97
+ context 'when the object is "old"' do
98
+ it 'shoule be true' do
99
+ new_time = last_modified + subject.ttl + 100
100
+
101
+ Timecop.freeze(new_time) do
102
+ expect(subject.expired?).to be true
103
+ end
104
+ end
105
+ end
106
+ end
107
+ end
@@ -4,15 +4,14 @@ describe Alephant::Broker::LoadStrategy::HTTP do
4
4
  subject { described_class.new(url_generator) }
5
5
 
6
6
  let(:component_meta) do
7
- double(
8
- "Alephant::Broker::ComponentMeta",
9
- :cache_key => "cache_key",
10
- :id => "test",
11
- :options => {}
7
+ instance_double(Alephant::Broker::ComponentMeta,
8
+ :id => "test",
9
+ :options => {},
10
+ :component_key => "cache_key"
12
11
  )
13
12
  end
14
13
  let(:url_generator) { double(:generate => "http://foo.bar") }
15
- let(:cache) { double("Alephant::Broker::Cache::Client") }
14
+ let(:cache) { instance_double(Alephant::Broker::Cache::Client) }
16
15
  let(:body) { "<h1>Batman!</h1>" }
17
16
  let(:content) do
18
17
  {
@@ -39,7 +38,6 @@ describe Alephant::Broker::LoadStrategy::HTTP do
39
38
  context "content not in cache" do
40
39
  before :each do
41
40
  allow(cache).to receive(:get).and_yield
42
- allow(component_meta).to receive(:'cached=').with(false) { false }
43
41
  end
44
42
 
45
43
  context "and available over HTTP" do
@@ -0,0 +1,88 @@
1
+ require "spec_helper"
2
+
3
+ RSpec.describe Alephant::Broker::LoadStrategy::Revalidate::Fetcher do
4
+ subject { described_class.new(component_meta) }
5
+
6
+ let(:component_meta) do
7
+ Alephant::Broker::ComponentMeta.new("test", "test_batch", {})
8
+ end
9
+
10
+ let(:lookup_double) { instance_double(Alephant::Lookup::LookupHelper) }
11
+ let(:storage_double) { instance_double(Alephant::Storage) }
12
+
13
+ before do
14
+ allow(Alephant::Lookup).to receive(:create).and_return(lookup_double)
15
+ allow(Alephant::Storage).to receive(:new).and_return(storage_double)
16
+ allow(Alephant::Broker).to receive(:config).and_return({})
17
+ allow_any_instance_of(Logger).to receive(:info)
18
+ end
19
+
20
+ describe "#fetch" do
21
+ context "when there is something in DynamoDB & S3" do
22
+ let(:content_body) { "<h1>w00t</h1>" }
23
+ let(:content_type) { "text/html" }
24
+
25
+ let(:content) do
26
+ instance_double(AWS::S3::S3Object,
27
+ :content_type => "test/content",
28
+ :read => "Test",
29
+ :metadata => {
30
+ "ttl" => 30,
31
+ "head_ETag" => "123",
32
+ "head_Last-Modified" => "Mon, 11 Apr 2016 10:39:57 GMT"
33
+ }
34
+ )
35
+ end
36
+
37
+ before do
38
+ allow(lookup_double)
39
+ .to receive(:read)
40
+ .and_return(spy(:location => "/foo/bar"))
41
+
42
+ allow(storage_double)
43
+ .to receive(:get)
44
+ .with("/foo/bar")
45
+ .and_return(content)
46
+ end
47
+
48
+ it "fetches and returns the content as a CachedObject" do
49
+ returned_data = subject.fetch
50
+ expect(returned_data.s3_obj).to eq(content)
51
+ end
52
+ end
53
+
54
+ context "when there is NO entry in DynamoDB" do
55
+ before do
56
+ allow(lookup_double)
57
+ .to receive(:read)
58
+ .and_return(spy(:location => nil))
59
+ end
60
+
61
+ it "raises an Alephant::Broker::Errors::ContentNotFound error" do
62
+ expect { subject.fetch }
63
+ .to raise_error(Alephant::Broker::Errors::ContentNotFound)
64
+ end
65
+ end
66
+
67
+ context "when there is an entry in DynamoDB" do
68
+ before do
69
+ allow(lookup_double)
70
+ .to receive(:read)
71
+ .and_return(spy(:location => "/foo/bar"))
72
+ end
73
+
74
+ context "but there is NO content in S3" do
75
+ before do
76
+ allow(storage_double)
77
+ .to receive(:get)
78
+ .and_raise(AWS::S3::Errors::NoSuchKey.new(nil, nil))
79
+ end
80
+
81
+ it "raises an Alephant::Broker::Errors::ContentNotFound error" do
82
+ expect { subject.fetch }
83
+ .to raise_error(Alephant::Broker::Errors::ContentNotFound)
84
+ end
85
+ end
86
+ end
87
+ end
88
+ end
@@ -0,0 +1,69 @@
1
+ require "spec_helper"
2
+
3
+ RSpec.describe Alephant::Broker::LoadStrategy::Revalidate::Refresher do
4
+ subject { described_class.new(component_meta) }
5
+
6
+ let(:component_meta) do
7
+ Alephant::Broker::ComponentMeta.new("test", "test_batch", {})
8
+ end
9
+
10
+ let(:sqs_double) { instance_double(AWS::SQS, :queues => sqs_queues_double) }
11
+ let(:sqs_queue_double) { instance_double(AWS::SQS::Queue) }
12
+ let(:sqs_queues_double) { instance_double(AWS::SQS::QueueCollection, :url_for => "example.com", :[] => sqs_queue_double) }
13
+
14
+ let(:config) { { :aws_account_id => "12345", :sqs_queue_name => "bob" } }
15
+
16
+ let(:cache) { subject.send(:cache) }
17
+ let(:cache_key) { subject.send(:cache_key) }
18
+ let(:inflight_cache_key) { subject.send(:inflight_cache_key) }
19
+
20
+ before do
21
+ allow_any_instance_of(Logger).to receive(:info)
22
+ allow_any_instance_of(Logger).to receive(:debug)
23
+ allow(AWS::SQS).to receive(:new).and_return(sqs_double)
24
+ allow(Alephant::Broker).to receive(:config).and_return(config)
25
+ end
26
+
27
+ describe "#refresh" do
28
+ context "when there is already a 'inflight' message in the cache" do
29
+ before do
30
+ expect(cache)
31
+ .to receive(:get)
32
+ .with(subject.send(:inflight_cache_key))
33
+ .and_return("true")
34
+ end
35
+
36
+ it "does nothing" do
37
+ expect(cache).to_not receive(:set)
38
+
39
+ subject.refresh
40
+ end
41
+ end
42
+
43
+ context "when there is NOT already a 'inflight' message in the cache" do
44
+ before do
45
+ expect(cache)
46
+ .to receive(:get)
47
+ .with(inflight_cache_key)
48
+ .and_return(nil)
49
+ end
50
+
51
+ it "adds a message to the SQS queue, ",
52
+ "and puts a 'inflight' message in the cache" do
53
+ expect(cache).to receive(:set).with(inflight_cache_key, true, described_class::INFLIGHT_CACHE_TTL)
54
+ expect(sqs_queue_double).to receive(:send_message)
55
+
56
+ subject.refresh
57
+ end
58
+
59
+ context "there was a problem pushing a message onto the queue" do
60
+ it "does NOT put an 'inflight' message in the cache" do
61
+ expect(cache).to_not receive(:set)
62
+ expect(sqs_queue_double).to receive(:send_message).and_raise
63
+
64
+ expect { subject.refresh }.to raise_error
65
+ end
66
+ end
67
+ end
68
+ end
69
+ end