alephant-broker 3.14.0 → 3.15.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a1de9bc629522c3ada2ac0f2ad0c08aa5024a037
-  data.tar.gz: b43733742699f3a88442386e88c4e77c3c782373
+  metadata.gz: 18fa7932242b113184d3dd00fcea6fd0700d6bd8
+  data.tar.gz: 4b8c2034dfe9275d18aa1163bc65bd47d8463cb8
 SHA512:
-  metadata.gz: 5d36de810da755b2210767bdeaf5e1d0230d65e1f43aea5266e19d569665f3b51255200fdb4ab14eafa925735f9b18b43f92c4403e20993b016cf9f4f68f64e3
-  data.tar.gz: 1ff51c8a99183673ba61538b20d9d12c6f1ee9d89091a88057bdad4d8d6c38230cf252242fc4569ea457c73d9cbe76d2ba3f24d5c4e411cdd2cc297de2008901
+  metadata.gz: 3183c49f2222401e560b929fed1887e3b67c169bff1b33764489955a7cf14fcb144d40d79e9006cb1f42d282989ee1cafec562bfebb81845a42577f3c76df24e
+  data.tar.gz: 5fe75198d006255e0573c4ac4fec30e49b622e75f4059ea1089edcc53fa7f61cfebd12b7c79ab9363b59ba25f61f9b31b7900e6be28457351399a518c4831b89

data/.travis.yml

@@ -1,11 +1,24 @@
+sudo: false
+cache: bundler
 language: ruby
+
+# Ruby build matrix
 rvm:
-  - "jruby-1.7.16"
-notifications:
-  email:
-    recipients:
-      - FutureMediaNewsRubyGems@bbc.co.uk
-    on_failure: change
-    on_success: never
-sudo: false
-script: bundle exec rake all
+  - 2.0
+  - 2.1
+  - 2.2
+  - 2.3.0
+  - 2.3.1
+  - jruby
+
+# Environment variables
+env:
+  - RUBYOPT="-W0"
+
+# Ensure we don't build for *every* commit (doesn't apply to PR builds)
+branches:
+  only:
+  - master
+
+script:
+  - bundle exec rspec --format documentation

data/README.md

@@ -28,7 +28,6 @@ The **Broker** is capable of retrieving rendered templates from either [S3](http
 
 ```ruby
 require 'alephant/broker'
-require 'alephant/broker/load_strategy/s3'
 
 config = {
   :s3_bucket_id      => 'test_bucket',
@@ -55,7 +54,6 @@ end
 
 ```ruby
 require 'alephant/broker'
-require 'alephant/broker/load_strategy/http'
 
 class UrlGenerator < Alephant::Broker::LoadStrategy::HTTP::URL
   def generate(id, options)
@@ -87,7 +85,7 @@ The HTML load strategy relies upon being given a URLGenerator, which is used to
 * include a `#generate` method which takes `id` (string) and `options` (hash) as parameters.
 
 ```ruby
-require 'alephant/broker/load_strategy/http'
+require 'alephant/broker'
 require 'rack'
 
 class UrlGenerator < Alephant::Broker::LoadStrategy::HTTP::URL
@@ -103,6 +101,40 @@ class UrlGenerator < Alephant::Broker::LoadStrategy::HTTP::URL
 end
 ```
 
+##### Revalidate Strategy
+
+```ruby
+require 'alephant/broker'
+
+config = {
+  :s3_bucket_id                => 'test_bucket',
+  :s3_object_path              => 'foo',
+  :lookup_table_name           => 'test_lookup',
+  :sqs_queue_name              => 'test_queue',
+  :elasticache_config_endpoint => 'test_elisticache'
+  :revalidate_cache_ttl        => 30
+}
+
+Alephant::Broker::Application.new(
+  Alephant::Broker::LoadStrategy::Revalidate::Strategy.new,
+  config
+).call(request).tap do |response|
+  puts "status:  #{response.status}"
+  puts "content: #{response.content}"
+end
+```
+
+The "Revalidate" strategy uses the following process flow:
+
+![https://raw.githubusercontent.com/BBC-News/Documentation/master/Alephant/Diagrams/Alephant%20Revalidate%20Broker.png?token=AADB7YskdW22P6z-hDjfk12UiR7bZA8Jks5XkI7EwA%3D%3D](https://raw.githubusercontent.com/BBC-News/Documentation/master/Alephant/Diagrams/Alephant%20Revalidate%20Broker.png?token=AADB7YskdW22P6z-hDjfk12UiR7bZA8Jks5XkI7EwA%3D%3D)
+
+And uses the following AWS resources:
+
+* S3 - for storage of rendered components
+* DynamoDB - for recording S3 component locations
+* Elasticache - for caching S3 content and recording "inflight" messages
+* SQS - for communication between the broker and renderer
+
 ### Rack App
 
 Create **config.ru** using example below, and then run:
@@ -144,6 +176,3 @@ This version is added as a header to the response in the following format:
 5. Create a new [Pull Request](https://github.com/BBC-News/alephant-broker/pulls).
 
 Feel free to create a new [issue](https://github.com/BBC-News/alephant-broker/issues/new) if you find a bug.
-
-
-

data/alephant-broker.gemspec

@@ -26,13 +26,14 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "pry-remote"
   spec.add_development_dependency "pry-nav"
   spec.add_development_dependency "rake-rspec", ">= 0.0.2"
-
   spec.add_development_dependency "bundler", "~> 1.5"
   spec.add_development_dependency "rake"
+  spec.add_development_dependency 'rack', '< 2.0.0' # NOTE: rack 2.0 requires Ruby 2.2+
   spec.add_development_dependency "rack-test"
   spec.add_development_dependency "simplecov"
+  spec.add_development_dependency "timecop"
 
-  spec.add_runtime_dependency "alephant-lookup"
+  spec.add_runtime_dependency "alephant-lookup", ">= 2.0.2"
   spec.add_runtime_dependency "alephant-storage", ">= 1.1.1"
   spec.add_runtime_dependency "alephant-logger", "1.2.0"
   spec.add_runtime_dependency 'alephant-sequencer'

data/lib/alephant/broker/cache.rb

@@ -3,6 +3,7 @@ module Alephant
     module Cache
       require "alephant/broker/cache/client"
       require "alephant/broker/cache/null_client"
+      require "alephant/broker/cache/cached_object"
     end
   end
 end

data/lib/alephant/broker/cache/cached_object.rb

@@ -0,0 +1,71 @@
+require 'alephant/logger'
+require 'time'
+
+module Alephant
+  module Broker
+    module Cache
+      class CachedObject
+        include Logger
+        attr_reader :s3_obj
+
+        DEFAULT_TTL = 10
+
+        def initialize(obj)
+          logger.info(event:   'SettingCachedObject',
+                      content: obj,
+                      method:  "#{self.class}#initialize")
+
+          @s3_obj = obj
+        end
+
+        def update(obj)
+          logger.info(event:       'UpdatingCachedObject',
+                      old_content: @s3_obj,
+                      new_content: obj,
+                      method:      "#{self.class}#update")
+
+          @s3_obj = obj
+        end
+
+        def updated
+          time = metadata[:'head_Last-Modified']
+          Time.parse(time)
+        rescue TypeError, ArgumentError => error
+          logger.info(event: 'ErrorCaught', method: "#{self.class}#updated", error: error)
+          Time.now
+        end
+
+        def ttl
+          Integer(metadata[:ttl] || metadata['ttl'])
+        rescue TypeError => error
+          logger.info(event: 'ErrorCaught', method: "#{self.class}#ttl", error: error)
+          Integer(Broker.config[:revalidate_cache_ttl] || DEFAULT_TTL)
+        end
+
+        def expired?
+          result = (updated + ttl) < Time.now
+
+          logger.info(event:            'Expired?',
+                      updated:          updated,
+                      ttl:              ttl,
+                      updated_plus_ttl: (updated + ttl),
+                      now:              Time.now,
+                      result:           result,
+                      method:           "#{self.class}#expired?")
+
+          result
+        end
+
+        def to_h(obj = nil)
+          obj || s3_obj
+        end
+
+        private
+
+        def metadata
+          s3_obj.fetch(:meta, {})
+        end
+      end
+    end
+  end
+end

data/lib/alephant/broker/cache/client.rb

@@ -1,5 +1,5 @@
-require "dalli-elasticache"
-require "alephant/logger"
+require 'dalli-elasticache'
+require 'alephant/logger'
 
 module Alephant
   module Broker
@@ -11,45 +11,57 @@ module Alephant
 
         def initialize
           if config_endpoint.nil?
-            logger.debug "Broker::Cache::Client#initialize: No config endpoint, NullClient used"
-            logger.metric "NoConfigEndpoint"
+            logger.debug 'Broker::Cache::Client#initialize: No config endpoint, NullClient used'
+            logger.metric 'NoConfigEndpoint'
             @client = NullClient.new
           else
-            @elasticache ||= ::Dalli::ElastiCache.new(config_endpoint, :expires_in => ttl)
+            @elasticache ||= ::Dalli::ElastiCache.new(config_endpoint, expires_in: ttl)
             @client ||= @elasticache.client
           end
         end
 
         def get(key)
-          versioned_key = versioned key
-          result = @client.get versioned_key
-          logger.info "Broker::Cache::Client#get key: #{versioned_key} - #{result ? 'hit' : 'miss'}"
-          logger.metric "GetKeyMiss" unless result
-          result ? result : set(key, block.call)
-        rescue StandardError => e
+          versioned_key = versioned(key)
+          result        = @client.get(versioned_key)
+
+          logger.info("Broker::Cache::Client#get key: #{versioned_key} - #{result ? 'hit' : 'miss'}")
+          logger.metric('GetKeyMiss') unless result
+
+          return result if result
+
+          set(key, yield) if block_given?
+        rescue StandardError => error
+          logger.info(event: 'ErrorCaught', method: "#{self.class}#get", error: error)
           yield if block_given?
         end
 
-        def set(key, value, ttl = nil)
-          value.tap { |o| @client.set(versioned(key), o, ttl) }
+        def set(key, value, custom_ttl = nil)
+          versioned_key = versioned(key)
+          set_ttl       = custom_ttl || ttl
+
+          logger.info("#{self.class}#set - key: #{versioned_key}, value: #{value}, ttl: #{set_ttl}")
+
+          @client.set(versioned_key, value, set_ttl)
+
+          value
         end
 
         private
 
         def config_endpoint
-          Broker.config["elasticache_config_endpoint"]
+          Broker.config[:elasticache_config_endpoint] || Broker.config['elasticache_config_endpoint']
         end
 
         def ttl
-          Broker.config["elasticache_ttl"] || DEFAULT_TTL
+          Broker.config[:elasticache_ttl] || Broker.config['elasticache_ttl'] || DEFAULT_TTL
         end
 
         def versioned(key)
-          [key, cache_version].compact.join("_")
+          [key, cache_version].compact.join('_')
         end
 
         def cache_version
-          Broker.config["elasticache_cache_version"]
+          Broker.config[:elasticache_cache_version] || Broker.config['elasticache_cache_version']
         end
       end
     end

data/lib/alephant/broker/cache/null_client.rb

@@ -2,12 +2,20 @@ module Alephant
   module Broker
     module Cache
       class NullClient
-        def get(_key)
-          yield
+        def initialize
+          @cache = {}
         end
 
-        def set(_key, value, _ttl = nil)
-          value
+        def get(key)
+          data = @cache[key]
+
+          return data if data
+
+          set(key, yield) if block_given?
+        end
+
+        def set(key, value, _ttl = nil)
+          @cache[key] = value
         end
       end
     end

data/lib/alephant/broker/component.rb

@@ -35,7 +35,9 @@ module Alephant
       end
 
       def status
-        data[:meta].key?("status") ? data[:meta]["status"] : 200
+        meta = data.fetch(:meta, {})
+
+        meta[:status] || meta['status'] || 200
       end
 
       private

data/lib/alephant/broker/component_meta.rb

@@ -17,6 +17,8 @@ module Alephant
         Crimp.signature options
       end
 
+      # NOTE: This is in use in `alephant-publisher-queue` also, so if you
+      #       change this, you'll need to change this there also.
       def component_key
         "#{id}/#{opts_hash}"
       end

data/lib/alephant/broker/error_component.rb

@@ -30,7 +30,7 @@ module Alephant
       end
 
       def debug?
-        Broker.config.fetch("debug", false)
+        Broker.config[:debug] || Broker.config["debug"] || false
       end
     end
   end

data/lib/alephant/broker/load_strategy.rb

@@ -4,6 +4,7 @@ module Alephant
       require "alephant/broker/load_strategy/http"
       require "alephant/broker/load_strategy/s3/archived"
       require "alephant/broker/load_strategy/s3/sequenced"
+      require "alephant/broker/load_strategy/revalidate/strategy"
     end
   end
 end

data/lib/alephant/broker/load_strategy/http.rb

@@ -23,7 +23,7 @@ module Alephant
           fetch_object(component_meta)
         rescue
           logger.metric "CacheMiss"
-          cache.set(component_meta.cache_key, content(component_meta))
+          cache.set(component_meta.component_key, content(component_meta))
         end
 
         private
@@ -35,7 +35,7 @@ module Alephant
         end
 
         def fetch_object(component_meta)
-          cache.get(component_meta.cache_key) { content component_meta }
+          cache.get(component_meta.component_key) { content component_meta }
         end
 
         def content(component_meta)
@@ -52,7 +52,6 @@ module Alephant
 
         def request(component_meta)
           before = Time.new
-          component_meta.cached = false
 
           Faraday.get(url_for(component_meta)).tap do |r|
             unless r.success?

data/lib/alephant/broker/load_strategy/revalidate/fetcher.rb

@@ -0,0 +1,51 @@
+require 'alephant/broker/errors'
+
+module Alephant
+  module Broker
+    module LoadStrategy
+      module Revalidate
+        class Fetcher
+          include Logger
+
+          attr_reader :component_meta
+
+          def initialize(component_meta)
+            @component_meta = component_meta
+          end
+
+          def fetch
+            Alephant::Broker::Cache::CachedObject.new(s3.get(s3_path))
+          rescue AWS::S3::Errors::NoSuchKey, InvalidCacheKey => error
+            logger.info(event: 'ErrorCaught', method: "#{self.class}#fetch", error: error)
+            logger.metric('S3InvalidCacheKey')
+            raise Alephant::Broker::Errors::ContentNotFound
+          end
+
+          private
+
+          def s3_path
+            lookup_read = lookup.read(component_meta.id, component_meta.options, 1)
+
+            raise InvalidCacheKey if lookup_read.location.nil?
+
+            lookup_read.location
+          end
+
+          def s3
+            @s3 ||= Alephant::Storage.new(
+              Broker.config[:s3_bucket_id],
+              Broker.config[:s3_object_path]
+            )
+          end
+
+          def lookup
+            @lookup ||= Alephant::Lookup.create(
+              Broker.config[:lookup_table_name],
+              Broker.config
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/alephant/broker/load_strategy/revalidate/refresher.rb

@@ -0,0 +1,75 @@
+module Alephant
+  module Broker
+    module LoadStrategy
+      module Revalidate
+        class Refresher
+          include Logger
+
+          INFLIGHT_CACHE_TTL = 120 # expire the inflight key after 2 minutes
+
+          attr_reader :component_meta
+
+          def initialize(component_meta)
+            @component_meta = component_meta
+          end
+
+          def refresh
+            inflight = cache.get(inflight_cache_key)
+
+            logger.info(event: 'Inflight?', cache_val: inflight, method: "#{self.class}#refresh")
+
+            return if inflight
+
+            logger.info(event: 'QueueMessage', message: message, method: "#{self.class}#refresh")
+
+            queue.send_message(message)
+            cache.set(inflight_cache_key, true, INFLIGHT_CACHE_TTL)
+          end
+
+          private
+
+          def message
+            ::JSON.generate(id:        component_meta.id,
+                            batch_id:  component_meta.batch_id,
+                            options:   component_meta.options,
+                            timestamp: Time.now.to_s)
+          end
+
+          def queue
+            @queue ||= proc do
+              client = AWS::SQS.new
+              url    = client.queues.url_for(Broker.config[:sqs_queue_name], queue_options)
+
+              client.queues[url]
+            end.call
+          end
+
+          def queue_options
+            opts = {}
+            opts[:queue_owner_aws_account_id] = aws_acc_id if aws_acc_id
+
+            logger.info(event: 'SQSQueueOptionsConfigured', options: opts, method: "#{self.class}#queue_options")
+
+            opts
+          end
+
+          def aws_acc_id
+            Broker.config[:aws_account_id]
+          end
+
+          def cache
+            @cache ||= Cache::Client.new
+          end
+
+          def cache_key
+            component_meta.component_key
+          end
+
+          def inflight_cache_key
+            "inflight-#{cache_key}"
+          end
+        end
+      end
+    end
+  end
+end