alephant-publisher-queue 2.3.1 → 2.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0d60cd615024dd56ba59fa1f6a316209cdc41b33
-  data.tar.gz: 20c7079768247abb47894bb1c63adde51a68e1ae
+  metadata.gz: e90e5d717583700bb627398ffec0f305383ab17b
+  data.tar.gz: 7f6fdf2a4159bb0276e512829bb9d8d4788775e5
 SHA512:
-  metadata.gz: 53244a4df986d2819a3283af1db929c1437b68e35d79ed3d4743f9414b4ef8b7113ac2784da905e8a75491d58c917149bc12926f86d003bd1ad0cc9011745872
-  data.tar.gz: db98c4ca0561ac8abc2b35be49592325b95048d65a7636968f41153e8f2c75a5476d20864145af27e95c2caf5af9aa071412c6453dcc632bfea650fa3c6d02dc
+  metadata.gz: acdc9518ef0f946002f0e6b07a8cb122076e6d9661b37fcee23339bdc221226b01c9d7300d936879f0fd919591701ce8749bcaf451217eb97c0532f6781f4bd6
+  data.tar.gz: cf959951bd12ec476daf6e473560b0c73ac52a3d6eb29f9ce21419af704298cae47a861eac42e66a2f6f96776d4e168c7c134ac519f768c08ab970810ac62b93

data/.travis.yml

@@ -1,9 +1,24 @@
+sudo: false
+cache: bundler
 language: ruby
+
+# Ruby build matrix
 rvm:
-  - "jruby"
-notifications:
-  email:
-    recipients:
-      - kenoir@gmail.com
-    on_failure: change
-    on_success: never
+  - 2.0
+  - 2.1
+  - 2.2
+  - 2.3.0
+  - 2.3.1
+  - jruby
+
+# Environment variables
+env:
+  - RUBYOPT="-W0"
+
+# Ensure we don't build for *every* commit (doesn't apply to PR builds)
+branches:
+  only:
+  - master
+
+script:
+  - bundle exec rspec --format documentation

data/README.md

@@ -11,6 +11,7 @@ Static publishing to S3 based on SQS messages.
   - S3 bucket.
   - SQS Queue.
   - Dynamo DB table.
+  - Elasticache (if using the "revalidate" pattern)
 
 ## Migrating from [Alephant::Publisher](https://github.com/BBC-News/alephant-publisher)
 
@@ -91,7 +92,7 @@ end
 {{ content }}
 ```
 
-## Usage
+## Usage (standard setup - non-revalidate)
 
 ```ruby
 require "alephant/logger"
@@ -101,9 +102,9 @@ module MyApp
   def self.run!
     loop do
       Alephant::Publisher::Queue.create(options).run!
-    rescue => e
-      Alephant::Logger.get_logger.warn "Error: #{e.message}"
     end
+  rescue => e
+    Alephant::Logger.get_logger.error "Error: #{e.message}"
   end
 
   private
@@ -150,6 +151,111 @@ S3 Path:
 S3 / bucket-id / example-s3-path / renderer-id / foo / 7e0c33c476b1089500d5f172102ec03e / 1
 ```
 
+## Usage (revalidate pattern)
+
+```ruby
+require "addressable/uri"
+require "alephant/logger"
+require "alephant/publisher/queue"
+
+module MyApp
+  class UrlGenerator
+    class << self
+      # This function is called to generate the URL to be requested as
+      # part of the rendering process. The return must be a URL as a string.
+      def generate(opts)
+        "http://example.com/?#{url_params(opts)}"
+      end
+
+      private
+
+      def url_params(params_hash)
+        uri = Addressable::URI.new
+        uri.query_values = params_hash
+        uri.query
+      end
+    end
+  end
+
+  class HttpResponseProcessor
+    class << self
+      # This function is called upon a successful HTTP response.
+      #
+      # Use it to modify or process the response of your HTTP request
+      # as you please, but there is one rule - the return value MUST
+      # be a JSON object.
+      def process(opts, status, body)
+        # our response is already JSON, pass it through
+        body
+      end
+
+      # If you wish to vary your revalidate TTL on a per-endpoint (or
+      # other logic) basis, you can do it here - simply return an Integer
+      # value.
+      #
+      # If nil is returned the 'revalidate_cache_ttl' config setting on the
+      # broker will be used as the default TTL, otherwise the default in
+      # 'alephant-broker' will be used.
+      def self.ttl(opts)
+        # 30s revalidate time for all
+        30
+      end
+    end
+  end
+
+  def self.run!
+    loop do
+      Alephant::Publisher::Queue.create(options, processor).run!
+    end
+  rescue => e
+    Alephant::Logger.get_logger.error "Error: #{e.message}"
+  end
+
+  private
+
+  def self.processor
+    Alephant::Publisher::Queue::RevalidateProcessor.new(options, UrlGenerator, HttpResponseProcessor)
+  end
+
+  def self.options
+    Alephant::Publisher::Queue::Options.new.tap do |opts|
+      opts.add_queue(
+        :aws_account_id => 'example',
+        :sqs_queue_name => 'test_queue'
+      )
+      opts.add_writer(
+        :lookup_table_name    => 'lookup-dynamo-table',
+        :s3_bucket_id         => 'bucket-id',
+        :s3_object_path       => 'example-s3-path',
+        :view_path            => 'path/to/views'
+      )
+      opts.add_cache(
+        :elasticache_config_endpoint => 'example',
+        :elasticache_cache_version   => '100',
+        :revalidate_cache_ttl        => '30'
+      )
+    end
+  end
+end
+```
+
+Add a message to your SQS queue, with the following format (`id`, `batch_id`, `options`):
+
+```json
+{
+  "id": "renderer_id",
+  "batch_id": null,
+  "options": {
+    "id": "foo",
+    "type": "chart"
+  }
+}
+```
+
+This will then make a HTTP GET request to the configured endpoint (via `UrlGenerator`), process the response (via `HttpResponseProcessor`), render and store your content.
+
+You will not ordinarily need to push messages onto SQS manually, this will be handled via the broker in real use.
+
 ## Preview Server
 
 [Alephant Preview](https://github.com/BBC-News/alephant-preview) allows you to see the HTML generated by your templates, both standalone and in the context of a page.

data/alephant-publisher-queue.gemspec

@@ -32,6 +32,8 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency "rake"
   spec.add_runtime_dependency "aws-sdk", "~> 1.0"
   spec.add_runtime_dependency "crimp"
+  spec.add_runtime_dependency "faraday"
+  spec.add_runtime_dependency "dalli-elasticache"
   spec.add_runtime_dependency "alephant-support"
   spec.add_runtime_dependency "alephant-sequencer", "~> 3"
   spec.add_runtime_dependency "alephant-cache"

data/lib/alephant/publisher/queue.rb

@@ -2,9 +2,11 @@ require_relative "env"
 
 require "alephant/publisher/queue/version"
 require "alephant/publisher/queue/options"
+require "alephant/publisher/queue/publisher"
 require "alephant/publisher/queue/sqs_helper/queue"
 require "alephant/publisher/queue/sqs_helper/archiver"
 require "alephant/publisher/queue/processor"
+require "alephant/publisher/queue/revalidate_processor"
 require "alephant/logger"
 require "alephant/cache"
 require "json"
@@ -12,94 +14,10 @@ require "json"
 module Alephant
   module Publisher
     module Queue
-      def self.create(opts = {}, processor = nil)
-        processor ||= Processor.new(opts.writer)
+      def self.create(opts, processor = nil)
+        processor ||= Processor.new(opts)
         Publisher.new(opts, processor)
       end
-
-      class Publisher
-        include Alephant::Logger
-
-        VISIBILITY_TIMEOUT = 60
-        RECEIVE_WAIT_TIME  = 15
-
-        attr_reader :queue, :executor, :opts, :processor
-
-        def initialize(opts, processor = nil)
-          @opts = opts
-          @processor = processor
-
-          @queue = Alephant::Publisher::Queue::SQSHelper::Queue.new(
-            aws_queue,
-            archiver,
-            opts.queue[:visibility_timeout] || VISIBILITY_TIMEOUT,
-            opts.queue[:receive_wait_time]  || RECEIVE_WAIT_TIME
-          )
-        end
-
-        def run!
-          loop { processor.consume(@queue.message) }
-        end
-
-        private
-
-        def archiver
-          Alephant::Publisher::Queue::SQSHelper::Archiver.new(archive_cache, archiver_opts)
-        end
-
-        def archiver_opts
-          options = {
-            :async_store         => true,
-            :log_archive_message => true,
-            :log_validator       => opts.queue[:log_validator]
-          }
-          options.each do |key, _value|
-            options[key] = opts.queue[key] == "true" if whitelist_key(opts.queue, key)
-          end
-        end
-
-        def whitelist_key(options, key)
-          options.key?(key) && key != :log_validator
-        end
-
-        def archive_cache
-          Alephant::Cache.new(
-            opts.writer[:s3_bucket_id],
-            opts.writer[:s3_object_path]
-          )
-        end
-
-        def get_region
-          opts.queue[:sqs_account_region] || AWS.config.region
-        end
-
-        def sqs_client
-          @sqs_client ||= AWS::SQS.new(:region => get_region)
-        end
-
-        def sqs_queue_options
-          (opts.queue[:aws_account_id].nil? ? {} : fallback).tap do |ops|
-            logger.info(
-              "event"   => "SQSQueueOptionsConfigured",
-              "options" => ops,
-              "method"  => "#{self.class}#sqs_queue_options"
-            )
-          end
-        end
-
-        def fallback
-          {
-            :queue_owner_aws_account_id => opts.queue[:aws_account_id]
-          }
-        end
-
-        def aws_queue
-          queue_url = sqs_client.queues.url_for(
-            opts.queue[:sqs_queue_name], sqs_queue_options
-          )
-          sqs_client.queues[queue_url]
-        end
-      end
     end
   end
 end

data/lib/alephant/publisher/queue/options.rb

@@ -9,7 +9,7 @@ module Alephant
       class Options
         include Alephant::Logger
 
-        attr_reader :queue, :writer
+        attr_reader :queue, :writer, :cache
 
         QUEUE_OPTS = [
           :receive_wait_time,
@@ -20,7 +20,7 @@ module Alephant
           :log_archive_message,
           :log_validator,
           :async_store
-        ]
+        ].freeze
 
         WRITER_OPTS = [
           :lookup_table_name,
@@ -31,38 +31,47 @@ module Alephant
           :sequence_id_path,
           :sequencer_table_name,
           :view_path
-        ]
+        ].freeze
+
+        CACHE_OPTS = [
+          :elasticache_config_endpoint,
+          :elasticache_cache_version,
+          :revalidate_cache_ttl
+        ].freeze
 
         def initialize
           @queue  = {}
           @writer = {}
+          @cache  = {}
         end
 
         def add_queue(opts)
-          execute @queue, QUEUE_OPTS, opts
+          execute(@queue, QUEUE_OPTS, opts)
         end
 
         def add_writer(opts)
-          execute @writer, WRITER_OPTS, opts
+          execute(@writer, WRITER_OPTS, opts)
+        end
+
+        def add_cache(opts)
+          execute(@cache, CACHE_OPTS, opts)
         end
 
         private
 
         def execute(instance, type, opts)
-          begin
-            validate type, opts
-            instance.merge! opts
-          rescue InvalidKeySpecifiedError => e
-            logger.metric "QueueOptionsInvalidKeySpecified"
-            logger.error(
-              "event"     => "QueueOptionsKeyInvalid",
-              "class"     => e.class,
-              "message"   => e.message,
-              "backtrace" => e.backtrace.join.to_s,
-              "method"    => "#{self.class}#validate"
-            )
-            puts e.message
-          end
+          validate(type, opts)
+          instance.merge!(opts)
+        rescue InvalidKeySpecifiedError => e
+          logger.metric "QueueOptionsInvalidKeySpecified"
+          logger.error(
+            "event"     => "QueueOptionsKeyInvalid",
+            "class"     => e.class,
+            "message"   => e.message,
+            "backtrace" => e.backtrace.join.to_s,
+            "method"    => "#{self.class}#validate"
+          )
+          puts e.message
         end
 
         def validate(type, opts)

data/lib/alephant/publisher/queue/processor.rb

@@ -1,25 +1,27 @@
 require "alephant/publisher/queue/writer"
-require "alephant/publisher/queue/base_processor"
 
 module Alephant
   module Publisher
     module Queue
-      class Processor < BaseProcessor
-        attr_reader :writer_config
+      class Processor
+        attr_reader :opts
 
-        def initialize(writer_config = {})
-          @writer_config = writer_config
+        def initialize(opts = nil)
+          @opts = opts
         end
 
         def consume(msg)
-          unless msg.nil?
-            write msg
-            msg.delete
-          end
+          return if msg.nil?
+          write(msg)
+          msg.delete
         end
 
         private
 
+        def writer_config
+          opts ? opts.writer : {}
+        end
+
         def write(msg)
           Writer.new(writer_config, msg).run!
         end

data/lib/alephant/publisher/queue/publisher.rb

@@ -0,0 +1,89 @@
+module Alephant
+  module Publisher
+    module Queue
+      class Publisher
+        include Alephant::Logger
+
+        VISIBILITY_TIMEOUT = 60
+        RECEIVE_WAIT_TIME  = 15
+
+        attr_reader :queue, :executor, :opts, :processor
+
+        def initialize(opts, processor = nil)
+          @opts = opts
+          @processor = processor
+
+          @queue = Alephant::Publisher::Queue::SQSHelper::Queue.new(
+            aws_queue,
+            archiver,
+            opts.queue[:visibility_timeout] || VISIBILITY_TIMEOUT,
+            opts.queue[:receive_wait_time]  || RECEIVE_WAIT_TIME
+          )
+        end
+
+        def run!
+          loop { processor.consume(@queue.message) }
+        end
+
+        private
+
+        def archiver
+          Alephant::Publisher::Queue::SQSHelper::Archiver.new(archive_cache, archiver_opts)
+        end
+
+        def archiver_opts
+          options = {
+            :async_store         => true,
+            :log_archive_message => true,
+            :log_validator       => opts.queue[:log_validator]
+          }
+          options.each do |key, _value|
+            options[key] = opts.queue[key] == "true" if whitelist_key(opts.queue, key)
+          end
+        end
+
+        def whitelist_key(options, key)
+          options.key?(key) && key != :log_validator
+        end
+
+        def archive_cache
+          Alephant::Cache.new(
+            opts.writer[:s3_bucket_id],
+            opts.writer[:s3_object_path]
+          )
+        end
+
+        def get_region
+          opts.queue[:sqs_account_region] || AWS.config.region
+        end
+
+        def sqs_client
+          @sqs_client ||= AWS::SQS.new(region: get_region)
+        end
+
+        def sqs_queue_options
+          (opts.queue[:aws_account_id].nil? ? {} : fallback).tap do |ops|
+            logger.info(
+              "event"   => "SQSQueueOptionsConfigured",
+              "options" => ops,
+              "method"  => "#{self.class}#sqs_queue_options"
+            )
+          end
+        end
+
+        def fallback
+          {
+            :queue_owner_aws_account_id => opts.queue[:aws_account_id]
+          }
+        end
+
+        def aws_queue
+          queue_url = sqs_client.queues.url_for(
+            opts.queue[:sqs_queue_name], sqs_queue_options
+          )
+          sqs_client.queues[queue_url]
+        end
+      end
+    end
+  end
+end