alephant-publisher-queue 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: b72175c8d5c49896aef2c2bc39cd254ddf0a9dc6
+  data.tar.gz: b1ccc179c4ff8eb62bd5cbe9bc8b2d95d4e6fc1b
+SHA512:
+  metadata.gz: 740f828042a9107e2ab146f1ff1c2a1e43ac78aa18b147f5ad28ce33acae36bd4563ef84a937f911956a8ff26140bf019022fe75b0429d16c31931f91afed691
+  data.tar.gz: e6c7923dad564f59638387c5f05a330ad2d321928b069cbc63e0a845f99ba7c21fe64416c76246a657de8d8c4ffe7b0a78f37a4a3eede2fd2b9ae31c6dbd7568

data/.gitignore

@@ -0,0 +1,23 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+.rspec
+.ruby-version
+/config/*.yaml
+/config/*.yml
+/components
+*.swp

data/.travis.yml

@@ -0,0 +1,9 @@
+language: ruby
+rvm:
+  - "jruby"
+notifications:
+  email:
+    recipients:
+      - kenoir@gmail.com
+    on_failure: change
+    on_success: never

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in alephant-publishing.gemspec
+gemspec

data/Guardfile

@@ -0,0 +1,6 @@
+guard 'rspec' do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/.+\.rb$})
+  watch(%r{^spec/support/(.+)\.rb$})                  { "spec" }
+  watch('spec/spec_helper.rb')                        { "spec" }
+end

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Integralist
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,226 @@
+# Alephant::Publisher::Queue
+
+Static publishing to S3 based on SQS messages.
+
+[![Build Status](https://travis-ci.org/BBC-News/alephant-publisher-queue.png?branch=master)](https://travis-ci.org/BBC-News/alephant-publisher-queue) [![Dependency Status](https://gemnasium.com/BBC-News/alephant-publisher-queue.png)](https://gemnasium.com/BBC-News/alephant-publisher-queue) [![Gem Version](https://badge.fury.io/rb/alephant-publisher-queue.png)](http://badge.fury.io/rb/alephant-publisher-queue)
+
+## Dependencies
+
+- JRuby 1.7.8
+- An AWS account (you'll need to create):
+  - An S3 bucket
+  - An SQS Queue (if no sequence id provided then `sequence_id` will be used)
+  - A Dynamo DB table (optional, will attempt to create if can't be found)
+
+## Migrating from [Alephant::Publisher](https://github.com/BBC-News/alephant-publisher)
+
+1. Add the new gem in your Gemfile (`gem 'alephant-publisher-queue'`).
+2. Run `bundle install`.
+3. Require the new gem in your app (`require 'alephant/publisher/queue'`).
+4. Note that the namespace has changed from `Alephant::Publisher` to `Alephant::Publisher::Queue`.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'alephant-publisher-queue'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install alephant-publisher-queue
+
+## Setup
+
+Ensure you have a `config/aws.yml` in the format:
+
+```yaml
+access_key_id: ACCESS_KEY_ID
+secret_access_key: SECRET_ACCESS_KEY
+```
+
+## Usage
+
+**In your application:**
+
+```rb
+require 'alephant'
+
+sequential_proc = Proc.new do |last_seen_id, data|
+  last_seen_id < data['sequence_id'].to_i
+end
+
+set_last_seen_proc = Proc.new do |data|
+  data['sequence_id'].to_i
+end
+
+opts = {
+  :s3_bucket_id         => 'bucket-id',
+  :s3_object_path       => 'path/to/object',
+  :s3_object_id         => 'object_id',
+  :sequencer_table_name => 'your_dynamo_db_table',
+  :sqs_queue_url        => 'https://your_amazon_sqs_queue_url',
+  :sequential_proc      => sequential_proc,
+  :set_last_seen_proc   => set_last_seen_proc,
+  :lookup_table_name    => 'your_lookup_table'
+}
+
+logger = Logger.new
+
+thread = Alephant::Alephant.new(opts, logger).run!
+thread.join
+```
+
+Publisher requires both queue options and writer options to be provided. To ensure a standard format you should use the `Options` class to generate your options before passing them onto the Publisher...
+
+```ruby
+opts = Alephant::Publisher::Options.new
+# => #<Alephant::Publisher::Options:0x0602f958 @queue={}, @writer={}>
+
+opts.queue
+# => {}
+# empty to start with
+
+opts.writer
+# => {}
+# empty to start with
+
+opts.add_queue(:foo => "bar")
+# The key 'foo' is invalid
+# => nil
+
+opts.queue
+# => {}
+# still empty as the foo key was invalid
+
+opts.add_queue(:sqs_queue_url => "bar")
+# => {:sqs_queue_url=>"bar"}
+
+opts.queue
+# => {:sqs_queue_url=>"bar"}
+
+opts.add_writer(:sqs_queue_url => "bar")
+# The key 'sqs_queue_url' is invalid
+# => nil
+# the sqs_queue_url key was valid for the queue options,
+# but is invalid when trying to add it to the writer options
+
+opts.add_writer(:msg_vary_id_path => "bar")
+=> {:msg_vary_id_path=>"bar"}
+
+opts.writer
+=> {:msg_vary_id_path=>"bar"}
+```
+
+logger is optional, and must confirm to the Ruby standard logger interface
+
+Provide a view in a folder (fixtures are optional):
+
+```
+└── views
+    ├── models
+    │   └── foo.rb
+    ├── fixtures
+    │   └── foo.json
+    └── templates
+        └── foo.mustache
+```
+
+**SQS Message Format**
+
+```json
+{
+  "content": "hello world",
+  "sequential_id": 1
+}
+```
+
+**foo.rb**
+
+```rb
+module MyApp
+  module Views
+    class Foo < Alephant::Views::Base
+      def content
+        @data['content']
+      end
+    end
+  end
+end
+```
+
+**foo.mustache**
+
+```mustache
+{{content}}
+```
+
+**S3 Output**
+
+```
+hello world
+```
+
+## Build the gem locally
+
+If you want to test a modified version of the gem within your application without publishing it then you can follow these steps...
+
+- `gem uninstall alephant-publisher-queue`
+- `gem build alephant-publisher-queue.gemspec` (this will report the file generated which you reference in the next command)
+- `gem install ./alephant-publisher-queue-0.0.1.gem`
+
+Now you can test the gem from within your application as you've installed the gem from the local version rather than your published version
+
+## Preview Server
+
+`alephant preview`
+
+The included preview server allows you to see the html generated by your
+templates, both standalone and in the context of a page.
+
+**Standalone**
+
+`/component/:id/?:fixture?`
+
+### Full page preview
+
+When viewing the component in the context of a page, you'll need to retrieve a
+mustache template to provide the page context.
+
+When performing an update a regex is applied to replace the static hostnames in
+the retrieved html.
+
+**Environment Variables**
+
+```sh
+STATIC_HOST_REGEX="static.(sandbox.dev|int|test|stage|live).yourapp(i)?.com\/"
+PREVIEW_TEMPLATE_URL="http://yourapp.com/template"
+```
+
+**Example Remote Template**
+
+`id` is the component/folder name  
+
+`template` is the mustache template file name  
+
+`location_in_page` should be something like (for example) `page_head` (specified within a `preview.mustache` file that the consuming application needs to create).
+
+- `http://localhost:4567/component/id/template`
+- `http://localhost:4567/preview/id/template/location_in_page`
+
+`alephant update`
+
+**In page**
+
+`/preview/:id/:region/?:fixture?`
+
+## Contributing
+
+1. [Fork it!](http://github.com/BBC-News/alephant-publisher-queue/fork)
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,9 @@
+$:.unshift File.join(File.dirname(__FILE__), 'lib')
+
+require 'rspec/core/rake_task'
+require 'bundler/gem_tasks'
+require 'alephant/publisher/queue'
+
+require 'rake/rspec'
+
+task :default => :spec

data/alephant-publisher-queue.gemspec

@@ -0,0 +1,40 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'alephant/publisher/queue/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "alephant-publisher-queue"
+  spec.version       = Alephant::Publisher::Queue::VERSION
+  spec.authors       = ["revett"]
+  spec.email         = ["charlierevett@gmail.com"]
+  spec.summary       = "Static publishing to S3 based on SQS messages"
+  spec.description   = "Static publishing to S3 based on SQS messages"
+  spec.homepage      = "https://github.com/BBC-News/alephant-publisher-queue"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.5"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "rspec-nc"
+  spec.add_development_dependency "guard"
+  spec.add_development_dependency "guard-rspec"
+  spec.add_development_dependency "pry"
+  spec.add_development_dependency "pry-remote"
+  spec.add_development_dependency "pry-nav"
+  spec.add_development_dependency 'rake-rspec'
+
+  spec.add_runtime_dependency 'rake'
+  spec.add_runtime_dependency 'aws-sdk', '~> 1.0'
+  spec.add_runtime_dependency 'crimp'
+  spec.add_runtime_dependency 'alephant-support'
+  spec.add_runtime_dependency 'alephant-sequencer'
+  spec.add_runtime_dependency 'alephant-cache'
+  spec.add_runtime_dependency 'alephant-logger'
+  spec.add_runtime_dependency 'alephant-lookup'
+  spec.add_runtime_dependency 'alephant-renderer', '~> 0.1.0'
+end

data/lib/alephant/publisher/env.rb

@@ -0,0 +1,11 @@
+require 'aws-sdk'
+require 'yaml'
+
+config_file = 'config/aws.yaml'
+
+AWS.eager_autoload!
+
+if File.exists? config_file
+  config = YAML.load(File.read(config_file))
+  AWS.config(config)
+end

data/lib/alephant/publisher/queue.rb

@@ -0,0 +1,72 @@
+require_relative 'env'
+
+require 'alephant/publisher/queue/version'
+require 'alephant/publisher/queue/options'
+require 'alephant/publisher/queue/sqs_helper/queue'
+require 'alephant/publisher/queue/sqs_helper/archiver'
+require 'alephant/logger'
+require 'alephant/publisher/queue/processor'
+require 'json'
+
+module Alephant
+  module Publisher
+    module Queue
+      include Logger
+
+      def self.create(opts = {}, processor = nil)
+        processor ||= Processor.new(opts.writer)
+        Publisher.new(opts, processor)
+      end
+
+      class Publisher
+        VISIBILITY_TIMEOUT = 60
+        RECEIVE_WAIT_TIME  = 15
+
+        attr_reader :queue, :executor, :opts, :processor
+
+        def initialize(opts, processor = nil)
+          @opts = opts
+          @processor = processor
+
+          @queue = SQSHelper::Queue.new(
+            aws_queue,
+            archiver,
+            opts.queue[:visibility_timeout] || VISIBILITY_TIMEOUT,
+            opts.queue[:receive_wait_time]  || RECEIVE_WAIT_TIME,
+          )
+        end
+
+        def run!
+          loop { processor.consume(@queue.message) }
+        end
+
+        private
+
+        def archiver
+          SQSHelper::Archiver.new(archive_cache)
+        end
+
+        def archive_cache
+          Cache.new(
+            opts.writer[:s3_bucket_id],
+            opts.writer[:s3_object_path]
+          )
+        end
+
+        def sqs_client
+          @sqs_client ||= AWS::SQS.new
+        end
+
+        def sqs_queue_options
+          opts.queue[:aws_account_id].nil? ? {} : { :queue_owner_aws_account_id => opts.queue[:aws_account_id] }
+        end
+
+        def aws_queue
+          queue_url = sqs_client.queues.url_for(opts.queue[:sqs_queue_name], sqs_queue_options)
+          sqs_client.queues[queue_url]
+        end
+      end
+    end
+  end
+end
+

data/lib/alephant/publisher/queue/options.rb

@@ -0,0 +1,62 @@
+require 'aws-sdk'
+require 'alephant/logger'
+
+module Alephant
+  module Publisher
+    module Queue
+      class InvalidKeySpecifiedError < StandardError; end
+
+      class Options
+        attr_reader :queue, :writer
+
+        QUEUE_OPTS = [
+          :receive_wait_time,
+          :sqs_queue_name,
+          :visibility_timeout,
+          :aws_account_id
+        ]
+
+        WRITER_OPTS = [
+          :lookup_table_name,
+          :msg_vary_id_path,
+          :renderer_id,
+          :s3_bucket_id,
+          :s3_object_path,
+          :sequence_id_path,
+          :sequencer_table_name,
+          :view_path
+        ]
+
+        def initialize
+          @queue  = {}
+          @writer = {}
+        end
+
+        def add_queue(opts)
+          execute @queue, QUEUE_OPTS, opts
+        end
+
+        def add_writer(opts)
+          execute @writer, WRITER_OPTS, opts
+        end
+
+        private
+
+        def execute(instance, type, opts)
+          begin
+            validate type, opts
+            instance.merge! opts
+          rescue Exception => e
+            puts e.message
+          end
+        end
+
+        def validate(type, opts)
+          opts.each do |key, value|
+            raise InvalidKeySpecifiedError, "The key '#{key}' is invalid" unless type.include? key.to_sym
+          end
+        end
+      end
+    end
+  end
+end

data/lib/alephant/publisher/queue/processor.rb

@@ -0,0 +1,29 @@
+require 'alephant/publisher/queue/writer'
+require 'alephant/publisher/queue/processor/base'
+
+module Alephant
+  module Publisher
+    module Queue
+      class Processor < BaseProcessor
+        attr_reader :writer_config
+
+        def initialize(writer_config = {})
+          @writer_config = writer_config
+        end
+
+        def consume(msg)
+          unless msg.nil?
+            write msg
+            msg.delete
+          end
+        end
+
+        private
+
+        def write(msg)
+          Writer.new(writer_config, msg).run!
+        end
+      end
+    end
+  end
+end

data/lib/alephant/publisher/queue/processor/base.rb

@@ -0,0 +1,13 @@
+module Alephant
+  module Publisher
+    module Queue
+      class BaseProcessor
+        
+        def consume(msg)
+          raise NotImplementedError.new("You must implement the #consume(msg) method")
+        end
+
+      end
+    end
+  end
+end

data/lib/alephant/publisher/queue/sqs_helper/archiver.rb

@@ -0,0 +1,47 @@
+require 'date'
+
+module Alephant
+  module Publisher
+    module Queue
+      module SQSHelper
+        class Archiver
+          attr_reader :cache, :async
+
+          def initialize(cache, async = true)
+            @async = async
+            @cache = cache
+          end
+
+          def see(message)
+            return if message.nil?
+            message.tap { |m| async ? async_store(m) : store(m) }
+          end
+
+          private
+
+          def async_store(m)
+            Thread.new { store(m) }
+          end
+
+          def store(m)
+            cache.put("archive/#{date_key}/#{m.id}", m.body, meta_for(m))
+          end
+
+          def date_key
+            DateTime.now.strftime('%d-%m-%Y_%H')
+          end
+
+          def meta_for(m)
+            {
+              :id                => m.id,
+              :md5               => m.md5,
+              :logged_at         => DateTime.now.to_s,
+              :queue             => m.queue.url,
+            }
+          end
+        end
+      end
+    end
+  end
+end
+

data/lib/alephant/publisher/queue/sqs_helper/queue.rb

@@ -0,0 +1,53 @@
+require 'aws-sdk'
+require 'alephant/logger'
+
+module Alephant
+  module Publisher
+    module Queue
+      module SQSHelper
+        class Queue
+          WAIT_TIME = 5
+          VISABILITY_TIMEOUT = 300
+
+          include Logger
+
+          attr_reader :queue, :timeout, :wait_time, :archiver
+
+          def initialize(queue, archiver = nil, timeout = VISABILITY_TIMEOUT, wait_time = WAIT_TIME)
+            @queue     = queue
+            @archiver  = archiver
+            @timeout   = timeout
+            @wait_time = wait_time
+
+            logger.debug("Queue#initialize: reading from #{queue.url}")
+          end
+
+          def message
+            receive.tap { |m| process(m) unless m.nil? }
+          end
+
+          private
+
+          def process(m)
+            logger.info("Queue#message: received #{m.id}")
+            archive m
+          end
+
+          def archive(m)
+            archiver.see(m) unless archiver.nil?
+          rescue StandardError => e
+            logger.warn("Queue#archive: archive failed (#{e.message})");
+          end
+
+          def receive
+            queue.receive_message({
+              :visibility_timeout => timeout,
+              :wait_time_seconds  => wait_time
+            })
+          end
+        end
+      end
+    end
+  end
+end
+

data/lib/alephant/publisher/queue/version.rb

@@ -0,0 +1,7 @@
+module Alephant
+  module Publisher
+    module Queue
+      VERSION = "0.0.1"
+    end
+  end
+end

data/lib/alephant/publisher/queue/writer.rb

@@ -0,0 +1,109 @@
+require 'crimp'
+
+require 'alephant/cache'
+require 'alephant/lookup'
+require 'alephant/logger'
+require 'alephant/sequencer'
+require 'alephant/support/parser'
+require 'alephant/renderer'
+
+module Alephant
+  module Publisher
+    module Queue
+      class Writer
+        include Logger
+
+        attr_reader :config, :message, :cache, :parser, :renderer
+
+        def initialize(config, message)
+          @config   = config
+          @message  = message
+          @renderer = Alephant::Renderer.create(config, data)
+        end
+
+        def cache
+          Cache.new(
+            config[:s3_bucket_id],
+            config[:s3_object_path]
+          )
+        end
+
+        def parser
+          @parser ||= Support::Parser.new(
+            config[:msg_vary_id_path]
+          )
+        end
+
+        def run!
+          batch? ? batch.sequence(message, &perform) : perform.call
+        end
+
+        protected
+
+        def perform
+          Proc.new { views.each { |id, view| write(id, view) } }
+        end
+
+        def write(id, view)
+          seq_for(id).sequence(message) do
+            store(id, view, location_for(id))
+          end
+        end
+
+        def store(id, view, location)
+          cache.put(location, view.render, view.content_type, :msg_id => message.id)
+          lookup.write(id, options, seq_id, location)
+        end
+
+        def location_for(id)
+          "#{config[:renderer_id]}/#{id}/#{opt_hash}/#{seq_id}"
+        end
+
+        def batch
+          @batch ||= (views.count > 1) ? seq_for(config[:renderer_id]) : nil
+        end
+
+        def batch?
+          !batch.nil?
+        end
+
+        def seq_for(id)
+          Sequencer.create(
+            config[:sequencer_table_name],
+            seq_key_from(id),
+            config[:sequence_id_path],
+            config[:keep_all_messages] == 'true'
+          )
+        end
+
+        def seq_key_from(id)
+          "#{id}/#{opt_hash}"
+        end
+
+        def seq_id
+          @seq_id ||= Sequencer::Sequencer.sequence_id_from(message, config[:sequence_id_path])
+        end
+
+        def views
+          @views ||= renderer.views
+        end
+
+        def opt_hash
+          @opt_hash ||= Crimp.signature(options)
+        end
+
+        def options
+          @options ||= data[:options]
+        end
+
+        def data
+          @data ||= parser.parse(message)
+        end
+
+        def lookup
+          Lookup.create(config[:lookup_table_name])
+        end
+      end
+    end
+  end
+end

data/spec/archiver_spec.rb

@@ -0,0 +1,33 @@
+require 'spec_helper'
+
+describe Alephant::Publisher::Queue::SQSHelper::Archiver do
+  describe "#see" do
+    it "calls cache put with the correct params" do
+      time_now = DateTime.parse("Feb 24 1981")
+      allow(DateTime).to receive(:now).and_return(time_now)
+
+      q = double("queue").as_null_object
+      c = double("cache").as_null_object
+
+      expect(q).to receive(:url).and_return('url')
+
+      m = Struct.new(:id, :body, :md5, :queue).new('id', 'body', 'md5', q)
+
+      expect(c).to receive(:put).with(
+        "archive/#{time_now.strftime('%d-%m-%Y_%H')}/id",
+        "body",
+        {
+          :id        => "id",
+          :md5       => "md5",
+          :logged_at => time_now.to_s,
+          :queue     => "url"
+        }
+      )
+
+      instance = Alephant::Publisher::Queue::SQSHelper::Archiver.new(c, false)
+
+      instance.see(m)
+    end
+  end
+end
+

data/spec/processor_spec.rb

@@ -0,0 +1,19 @@
+require 'spec_helper'
+
+describe Alephant::Publisher::Queue::Processor do
+
+  before(:each) do
+    allow_any_instance_of(Alephant::Publisher::Queue::Writer).to receive(:initialize)
+    allow_any_instance_of(Alephant::Publisher::Queue::Writer).to receive(:run!)
+  end
+
+  describe "#consume(msg)" do
+    it "Consume the message and deletes it" do
+
+      msg = double('AWS::SQS::ReceivedMessage', :delete => nil)
+      expect(msg).to receive(:delete)
+      subject.consume(msg)
+
+    end
+  end
+end

data/spec/publisher_spec.rb

@@ -0,0 +1,41 @@
+require 'spec_helper'
+
+describe Alephant::Publisher::Queue do
+  let(:options)       { Alephant::Publisher::Queue::Options.new }
+  let(:queue)         { double('AWS::SQS::Queue', :url => nil ) }
+  let(:queue_double)  { double('AWS::SQS::QueueCollection', :[] => queue, :url_for => nil) }
+  let(:client_double) { double('AWS::SQS', :queues => queue_double) }
+
+  before(:each) do
+    expect(AWS::SQS).to receive(:new).and_return(client_double)
+  end
+
+  describe ".create" do
+    it "sets parser, sequencer, queue and writer" do
+      instance = Alephant::Publisher::Queue.create(options)
+      expect(instance.queue).to be_a Alephant::Publisher::Queue::SQSHelper::Queue
+    end
+
+    context "with account" do
+      it "creates a queue with an account number in the option hash" do
+        options = Alephant::Publisher::Queue::Options.new
+        options.add_queue({ :sqs_queue_name => 'bar', :aws_account_id => 'foo' })
+
+        expect(queue_double).to receive(:url_for).with('bar', { :queue_owner_aws_account_id => 'foo' })
+
+        Alephant::Publisher::Queue.create(options)
+      end
+    end
+
+    context "without account" do
+      it "creates a queue with an empty option hash" do
+        options = Alephant::Publisher::Queue::Options.new
+        options.add_queue({ :sqs_queue_name => 'bar' })
+
+        expect(queue_double).to receive(:url_for).with('bar', {})
+
+        Alephant::Publisher::Queue.create(options)
+      end
+    end
+  end
+end

data/spec/queue_spec.rb

@@ -0,0 +1,30 @@
+require 'spec_helper'
+
+describe Alephant::Publisher::Queue::SQSHelper::Queue do
+  describe "#message" do
+    it "returns a message" do
+      m = double("message").as_null_object
+      q = double("queue").as_null_object
+
+      expect(q).to receive(:receive_message).and_return(m)
+
+      instance = Alephant::Publisher::Queue::SQSHelper::Queue.new(q)
+
+      expect(instance.message).to eq(m)
+    end
+
+    it "call see(m) on the handed archiver" do
+      a = double("archiver").as_null_object
+      m = double("message").as_null_object
+      q = double("queue").as_null_object
+
+      expect(q).to receive(:receive_message).and_return(m)
+      expect(a).to receive(:see).with(m)
+
+      instance = Alephant::Publisher::Queue::SQSHelper::Queue.new(q, a)
+
+      expect(instance.message).to eq(m)
+    end
+  end
+end
+

data/spec/spec_helper.rb

@@ -0,0 +1,5 @@
+require 'pry'
+
+require 'aws-sdk'
+require 'alephant/publisher/queue'
+

data/spec/writer_spec.rb

@@ -0,0 +1,86 @@
+require 'spec_helper'
+
+describe Alephant::Publisher::Queue::Writer do
+  let(:opts) do
+    {
+      :lookup_table_name    => 'lookup_table_name',
+      :msg_vary_id_path     => '$.vary',
+      :renderer_id          => :renderer_id,
+      :s3_bucket_id         => :s3_bucket_id,
+      :s3_object_path       => :s3_object_path,
+      :sequence_id_path     => '$.sequence',
+      :sequencer_table_name => :sequencer_table_name,
+      :view_path            => :view_path
+    }
+  end
+
+  before(:each) do
+    AWS.stub!
+
+    allow_any_instance_of(Alephant::Cache).to receive(:initialize)
+      .with(
+        opts[:s3_bucket_id],
+        opts[:s3_object_path]
+      )
+
+    allow_any_instance_of(Alephant::Sequencer::SequenceTable).to receive(:create)
+
+    allow_any_instance_of(Alephant::Sequencer::Sequencer).to receive(:sequencer_id_from)
+      .and_return(1)
+
+    allow_any_instance_of(Alephant::Sequencer::Sequencer).to receive(:set_last_seen)
+
+    allow_any_instance_of(Alephant::Sequencer::Sequencer).to receive(:get_last_seen)
+
+    allow_any_instance_of(Alephant::Lookup::LookupTable).to receive(:create)
+
+    allow_any_instance_of(Alephant::Lookup::LookupTable).to receive(:table_name)
+
+    allow_any_instance_of(Alephant::Renderer::Renderer).to receive(:views).and_return({})
+  end
+
+  describe "#run!" do
+    let(:msg) do
+      data = {
+        "sequence" => "1",
+        "vary" => "foo"
+      }
+      Struct.new(:body,:id).new(data.to_json,'id')
+    end
+
+    let(:expected_location) do
+      'renderer_id/component_id/218c835cec343537589dbf1619532e4d/1'
+    end
+
+    let(:renderer) do
+      instance_double 'Alephant::Renderer::Renderer'
+    end
+
+    subject do
+      Alephant::Publisher::Queue::Writer.new(opts, msg)
+    end
+
+    it "should write the correct lookup location" do
+      allow_any_instance_of(Alephant::Cache).to receive(:put)
+
+      allow_any_instance_of(Alephant::Lookup::LookupHelper).to receive(:write)
+        .with(
+          "component_id",
+          {:variant=>"foo"},
+          1,
+          expected_location
+        )
+    end
+
+    it "should put the correct location, content to cache" do
+      allow_any_instance_of(Alephant::Lookup::LookupHelper).to receive(:write)
+
+      allow_any_instance_of(Alephant::Cache).to receive(:put)
+        .with(expected_location, "content", "foo/bar", :msg_id=>"id")
+    end
+
+    after do
+      subject.run!
+    end
+  end
+end

metadata

@@ -0,0 +1,325 @@
+--- !ruby/object:Gem::Specification
+name: alephant-publisher-queue
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- revett
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2014-10-10 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  prerelease: false
+  type: :development
+- !ruby/object:Gem::Dependency
+  name: rspec
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :development
+- !ruby/object:Gem::Dependency
+  name: rspec-nc
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :development
+- !ruby/object:Gem::Dependency
+  name: guard
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :development
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :development
+- !ruby/object:Gem::Dependency
+  name: pry
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :development
+- !ruby/object:Gem::Dependency
+  name: pry-remote
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :development
+- !ruby/object:Gem::Dependency
+  name: pry-nav
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :development
+- !ruby/object:Gem::Dependency
+  name: rake-rspec
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :development
+- !ruby/object:Gem::Dependency
+  name: rake
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: crimp
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: alephant-support
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: alephant-sequencer
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: alephant-cache
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: alephant-logger
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: alephant-lookup
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: alephant-renderer
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+  prerelease: false
+  type: :runtime
+description: Static publishing to S3 based on SQS messages
+email:
+- charlierevett@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .travis.yml
+- Gemfile
+- Guardfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- alephant-publisher-queue.gemspec
+- lib/alephant/publisher/env.rb
+- lib/alephant/publisher/queue.rb
+- lib/alephant/publisher/queue/options.rb
+- lib/alephant/publisher/queue/processor.rb
+- lib/alephant/publisher/queue/processor/base.rb
+- lib/alephant/publisher/queue/sqs_helper/archiver.rb
+- lib/alephant/publisher/queue/sqs_helper/queue.rb
+- lib/alephant/publisher/queue/version.rb
+- lib/alephant/publisher/queue/writer.rb
+- spec/archiver_spec.rb
+- spec/processor_spec.rb
+- spec/publisher_spec.rb
+- spec/queue_spec.rb
+- spec/spec_helper.rb
+- spec/writer_spec.rb
+homepage: https://github.com/BBC-News/alephant-publisher-queue
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.1.9
+signing_key:
+specification_version: 4
+summary: Static publishing to S3 based on SQS messages
+test_files:
+- spec/archiver_spec.rb
+- spec/processor_spec.rb
+- spec/publisher_spec.rb
+- spec/queue_spec.rb
+- spec/spec_helper.rb
+- spec/writer_spec.rb