alephant 0.0.9.9.1-java → 0.1.1-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f69ea72533208eadc0bfa7a753e1cda142ab50b2
-  data.tar.gz: b493462c0cfa05a935b3e09b152b12521cb64909
+  metadata.gz: e380898fbfc1de04d24909fbd8cc82f82c338c4d
+  data.tar.gz: 8d82eb8ad5ec19279c9547092e74d04aabc61ac1
 SHA512:
-  metadata.gz: 9b140cb3277fef033e6abf154690f9df92ef07936c9df636a19b9ef3d6e885f375a3697cac5568f476a34cf3fe8e82cb3a51351de05143916636c1541d74043b
-  data.tar.gz: e5fd40d103db18f36253518ce061c27460fdee0812827131ffbd266b9c6d06c377fdcc71ce69e5b370e820eed182b515a56452cd568842befef5477e6ed0d299
+  metadata.gz: 3ca79920b31f43796118ff510eea1735ff2fdfc396357dd867e5711ddc918b70de9d9774dd9380c320476f70ad373d8ad0ac58b7a548e3f14b4f874dec1af91a
+  data.tar.gz: 5d48de598a04829e0ed2a1fbddd909cad847cf79fc10fc2f9846ffcdd0a5521139ee5be51d2dc072531cb1aa19f473c10d109543f1dadc5ca46003d850d26003

data/.gitignore

@@ -8,3 +8,4 @@ Gemfile.lock
 /tmp
 /components
 
+*.swp

data/alephant.gemspec

@@ -8,14 +8,13 @@ Gem::Specification.new do |s|
   s.name          = 'alephant'
   s.version       = Alephant::VERSION
   s.date          = Time.now.strftime "%Y-%m-%d"
-  s.summary       = "Static Publishing in the Cloud"
+  s.summary       = "Static publishing in the cloud"
   s.description   = "Static publishing to S3 based on SQS messages"
   s.authors       = ["Robert Kenny"]
   s.email         = 'kenoir@gmail.com'
   s.license       = 'GPLv3'
 
   s.files         = `git ls-files`.split($/)
-  s.executables   = s.files.grep(%r{^bin/}) { |f| File.basename(f) }
   s.platform      = "java"
   s.homepage      = 'https://github.com/BBC-News/alephant'
   s.test_files    = s.files.grep(%r{^(test|spec|features)/})
@@ -36,5 +35,12 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'aws-sdk', '~> 1.0'
   s.add_runtime_dependency 'mustache', '>= 0.99.5'
   s.add_runtime_dependency 'jsonpath'
+
+  s.add_runtime_dependency 'alephant-sequencer'
+  s.add_runtime_dependency 'alephant-logger'
+  s.add_runtime_dependency 'alephant-cache'
+  s.add_runtime_dependency 'alephant-renderer'
+  s.add_runtime_dependency 'alephant-lookup'
+  s.add_runtime_dependency 'alephant-preview'
 end
 

data/lib/alephant.rb

@@ -1,63 +1,48 @@
 require_relative 'env'
 
-require 'alephant/models/logger'
-require 'alephant/models/queue'
-require 'alephant/models/cache'
-require 'alephant/models/renderer'
-require 'alephant/models/multi_renderer'
-require 'alephant/models/sequence_table'
-require 'alephant/models/sequencer'
+require 'alephant/models/render_mapper'
 require 'alephant/models/parser'
+require 'alephant/models/writer'
+require 'alephant/models/queue'
 
-require 'alephant/errors'
+require 'alephant/sequencer'
+require 'alephant/cache'
+require 'alephant/logger'
 require 'alephant/views'
+require 'alephant/renderer'
+require 'alephant/lookup'
 
 module Alephant
   class Alephant
-    attr_reader :sequencer, :queue, :cache, :renderer
+    include ::Alephant::Logger
 
-    VALID_OPTS = [
-      :s3_bucket_id,
-      :s3_object_path,
-      :s3_object_id,
-      :table_name,
-      :sqs_queue_id,
-      :view_path,
-      :component_id,
-      :sequence_id
-    ]
+    attr_reader :sequencer, :queue, :writer, :parser
 
     def initialize(opts = {}, logger = nil)
-      set_logger(logger)
-      set_opts(opts)
-
-      @logger = ::Alephant.logger
-      @sequencer = Sequencer.create(@table_name, @sqs_queue_id, @sequence_id)
-      @queue = Queue.new(@sqs_queue_id)
-      @cache = Cache.new(@s3_bucket_id, @s3_object_path)
-      @multi_renderer = MultiRenderer.new(@component_id, @view_path)
-      @parser = Parser.new
-    end
-
-    def set_logger(logger)
-      ::Alephant.logger = logger
-    end
-
-    def write(data)
-      @multi_renderer.render(data).each do |id, item|
-        @cache.put(id, item)
-      end
-    end
+      ::Alephant::Logger.set_logger(logger) unless logger.nil?
 
-    def receive(msg)
-      @logger.info("Alephant.receive: with id #{msg.id} and body digest: #{msg.md5}")
-
-      if @sequencer.sequential?(msg)
-        write @parser.parse msg.body
-        @sequencer.set_last_seen(msg)
-      else
-        @logger.warn("Alephant.receive: out of sequence message received #{msg.id} (discarded)")
-      end
+      @parser = Parser.new(
+        opts[:msg_vary_id_path]
+      )
+      @sequencer = Sequencer.create(
+        opts[:sequencer_table_name],
+        opts[:sqs_queue_url],
+        opts[:sequence_id_path]
+      )
+      @queue = Queue.new(
+        opts[:sqs_queue_url]
+      )
+      @writer = Writer.new(
+        opts.select do |k,v|
+          [
+            :renderer_id,
+            :s3_bucket_id,
+            :s3_object_path,
+            :view_path,
+            :lookup_table_name
+          ].include? k
+        end
+      )
     end
 
     def run!
@@ -66,15 +51,14 @@ module Alephant
       end
     end
 
+    def receive(msg)
+      write msg if sequencer.sequential?(msg)
+    end
+
     private
-    def set_opts(opts)
-      VALID_OPTS.each do | k |
-        v = opts.has_key?(k) ? opts[k] : nil
-        singleton_class.class_eval do
-          attr_accessor k
-        end
-        send("#{k}=", v)
-      end
+    def write(msg)
+      writer.write(parser.parse(msg), sequencer.sequence_id_from(msg))
+      sequencer.set_last_seen(msg)
     end
 
   end

data/lib/alephant/models/jsonpath_lookup.rb

@@ -0,0 +1,16 @@
+require 'jsonpath'
+require 'json'
+
+module Alephant
+  class JsonPathLookup
+    attr_reader :path
+    def initialize(path)
+      @path = path
+      @jsonpath = JsonPath.new(path)
+    end
+
+    def lookup(msg)
+      @jsonpath.on(JSON.parse(msg)).first
+    end
+  end
+end

data/lib/alephant/models/parser.rb

@@ -2,8 +2,26 @@ require 'json'
 
 module Alephant
   class Parser
-    def parse data
+    attr_reader :vary_lookup
+
+    def initialize(vary_jsonpath = nil)
+      @vary_lookup = vary_jsonpath ? JsonPathLookup.new(vary_jsonpath) : nil
+    end
+
+    def parse(msg)
+      symbolize(msg.body).tap { |o| o[:options] = options_for msg }
+    end
+
+    private
+    def symbolize(data)
       JSON.parse(data, :symbolize_names => true)
     end
+
+    def options_for(msg)
+      {}.tap do |o|
+        o[:variant] = vary_lookup.lookup(msg.body) if not vary_lookup.nil?
+      end
+    end
   end
-end
+end
+

data/lib/alephant/models/queue.rb

@@ -1,22 +1,23 @@
 require 'aws-sdk'
+require 'alephant/logger'
 
 module Alephant
   class Queue
+    include ::Alephant::Logger
+
     attr_accessor :q
 
     def initialize(id)
-      @logger = ::Alephant.logger
-
       @sqs = AWS::SQS.new
       @q = @sqs.queues[id]
 
       unless @q.exists?
         @q = @sqs.queues.create(id)
         sleep_until_queue_exists
-        @logger.info("Queue.initialize: created queue with id #{id}")
+        logger.info("Queue.initialize: created queue with id #{id}")
       end
 
-      @logger.info("Queue.initialize: ended with id #{id}")
+      logger.info("Queue.initialize: ended with id #{id}")
     end
 
     def sleep_until_queue_exists
@@ -24,7 +25,7 @@ module Alephant
     end
 
     def poll(*args, &block)
-      @logger.info("Queue.poll: polling with arguments #{args}")
+      logger.info("Queue.poll: polling with arguments #{args}")
       @q.poll(*args, &block)
     end
   end

data/lib/alephant/models/render_mapper.rb

@@ -0,0 +1,56 @@
+require 'alephant/logger'
+
+module Alephant
+  class RenderMapper
+    include ::Alephant::Logger
+    DEFAULT_LOCATION = 'components'
+
+    def initialize(component_id, view_base_path=nil)
+      self.base_path = "#{view_base_path}/#{component_id}" unless view_base_path.nil?
+      @component_id = component_id
+    end
+
+    def base_path
+      @base_path || DEFAULT_LOCATION
+    end
+
+    def base_path=(path)
+      @base_path = File.directory?(path) ? path : (raise "Invalid path")
+    end
+
+    def generate(data)
+      template_locations.reduce({}) do |obj, file|
+        template_id = template_id_for file
+        obj.tap { |o| o[template_id] = create_renderer(template_id, data) }
+      end
+    end
+
+    def create_renderer(template_file, data)
+      model = instantiate_model(template_file, data)
+      Renderer.create(template_file, base_path, model)
+    end
+
+    private
+    def instantiate_model(view_id, data)
+      require model_location_for view_id
+      Views.get_registered_class(view_id).new(data)
+    end
+
+    def model_location_for(template_file)
+      File.join(base_path, 'models', "#{template_file}.rb")
+    end
+
+   def template_locations
+      Dir[template_base_path]
+    end
+
+    def template_base_path
+      "#{base_path}/templates/*"
+    end
+
+    def template_id_for(template_location)
+      template_location.split('/').last.sub(/\.mustache/, '')
+    end
+
+  end
+end

data/lib/alephant/models/writer.rb

@@ -0,0 +1,44 @@
+require 'alephant/models/jsonpath_lookup'
+
+module Alephant
+  class Writer
+    attr_reader :mapper, :cache
+
+    def initialize(opts)
+      @renderer_id =
+        opts[:renderer_id]
+      @cache = Cache.new(
+        opts[:s3_bucket_id],
+        opts[:s3_object_path]
+      )
+      @mapper = RenderMapper.new(
+        opts[:renderer_id],
+        opts[:view_path]
+      )
+      @lookup_table_name =
+        opts[:lookup_table_name]
+    end
+
+    def write(data, version = nil)
+      mapper.generate(data).each do |id, r|
+        store(id, r.render, location_for(id, version), data[:options])
+      end
+    end
+
+    private
+    def store(id, content, location, options)
+      cache.put(location, content)
+      lookup(id).write(options, location)
+    end
+
+    def lookup(component_id)
+      Lookup.create(@lookup_table_name, component_id)
+    end
+
+    def location_for(component_id, version = nil)
+      base_name = "#{@renderer_id}_#{component_id}"
+      version ? "#{base_name}_#{version}" : base_name
+    end
+
+  end
+end

data/lib/alephant/version.rb

@@ -1,3 +1,3 @@
 module Alephant
-  VERSION = "0.0.9.9.1"
+  VERSION = "0.1.1"
 end

data/lib/env.rb

@@ -3,8 +3,6 @@ $LOAD_PATH << File.dirname(__FILE__)
 require 'aws-sdk'
 require 'yaml'
 
-require 'alephant/util/string'
-
 config_file = File.join('config', 'aws.yaml')
 
 if File.exists? config_file

data/spec/alephant_spec.rb

@@ -1,229 +1,83 @@
 require 'spec_helper'
 
 describe Alephant::Alephant do
-  subject { Alephant::Alephant }
 
-  describe "initialize(opts = {})" do
-    before(:each) do
-      sequencer      = double()
-      queue          = double()
-      cache          = double()
-      multi_renderer = double()
-
-      Alephant::Sequencer.stub(:create).and_return(sequencer)
-      Alephant::Queue.any_instance.stub(:initialize).and_return(queue)
-      Alephant::Cache.any_instance.stub(:initialize).and_return(cache)
-      Alephant::MultiRenderer.any_instance.stub(:initialize).and_return(multi_renderer)
-    end
-
-    it "sets specified options" do
-      instance = subject.new({
-        :s3_bucket_id   => :s3_bucket_id,
-        :s3_object_path => :s3_object_path,
-        :s3_object_id   => :s3_object_id,
-        :table_name     => :table_name,
-        :sqs_queue_id   => :sqs_queue_id,
-        :component_id   => :component_id,
-        :sequence_id    => :sequence_id
-      })
-
-      expect(instance.s3_bucket_id).to eq(:s3_bucket_id);
-      expect(instance.s3_object_path).to eq(:s3_object_path);
-      expect(instance.s3_object_id).to eq(:s3_object_id);
-      expect(instance.table_name).to eq(:table_name);
-      expect(instance.sqs_queue_id).to eq(:sqs_queue_id);
-      expect(instance.component_id).to eq(:component_id);
-      expect(instance.sequence_id).to eq(:sequence_id);
-    end
-
-    it "sets unspecified options to nil" do
-      instance = subject.new
-
-      expect(instance.s3_bucket_id).to eq(nil);
-      expect(instance.s3_object_path).to eq(nil);
-      expect(instance.s3_object_id).to eq(nil);
-      expect(instance.table_name).to eq(nil);
-      expect(instance.sqs_queue_id).to eq(nil);
-      expect(instance.component_id).to eq(nil);
-      expect(instance.sequence_id).to eq(nil);
-    end
-
-    context "initializes @sequencer" do
-      it "with Sequencer.create(:table_name, @sqs_queue_id)" do
-        Alephant::Sequencer
-          .should_receive(:create)
-          .with(:table_name, :sqs_queue_id, :sequence_id)
-
-        instance = subject.new({
-          :table_name   => :table_name,
-          :sqs_queue_id => :sqs_queue_id,
-          :sequence_id => :sequence_id
-        })
-      end
-    end
-
-    context "initializes @queue" do
-      it "with Queue.new(@sqs_queue_id)" do
-        Alephant::Queue
-          .should_receive(:new)
-          .with(:sqs_queue_id)
-
-        instance = subject.new({
-          :sqs_queue_id => :sqs_queue_id
-        })
-      end
-    end
-
-    context "initializes @cache" do
-      it "with Cache.new(@s3_bucket_id, @s3_object_path)" do
-        Alephant::Cache
-          .should_receive(:new)
-          .with(:s3_bucket_id, :s3_object_path)
-
-        instance = subject.new({
-          :s3_bucket_id   => :s3_bucket_id,
-          :s3_object_path => :s3_object_path
-        })
-      end
-    end
-
-    context "initializes @multi_renderer" do
-      it "MultiRenderer class to be initialized" do
-        Alephant::MultiRenderer
-          .should_receive(:new)
-          .with('foo', 'components')
-
-        instance = subject.new({
-          :view_path    => 'components',
-          :component_id => 'foo'
-        })
-      end
-    end
+  before(:each) do
+    Alephant::Writer.any_instance.stub(:initialize)
+    Alephant::Queue.any_instance.stub(:initialize)
+    Alephant::Parser.any_instance.stub(:initialize)
+    Alephant::Sequencer::Sequencer.any_instance.stub(:initialize)
   end
 
-  describe "run!" do
-    before(:each) do
-      sequencer      = double()
-      queue          = double()
-      cache          = double()
-      multi_renderer = double()
-
-      Alephant::Sequencer.stub(:create).and_return(sequencer)
-      Alephant::Queue.any_instance.stub(:initialize).and_return(queue)
-      Alephant::Cache.any_instance.stub(:initialize).and_return(cache)
-      Alephant::MultiRenderer.any_instance.stub(:initialize).and_return(multi_renderer)
+  describe "#initialize(opts = {}, logger)" do
+    it "sets parser, sequencer, queue and writer" do
+      expect(subject.writer).to be_a Alephant::Writer
+      expect(subject.queue).to be_a Alephant::Queue
+      expect(subject.parser).to be_a Alephant::Parser
+      expect(subject.sequencer).to be_a Alephant::Sequencer::Sequencer
     end
+  end
 
+  describe "#run!" do
     it "returns a Thread" do
-      instance = subject.new({
-        :sqs_queue_id => :sqs_queue_id
-      })
-
-      expect(instance.run!).to be_a(Thread)
+      expect(subject.run!).to be_a(Thread)
     end
 
     it "calls @queue.poll" do
-      instance = subject.new({
-        :sqs_queue_id => :sqs_queue_id
-      })
-
-      instance.should_receive(:receive).with(:msg)
+      subject.should_receive(:receive).with(:msg)
 
       expect_any_instance_of(Alephant::Queue)
-        .to receive(:poll)
-        .and_yield(:msg)
+      .to receive(:poll)
+      .and_yield(:msg)
 
-      t = instance.run!
+      t = subject.run!
       t.join
     end
   end
 
-  describe "receive(msg)" do
-    before(:each) do
-      sequencer      = double()
-      queue          = double()
-      cache          = double()
-      multi_renderer = double()
-
-      Alephant::Queue.any_instance.stub(:initialize).and_return(queue)
-      Alephant::Cache.any_instance.stub(:initialize).and_return(cache)
-      Alephant::MultiRenderer.any_instance.stub(:initialize).and_return(multi_renderer)
-    end
-
-    it "writes data to cache if sequential order is true" do
-      data = "{ \"foo\":\"bar\" }"
-
-      msg = double()
-      msg.stub(:id).and_return(:id)
-      msg.stub(:md5).and_return(:md5)
-      msg.stub(:body).and_return(data)
-
-
-      Alephant::Sequencer::Sequencer
-        .any_instance
-        .stub(:sequential?)
-        .and_return(true)
-
-      Alephant::Sequencer::SequenceTable
-        .any_instance
-        .stub(:create)
-
-      Alephant::Sequencer::Sequencer
-        .any_instance
-        .stub(:set_last_seen)
-
-      instance = subject.new
+  describe "#receive(msg)" do
+    subject { Alephant::Alephant.new }
 
-      instance
-        .should_receive(:write)
-        .with(JSON.parse(data, :symbolize_names => true))
-
-      instance.receive(msg)
-    end
-  end
-
-  describe "write(data)" do
     before(:each) do
-      sequencer = double()
-      queue     = double()
-
+      Alephant::Parser.any_instance
+        .stub(:parse)
+        .and_return(:parsed_msg)
+      Alephant::Sequencer::Sequencer
+        .any_instance.stub(:sequence_id_from)
+        .and_return(:sequence_id)
       Alephant::Sequencer::Sequencer
-        .any_instance
-        .stub(:initialize)
-        .and_return(sequencer)
-
-      Alephant::Queue
-        .any_instance
-        .stub(:initialize)
-        .and_return(queue)
+        .any_instance.stub(:set_last_seen)
     end
 
-    it "puts rendered data into the S3 Cache" do
-      templates = {
-        :foo => 'content',
-        :bar => 'content'
-      }
+    context "message is nonsequential" do
+      before(:each) do
+        Alephant::Sequencer::Sequencer
+          .any_instance.stub(:sequential?)
+          .and_return(false)
+      end
 
-      Alephant::Cache
-        .any_instance
-        .should_receive(:put)
-        .with(:foo, templates[:foo])
+      it "should not call write" do
+        Alephant::Writer.any_instance
+          .should_not_receive(:write)
 
-      Alephant::Cache
-        .any_instance
-        .should_receive(:put)
-        .with(:bar, templates[:bar])
+        subject.receive(:msg)
+      end
+    end
 
-      Alephant::MultiRenderer
-        .any_instance
-        .stub(:render)
-        .and_return(templates)
+    context "message is sequential" do
+      before(:each) do
+        Alephant::Sequencer::Sequencer
+          .any_instance.stub(:sequential?)
+          .and_return(true)
+      end
 
-      instance = subject.new({
-        :s3_object_id => :s3_object_id
-      })
+      it "calls writer with a parsed message and sequence_id" do
+        Alephant::Writer.any_instance
+          .should_receive(:write).with(:parsed_msg, :sequence_id)
 
-      instance.write(:content)
+        subject.receive(:msg)
+      end
     end
   end
+
 end