alephant-publisher 0.1.3 → 0.1.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: be7e1421d4a9edd58b23cf1b67a39d0b77bc711f
-  data.tar.gz: 10abc065851746ceb2d215e61dae669653ac0432
+  metadata.gz: 8a18ed72b3f0273f7e8bedef7d101a3acebd5e79
+  data.tar.gz: 310f181c40f274f0cfca9081976c1b37690e6b2c
 SHA512:
-  metadata.gz: 06619711d680043e4c6153d5be311bef2e095645faa41d9f1e4841468ec7622cf0b3f2b51f6ec69b09267e0e90e682526fd69664c488ee03f7fad5fc5e8e36f1
-  data.tar.gz: ca6f76631797d41a818c2844627534ab536eb7f846fe0cdfc4528922a1b02a0cfdda0a2381040e61442a23b5f03479182c387262439944ba414baa97df12de69
+  metadata.gz: 5f28f89d431a8f264def3eb1b5c79a2425a49f242edc9e9500e30f490abdc00e8be568c468e87155bbcd2183c8e4d709b565357808e54cbc6f72878ff4c4f826
+  data.tar.gz: 5fa2be94fbdebf4c8a530245c957bf18040f0ccc81779af439a2e4726eff72c085a92a7ac5d848166f19351a5f8ad40d52a19d7b5962bc28f5a49c422f6b770d

data/lib/alephant/publisher.rb

@@ -2,19 +2,16 @@ require_relative 'env'
 
 require 'java'
 
-# setup executor
 java_import 'java.util.concurrent.ThreadPoolExecutor'
 java_import 'java.util.concurrent.TimeUnit'
 java_import 'java.util.concurrent.LinkedBlockingQueue'
 java_import 'java.util.concurrent.FutureTask'
-java_import 'java.util.concurrent.Callable'
 
+require 'alephant/publisher/version'
+require 'alephant/publisher/queue'
+require 'alephant/publisher/publish_task'
 require 'alephant/logger'
 
-require "alephant/publisher/version"
-require 'alephant/publisher/models/writer'
-require 'alephant/publisher/models/queue'
-
 module Alephant
   module Publisher
     include ::Alephant::Logger
@@ -23,70 +20,62 @@ module Alephant
       Publisher.new(opts, logger)
     end
 
-    private
-
     class Publisher
-      attr_reader :queue
+
+      VISIBILITY_TIMEOUT = 300
+      KEEP_ALIVE_TIMEOUT = 300
+      RECEIVE_WAIT_TIME  = 15
+      POOL_MIN_SIZE      = 2
+      POOL_MAX_SIZE      = 4
+
+      attr_reader :queue, :executor
 
       def initialize(opts, logger)
         ::Alephant::Logger.set_logger(logger) unless logger.nil?
 
         @opts = opts
         @queue = Queue.new(
-          opts[:sqs_queue_url]
+          opts[:sqs_queue_url],
+          opts[:visibility_timeout] || VISIBILITY_TIMEOUT,
+          opts[:receive_wait_time]  || RECEIVE_WAIT_TIME,
         )
-      end
 
-      def run!
-        core_pool_size    = @opts[:renderer_pool_min_size] || 2
-        maximum_pool_size = @opts[:renderer_pool_max_size] || 4
-        keep_alive_time   = @opts[:render_keep_alive_time] || 300
+        @writer_opts = opts.select do |k,v|
+          [
+            :msg_vary_id_path,
+            :sequencer_table_name,
+            :sequence_id_path,
+            :renderer_id,
+            :s3_bucket_id,
+            :s3_object_path,
+            :view_path,
+            :lookup_table_name
+          ].include? k
+        end
 
-        executor = ThreadPoolExecutor.new(
-          core_pool_size,
-          maximum_pool_size,
-          keep_alive_time,
+        @executor = ThreadPoolExecutor.new(
+          @opts[:renderer_pool_min_size] || POOL_MIN_SIZE,
+          @opts[:renderer_pool_max_size] || POOL_MAX_SIZE,
+          @opts[:render_keep_alive_time] || KEEP_ALIVE_TIMEOUT,
           TimeUnit::SECONDS,
           LinkedBlockingQueue.new
         )
+      end
 
-        @queue.poll do |msg|
-          task = FutureTask.new(PublishTask.new(@opts, msg))
-          executor.execute(task)
+      def run!
+        while true
+          executor.execute(
+            FutureTask.new(
+              PublishTask.new(
+                @writer_opts,
+                @queue.message
+              )
+            )
+          )
         end
 
         executor.shutdown()
       end
-
-    end
-
-    class PublishTask
-      include Callable
-
-      attr_reader :writer, :msg
-
-      def initialize(opts,msg)
-        @msg = msg
-        @writer = Writer.new(
-          opts.select do |k,v|
-            [
-              :msg_vary_id_path,
-              :sequencer_table_name,
-              :sequence_id_path,
-              :renderer_id,
-              :s3_bucket_id,
-              :s3_object_path,
-              :view_path,
-              :lookup_table_name
-            ].include? k
-          end
-        )
-      end
-
-      def call
-        writer.write(msg)
-      end
-
     end
   end
 end

data/lib/alephant/publisher/publish_task.rb

@@ -0,0 +1,29 @@
+require 'java'
+
+java_import 'java.util.concurrent.Callable'
+
+require 'alephant/publisher/writer'
+
+module Alephant
+  module Publisher
+    class PublishTask
+      include Callable
+
+      attr_reader :opts, :message
+
+      def initialize(opts, message)
+        @message = message
+        @opts = opts
+      end
+
+      def call
+        unless message.nil?
+          Writer.new(opts).write(message)
+          message.delete
+        end
+      end
+
+    end
+  end
+end
+

data/lib/alephant/publisher/{models/queue.rb → queue.rb}

@@ -4,11 +4,17 @@ require 'alephant/logger'
 module Alephant
   module Publisher
     class Queue
+      WAIT_TIME = 5
+      VISABILITY_TIMEOUT = 300
+
       include ::Alephant::Logger
 
+      attr_reader :timeout, :wait_time
       attr_accessor :q
 
-      def initialize(id)
+      def initialize(id, timeout = VISABILITY_TIMEOUT, wait_time = WAIT_TIME)
+        @timeout = timeout
+        @wait_time = wait_time
         @sqs = AWS::SQS.new
         @q = @sqs.queues[id]
 
@@ -25,10 +31,12 @@ module Alephant
         sleep 1 until @q.exists?
       end
 
-      def poll(*args, &block)
-        logger.info("Queue.poll: polling with arguments #{args}")
-        @q.poll(*args, &block)
+      def message
+        @q.receive_message({
+          :visibility_timeout => @timeout,
+          :wait_time_seconds => @wait_time
+        })
       end
     end
   end
-end
+end

data/lib/alephant/publisher/version.rb

@@ -1,5 +1,5 @@
 module Alephant
   module Publisher
-    VERSION = "0.1.3"
+    VERSION = "0.1.4"
   end
 end

data/lib/alephant/publisher/{models/write_operation.rb → write_operation.rb}

@@ -16,7 +16,7 @@ module Alephant
         @renderer_id        = opts[:renderer_id]
         @sequencer_opts     = opts[:sequencer_opts]
         @batch_sequencer    = sequencer_for(@renderer_id, @options)
-        @version            = @batch_sequencer.sequence_id_from(@msg)
+        @version            = @batch_sequencer.sequence_id_from(msg)
       end
 
       def sequencer_for(id, options)

data/lib/alephant/publisher/{models/writer.rb → writer.rb}

@@ -3,8 +3,8 @@ require 'alephant/views'
 require 'alephant/renderer'
 require 'alephant/lookup'
 
-require 'alephant/publisher/models/write_operation'
-require 'alephant/publisher/models/render_mapper'
+require 'alephant/publisher/write_operation'
+require 'alephant/publisher/render_mapper'
 
 module Alephant
   module Publisher

data/spec/publisher_spec.rb

@@ -4,89 +4,22 @@ describe Alephant::Publisher do
   let(:instance) { Alephant::Publisher.create }
 
   before(:each) do
-    Alephant::Publisher::Writer.any_instance.stub(:initialize)
     Alephant::Publisher::Queue.any_instance.stub(:initialize)
-    Alephant::Support::Parser.any_instance.stub(:initialize)
-    Alephant::Sequencer::Sequencer.any_instance.stub(:initialize)
   end
 
   describe "#initialize(opts = {}, logger)" do
     it "sets parser, sequencer, queue and writer" do
-      expect(instance.writer).to    be_a Alephant::Publisher::Writer
       expect(instance.queue).to     be_a Alephant::Publisher::Queue
-      expect(instance.parser).to    be_a Alephant::Support::Parser
-      expect(instance.sequencer).to be_a Alephant::Sequencer::Sequencer
     end
   end
+end
 
-  describe "#run!" do
-    it "returns a Thread" do
-      expect(instance.run!).to be_a(Thread)
-    end
-
-    it "calls @queue.poll" do
-      instance.should_receive(:receive).with(:msg)
-
-      expect_any_instance_of(Alephant::Publisher::Queue)
-      .to receive(:poll)
-      .and_yield(:msg)
-
-      t = instance.run!
-      t.join
-    end
+describe Alephant::Publisher::PublishTask do
+  before(:each) do
+    Alephant::Publisher::PublishTask.any_instance
+      .stub(:initialize)
   end
 
-  describe "#receive(msg)" do
-    before(:each) do
-      Alephant::Support::Parser
-        .any_instance
-        .stub(:parse)
-        .and_return(:parsed_msg)
-
-      Alephant::Sequencer::Sequencer
-        .any_instance
-        .stub(:sequence_id_from)
-        .and_return(:sequence_id)
-
-      Alephant::Sequencer::Sequencer
-        .any_instance
-        .stub(:set_last_seen)
-    end
-
-    context "message is nonsequential" do
-      before(:each) do
-        Alephant::Sequencer::Sequencer
-          .any_instance
-          .stub(:sequential?)
-          .and_return(false)
-      end
-
-      it "should not call write" do
-        Alephant::Publisher::Writer
-          .any_instance
-          .should_not_receive(:write)
-
-        instance.receive(:msg)
-      end
-    end
-
-    context "message is sequential" do
-      before(:each) do
-        Alephant::Sequencer::Sequencer
-          .any_instance
-          .stub(:sequential?)
-          .and_return(true)
-      end
-
-      it "calls writer with a parsed message and sequence_id" do
-        Alephant::Publisher::Writer
-          .any_instance
-          .should_receive(:write)
-          .with(:parsed_msg, :sequence_id)
-
-        instance.receive(:msg)
-      end
-    end
-  end
+  it { should respond_to(:call) }
 
 end

data/spec/spec_helper.rb

@@ -1,4 +1,6 @@
 require 'pry'
+
+require 'aws-sdk'
 require 'alephant/publisher'
 require 'alephant/publisher/models/writer'
 require 'alephant/publisher/models/queue'

data/spec/writer_spec.rb

@@ -1,89 +1,112 @@
 require 'spec_helper'
 
 describe Alephant::Publisher::Writer do
+  let(:opts) do
+    {
+      :s3_bucket_id => :s3_bucket_id,
+      :s3_object_path => :s3_object_path,
+      :renderer_id => :renderer_id,
+      :view_path => :view_path,
+      :lookup_table_name => 'lookup_table_name',
+      :sequencer_table_name => :sequencer_table_name,
+      :sequence_id_path => '$.sequence',
+      :msg_vary_id_path => '$.vary',
+    }
+  end
+
   before(:each) do
+    AWS.stub!
+
     Alephant::Publisher::RenderMapper
       .any_instance
-      .stub(:initialize)
+      .should_receive(:initialize)
+      .with(
+        opts[:renderer_id],
+        opts[:view_path]
+      )
 
-      Alephant::Cache
-        .any_instance
-        .stub(:initialize)
-  end
+    Alephant::Cache
+      .any_instance
+      .should_receive(:initialize)
+      .with(
+        opts[:s3_bucket_id],
+        opts[:s3_object_path]
+      )
 
-  subject do
-    Alephant::Publisher::Writer.new({
-      :renderer_id       => 'renderer_id',
-      :lookup_table_name => 'lookup_table_name'
-    })
-  end
+    Alephant::Sequencer::SequenceTable
+      .any_instance
+      .stub(:create)
 
-  describe "#write(data, version)" do
-    before(:each) do
-      Alephant::Publisher::RenderMapper
-        .any_instance
-        .stub(:generate)
-        .and_return({
-          'component_id' => Struct.new(:render).new('content')
-        })
+    Alephant::Sequencer::Sequencer
+      .any_instance
+      .stub(:sequencer_id_from)
+      .and_return(1)
 
-    end
+    Alephant::Sequencer::Sequencer
+      .any_instance
+      .stub(:set_last_seen)
 
-    it "should write the correct lookup location" do
-      options = { :key     => :value  }
-      data    = { :options => options }
+    Alephant::Sequencer::Sequencer
+      .any_instance
+      .stub(:get_last_seen)
 
-      Alephant::Cache.any_instance
-        .stub(:put)
+    Alephant::Lookup::LookupHelper
+      .any_instance
+      .stub(:create_lookup_table)
 
-      Alephant::Lookup
-        .should_receive(:create)
-        .with('lookup_table_name')
-        .and_call_original
+    Alephant::Lookup::LookupTable
+      .any_instance
+      .stub(:table_name)
 
-      Alephant::Lookup::LookupHelper.any_instance
-        .stub(:initialize)
+    Alephant::Publisher::RenderMapper
+      .any_instance
+      .stub(:generate)
+      .and_return({
+        'component_id' => Struct.new(:render).new('content')
+      })
 
-      Alephant::Lookup::LookupTable
-        .any_instance
-        .stub(:table_name)
+  end
 
-      Alephant::Lookup::LookupHelper.any_instance
-        .should_receive(:batch_write)
-        .with(
-          options,
-          'renderer_id/component_id/42de5e5c6f74b9fe4d956704a6d9e1c7/0',
-          'component_id'
-        )
+  subject do
+    Alephant::Publisher::Writer.new(opts)
+  end
 
-      Alephant::Lookup::LookupHelper.any_instance
-        .should_receive(:batch_process)
+  describe "#write(data, version)" do
+    let(:msg) do
+      data = {
+        "sequence" => "1",
+        "vary" => "foo"
+      }
+      Struct.new(:body).new(data.to_json)
+    end
 
-      subject.write(data, 0)
+    let(:expected_location) do
+      'renderer_id/component_id/218c835cec343537589dbf1619532e4d/1'
     end
 
-    it "should put the correct location, content to cache" do
-      Alephant::Lookup::LookupHelper
-        .any_instance
-        .stub(:initialize)
+    it "should write the correct lookup location" do
+      Alephant::Cache.any_instance.stub(:put)
 
       Alephant::Lookup::LookupHelper
         .any_instance
-        .stub(:batch_write)
+        .should_receive(:write)
+        .with(
+          {:variant=>"foo"},
+          expected_location
+        )
+    end
 
-      Alephant::Lookup::LookupHelper
-        .any_instance
-        .stub(:batch_process)
+    it "should put the correct location, content to cache" do
+      Alephant::Lookup::LookupHelper.any_instance.stub(:write)
 
-      Alephant::Lookup::LookupTable
+      Alephant::Cache
         .any_instance
-        .stub(:table_name)
-
-      Alephant::Cache.any_instance
         .should_receive(:put)
-        .with('renderer_id/component_id/35589a1cc0b3ca90fc52d0e711c0c434/0', 'content')
+        .with(expected_location, "content")
+    end
 
-      subject.write({}, 0)
+    after do
+      subject.write(msg)
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: alephant-publisher
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - Integralist
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2014-03-07 00:00:00.000000000 Z
+date: 2014-03-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -349,11 +349,12 @@ files:
 - alephant-publisher.gemspec
 - lib/alephant/env.rb
 - lib/alephant/publisher.rb
-- lib/alephant/publisher/models/queue.rb
-- lib/alephant/publisher/models/render_mapper.rb
-- lib/alephant/publisher/models/write_operation.rb
-- lib/alephant/publisher/models/writer.rb
+- lib/alephant/publisher/publish_task.rb
+- lib/alephant/publisher/queue.rb
+- lib/alephant/publisher/render_mapper.rb
 - lib/alephant/publisher/version.rb
+- lib/alephant/publisher/write_operation.rb
+- lib/alephant/publisher/writer.rb
 - spec/fixtures/components/foo/models/bar.rb
 - spec/fixtures/components/foo/models/foo.rb
 - spec/fixtures/components/foo/templates/bar.mustache