alephant-publisher 0.1.4 → 0.1.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8a18ed72b3f0273f7e8bedef7d101a3acebd5e79
-  data.tar.gz: 310f181c40f274f0cfca9081976c1b37690e6b2c
+  metadata.gz: 5c8298673d27ca51e22d1ff4e124bf838b91de45
+  data.tar.gz: 554f9a1eb1dd64e4f43b42418dc5494c67dae3fd
 SHA512:
-  metadata.gz: 5f28f89d431a8f264def3eb1b5c79a2425a49f242edc9e9500e30f490abdc00e8be568c468e87155bbcd2183c8e4d709b565357808e54cbc6f72878ff4c4f826
-  data.tar.gz: 5fa2be94fbdebf4c8a530245c957bf18040f0ccc81779af439a2e4726eff72c085a92a7ac5d848166f19351a5f8ad40d52a19d7b5962bc28f5a49c422f6b770d
+  metadata.gz: 1d36ef84078d02c36ce4b6fc2ddb665aff51a622ef9593ed3ae64faef783fb9603a7f602bbb557faddeb898d6089df492f9cd3e20ec06229058a639e31f36cf9
+  data.tar.gz: eb6f7446bb0abe43b1ba0c5088b543733e1b44de6c9187bca159e0afb54c893015d6c53b0018ac64c772adc2ef49140abea319d13c8cd6ccb72cf289911e91ba

data/alephant-publisher.gemspec

@@ -35,6 +35,7 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency 'mustache', '>= 0.99.5'
   spec.add_runtime_dependency 'jsonpath'
   spec.add_runtime_dependency 'crimp'
+  spec.add_runtime_dependency 'peach'
   spec.add_runtime_dependency 'alephant-support'
   spec.add_runtime_dependency 'alephant-sequencer'
   spec.add_runtime_dependency 'alephant-cache'

data/lib/alephant/publisher.rb

@@ -22,11 +22,12 @@ module Alephant
 
     class Publisher
 
-      VISIBILITY_TIMEOUT = 300
-      KEEP_ALIVE_TIMEOUT = 300
+      VISIBILITY_TIMEOUT = 60
+      KEEP_ALIVE_TIMEOUT = 60
       RECEIVE_WAIT_TIME  = 15
       POOL_MIN_SIZE      = 2
       POOL_MAX_SIZE      = 4
+      QUEUE_THROTTLE     = 0.5
 
       attr_reader :queue, :executor
 
@@ -72,6 +73,8 @@ module Alephant
               )
             )
           )
+
+          sleep QUEUE_THROTTLE while executor.getActiveCount == executor.getMaximumPoolSize
         end
 
         executor.shutdown()

data/lib/alephant/publisher/publish_task.rb

@@ -18,7 +18,7 @@ module Alephant
 
       def call
         unless message.nil?
-          Writer.new(opts).write(message)
+          Writer.new(opts, message).run!
           message.delete
         end
       end

data/lib/alephant/publisher/version.rb

@@ -1,5 +1,5 @@
 module Alephant
   module Publisher
-    VERSION = "0.1.4"
+    VERSION = "0.1.5"
   end
 end

data/lib/alephant/publisher/writer.rb

@@ -1,77 +1,110 @@
+require 'peach'
+require 'crimp'
+
 require 'alephant/cache'
 require 'alephant/views'
 require 'alephant/renderer'
 require 'alephant/lookup'
-
-require 'alephant/publisher/write_operation'
+require 'alephant/logger'
+require 'alephant/sequencer'
+require 'alephant/support/parser'
 require 'alephant/publisher/render_mapper'
 
 module Alephant
   module Publisher
     class Writer
-      attr_reader :mapper, :cache
+      include ::Alephant::Logger
+      attr_reader :config, :message, :cache, :parser, :mapper
+
+      def initialize(config, message)
+        @config   = config
+        @message  = message
 
-      def initialize(opts)
         @cache = Cache.new(
-          opts[:s3_bucket_id],
-          opts[:s3_object_path]
+          config[:s3_bucket_id],
+          config[:s3_object_path]
+        )
+
+        @parser = Support::Parser.new(
+          config[:msg_vary_id_path]
         )
 
         @mapper = RenderMapper.new(
-          opts[:renderer_id],
-          opts[:view_path]
+          config[:renderer_id],
+          config[:view_path]
         )
+      end
 
-        @lookup_table_name = opts[:lookup_table_name]
+      def run!
+        batch? ? batch.sequence(message, &perform) : perform.call
+      end
 
-        @renderer_id = opts[:renderer_id]
+      private
 
-        @write_opts = {
-          :sequencer_opts => {
-            :table_name => opts[:sequencer_table_name],
-            :id_path    => opts[:sequence_id_path]
-          },
-          :msg_vary_path => opts[:msg_vary_id_path],
-          :renderer_id => opts[:renderer_id]
-        }
+      def perform
+        Proc.new { renders.peach { |id, r| write(id, r) } }
       end
 
-      def write(msg)
-        write_op = WriteOperation.new(msg, @write_opts)
-
-        write_op.batch_sequencer.sequence(msg) do |msg|
-          mapper.generate(write_op.data).each do |component_id, renderer|
-            write_component(write_op, component_id, renderer)
+      def write(id, r)
+        begin
+          seq_for(id).sequence(message) do
+            store(id, r.render, location_for(id))
           end
+        rescue Exception => e
+          logger.warn "Alephant::Publisher::Writer#write: #{e.message}\n#{e.backtrace.join('\n')}"
+
+          raise e
         end
       end
 
-      private
+      def store(id, content, location)
+        cache.put(location, content)
+        lookup.write(id, options, seq_id, location)
+      end
 
-      def write_component(write_op, component_id, renderer)
-        location = location_for(component_id, write_op.options, write_op.version)
-        component_sequencer = write_op.sequencer_for(component_id, write_op.options)
+      def location_for(id)
+        "#{config[:renderer_id]}/#{id}/#{opt_hash}/#{seq_id}"
+      end
 
-        component_sequencer.sequence(write_op.msg) do |msg|
-          store(component_id, renderer.render, write_op.options, location)
-        end
+      def batch
+        @batch ||= (renders.count > 1) ? seq_for(config[:renderer_id]) : nil
       end
 
-      def store(component_id, content, options, location)
-        cache.put(location, content)
-        lookup_for(component_id).write(options, location)
+      def batch?
+        !batch.nil?
+      end
+
+      def seq_for(id)
+        Sequencer.create(config[:sequencer_table_name], seq_key_from(id), config[:sequence_id_path])
       end
 
-      def lookup_for(id)
-        Lookup.create(@lookup_table_name, id)
+      def seq_key_from(id)
+        "#{id}/#{opt_hash}"
       end
 
-      def location_for(component_id, options, version = nil)
-        options_hash = Crimp.signature(options)
-        base_name = "#{@renderer_id}/#{component_id}/#{options_hash}"
-        version ? "#{base_name}/#{version}" : base_name
+      def seq_id
+        @seq_id ||= Sequencer::Sequencer.sequence_id_from(message, config[:sequence_id_path])
       end
 
+      def renders
+        @renders ||= mapper.generate(data)
+      end
+
+      def opt_hash
+        @opt_hash ||= Crimp.signature(options)
+      end
+
+      def options
+        @options ||= data[:options]
+      end
+
+      def data
+        @data ||= parser.parse(message)
+      end
+
+      def lookup
+        Lookup.create(config[:lookup_table_name])
+      end
     end
   end
 end

data/spec/queue_spec.rb

@@ -42,28 +42,6 @@ describe Alephant::Publisher::Queue do
     end
   end
 
-  describe "poll(*args, &block)" do
-    it "calls @q.poll(*args, &block)" do
-      block = double()
-      block.should_receive(:called)
-
-      AWS::SQS::Queue
-        .any_instance
-        .stub(:exists?)
-        .and_return(true)
-
-      AWS::SQS::Queue
-        .any_instance
-        .should_receive(:poll)
-        .with(:arg)
-        .and_yield
-
-      subject.new(:id).poll(:arg) do
-        block.called
-      end
-    end
-  end
-
   describe "sleep_until_queue_exists" do
     context "@q.exists? == true" do
       it "should not call sleep" do

data/spec/spec_helper.rb

@@ -2,9 +2,4 @@ require 'pry'
 
 require 'aws-sdk'
 require 'alephant/publisher'
-require 'alephant/publisher/models/writer'
-require 'alephant/publisher/models/queue'
-require 'alephant/publisher/models/render_mapper'
-require 'alephant/renderer'
-require 'alephant/support/parser'
 

data/spec/writer_spec.rb

@@ -50,9 +50,9 @@ describe Alephant::Publisher::Writer do
       .any_instance
       .stub(:get_last_seen)
 
-    Alephant::Lookup::LookupHelper
+    Alephant::Lookup::LookupTable
       .any_instance
-      .stub(:create_lookup_table)
+      .stub(:create)
 
     Alephant::Lookup::LookupTable
       .any_instance
@@ -67,11 +67,7 @@ describe Alephant::Publisher::Writer do
 
   end
 
-  subject do
-    Alephant::Publisher::Writer.new(opts)
-  end
-
-  describe "#write(data, version)" do
+  describe "#run!" do
     let(:msg) do
       data = {
         "sequence" => "1",
@@ -84,6 +80,10 @@ describe Alephant::Publisher::Writer do
       'renderer_id/component_id/218c835cec343537589dbf1619532e4d/1'
     end
 
+    subject do
+      Alephant::Publisher::Writer.new(opts, msg)
+    end
+
     it "should write the correct lookup location" do
       Alephant::Cache.any_instance.stub(:put)
 
@@ -91,7 +91,9 @@ describe Alephant::Publisher::Writer do
         .any_instance
         .should_receive(:write)
         .with(
+          "component_id",
           {:variant=>"foo"},
+          1,
           expected_location
         )
     end
@@ -106,7 +108,7 @@ describe Alephant::Publisher::Writer do
     end
 
     after do
-      subject.write(msg)
+      subject.run!
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: alephant-publisher
 version: !ruby/object:Gem::Version
-  version: 0.1.4
+  version: 0.1.5
 platform: ruby
 authors:
 - Integralist
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2014-03-13 00:00:00.000000000 Z
+date: 2014-03-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -234,6 +234,20 @@ dependencies:
         version: '0'
   prerelease: false
   type: :runtime
+- !ruby/object:Gem::Dependency
+  name: peach
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :runtime
 - !ruby/object:Gem::Dependency
   name: alephant-support
   version_requirements: !ruby/object:Gem::Requirement
@@ -353,7 +367,6 @@ files:
 - lib/alephant/publisher/queue.rb
 - lib/alephant/publisher/render_mapper.rb
 - lib/alephant/publisher/version.rb
-- lib/alephant/publisher/write_operation.rb
 - lib/alephant/publisher/writer.rb
 - spec/fixtures/components/foo/models/bar.rb
 - spec/fixtures/components/foo/models/foo.rb
@@ -398,3 +411,4 @@ test_files:
 - spec/render_mapper_spec.rb
 - spec/spec_helper.rb
 - spec/writer_spec.rb
+has_rdoc:

data/lib/alephant/publisher/write_operation.rb

@@ -1,39 +0,0 @@
-require 'crimp'
-
-require 'alephant/sequencer'
-require 'alephant/support/parser'
-
-module Alephant
-  module Publisher
-    class WriteOperation
-      attr_reader :msg, :data, :options, :options_hash, :version, :batch_sequencer
-
-      def initialize(msg, opts)
-        @msg                = msg
-        @data               = Support::Parser.new(opts[:msg_vary_path]).parse(msg)
-        @options            = @data[:options]
-        @options_hash       = Crimp.signature(@options)
-        @renderer_id        = opts[:renderer_id]
-        @sequencer_opts     = opts[:sequencer_opts]
-        @batch_sequencer    = sequencer_for(@renderer_id, @options)
-        @version            = @batch_sequencer.sequence_id_from(msg)
-      end
-
-      def sequencer_for(id, options)
-        Sequencer.create(
-          @sequencer_opts[:table_name],
-          sequencer_id_from(id, options),
-          @sequencer_opts[:id_path]
-        )
-      end
-
-      private
-
-      def sequencer_id_from(id,options)
-        "#{id}/#{Crimp.signature(options)}"
-      end
-
-    end
-  end
-end
-