alephant-publisher 0.1.0 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 35bd92123f4d88d37f98565eada8b900944314f8
-  data.tar.gz: 6a878f67cbc84dc067ead0e14bdd7711451bf9b3
+  metadata.gz: be7e1421d4a9edd58b23cf1b67a39d0b77bc711f
+  data.tar.gz: 10abc065851746ceb2d215e61dae669653ac0432
 SHA512:
-  metadata.gz: bbae7c11ceff43c6e67a4a1d9e8180783ea7152f76b48cebf6f706632bb412a0963005876e664b16f43cb4b3ddc5afcdd1ce7365de2b394453f824f526a566db
-  data.tar.gz: ba18a78fa958da6e2d9cdb5313033f02a56ef7eee8ea999ad2814772686476ba900994cdbe49f18e8d9bdfdf233550ca6ed2a97f936e96c7c47ea992effc89f6
+  metadata.gz: 06619711d680043e4c6153d5be311bef2e095645faa41d9f1e4841468ec7622cf0b3f2b51f6ec69b09267e0e90e682526fd69664c488ee03f7fad5fc5e8e36f1
+  data.tar.gz: ca6f76631797d41a818c2844627534ab536eb7f846fe0cdfc4528922a1b02a0cfdda0a2381040e61442a23b5f03479182c387262439944ba414baa97df12de69

data/lib/alephant/publisher/models/write_operation.rb

@@ -0,0 +1,39 @@
+require 'crimp'
+
+require 'alephant/sequencer'
+require 'alephant/support/parser'
+
+module Alephant
+  module Publisher
+    class WriteOperation
+      attr_reader :msg, :data, :options, :options_hash, :version, :batch_sequencer
+
+      def initialize(msg, opts)
+        @msg                = msg
+        @data               = Support::Parser.new(opts[:msg_vary_path]).parse(msg)
+        @options            = @data[:options]
+        @options_hash       = Crimp.signature(@options)
+        @renderer_id        = opts[:renderer_id]
+        @sequencer_opts     = opts[:sequencer_opts]
+        @batch_sequencer    = sequencer_for(@renderer_id, @options)
+        @version            = @batch_sequencer.sequence_id_from(@msg)
+      end
+
+      def sequencer_for(id, options)
+        Sequencer.create(
+          @sequencer_opts[:table_name],
+          sequencer_id_from(id, options),
+          @sequencer_opts[:id_path]
+        )
+      end
+
+      private
+
+      def sequencer_id_from(id,options)
+        "#{id}/#{Crimp.signature(options)}"
+      end
+
+    end
+  end
+end
+

data/lib/alephant/publisher/models/writer.rb

@@ -1,4 +1,10 @@
-require 'crimp'
+require 'alephant/cache'
+require 'alephant/views'
+require 'alephant/renderer'
+require 'alephant/lookup'
+
+require 'alephant/publisher/models/write_operation'
+require 'alephant/publisher/models/render_mapper'
 
 module Alephant
   module Publisher
@@ -6,8 +12,6 @@ module Alephant
       attr_reader :mapper, :cache
 
       def initialize(opts)
-        @renderer_id = opts[:renderer_id]
-
         @cache = Cache.new(
           opts[:s3_bucket_id],
           opts[:s3_object_path]
@@ -19,32 +23,51 @@ module Alephant
         )
 
         @lookup_table_name = opts[:lookup_table_name]
+
+        @renderer_id = opts[:renderer_id]
+
+        @write_opts = {
+          :sequencer_opts => {
+            :table_name => opts[:sequencer_table_name],
+            :id_path    => opts[:sequence_id_path]
+          },
+          :msg_vary_path => opts[:msg_vary_id_path],
+          :renderer_id => opts[:renderer_id]
+        }
       end
 
-      def write(data, version = nil)
-        lookup = Lookup.create(@lookup_table_name)
+      def write(msg)
+        write_op = WriteOperation.new(msg, @write_opts)
 
-        mapper.generate(data).each do |id, r|
-          store(id, r.render, data[:options], version, lookup)
+        write_op.batch_sequencer.sequence(msg) do |msg|
+          mapper.generate(write_op.data).each do |component_id, renderer|
+            write_component(write_op, component_id, renderer)
+          end
         end
-
-        lookup.batch_process
       end
 
       private
 
-      def store(id, content, options, version, lookup)
-        location = location_for(
-          id,
-          Crimp.signature(options),
-          version
-        )
+      def write_component(write_op, component_id, renderer)
+        location = location_for(component_id, write_op.options, write_op.version)
+        component_sequencer = write_op.sequencer_for(component_id, write_op.options)
+
+        component_sequencer.sequence(write_op.msg) do |msg|
+          store(component_id, renderer.render, write_op.options, location)
+        end
+      end
 
+      def store(component_id, content, options, location)
         cache.put(location, content)
-        lookup.batch_write(options, location, id)
+        lookup_for(component_id).write(options, location)
+      end
+
+      def lookup_for(id)
+        Lookup.create(@lookup_table_name, id)
       end
 
-      def location_for(component_id, options_hash, version = nil)
+      def location_for(component_id, options, version = nil)
+        options_hash = Crimp.signature(options)
         base_name = "#{@renderer_id}/#{component_id}/#{options_hash}"
         version ? "#{base_name}/#{version}" : base_name
       end

data/lib/alephant/publisher/version.rb

@@ -1,5 +1,5 @@
 module Alephant
   module Publisher
-    VERSION = "0.1.0"
+    VERSION = "0.1.3"
   end
 end

data/lib/alephant/publisher.rb

@@ -1,15 +1,17 @@
 require_relative 'env'
 
-require 'alephant/support/parser'
-require 'alephant/sequencer'
-require 'alephant/cache'
+require 'java'
+
+# setup executor
+java_import 'java.util.concurrent.ThreadPoolExecutor'
+java_import 'java.util.concurrent.TimeUnit'
+java_import 'java.util.concurrent.LinkedBlockingQueue'
+java_import 'java.util.concurrent.FutureTask'
+java_import 'java.util.concurrent.Callable'
+
 require 'alephant/logger'
-require 'alephant/views'
-require 'alephant/renderer'
-require 'alephant/lookup'
 
 require "alephant/publisher/version"
-require 'alephant/publisher/models/render_mapper'
 require 'alephant/publisher/models/writer'
 require 'alephant/publisher/models/queue'
 
@@ -24,28 +26,53 @@ module Alephant
     private
 
     class Publisher
-      attr_reader :sequencer, :queue, :writer, :parser
+      attr_reader :queue
 
       def initialize(opts, logger)
         ::Alephant::Logger.set_logger(logger) unless logger.nil?
 
-        @parser = Support::Parser.new(
-          opts[:msg_vary_id_path]
+        @opts = opts
+        @queue = Queue.new(
+          opts[:sqs_queue_url]
         )
+      end
 
-        @sequencer = Sequencer.create(
-          opts[:sequencer_table_name],
-          opts[:sqs_queue_url],
-          opts[:sequence_id_path]
+      def run!
+        core_pool_size    = @opts[:renderer_pool_min_size] || 2
+        maximum_pool_size = @opts[:renderer_pool_max_size] || 4
+        keep_alive_time   = @opts[:render_keep_alive_time] || 300
+
+        executor = ThreadPoolExecutor.new(
+          core_pool_size,
+          maximum_pool_size,
+          keep_alive_time,
+          TimeUnit::SECONDS,
+          LinkedBlockingQueue.new
         )
 
-        @queue = Queue.new(
-          opts[:sqs_queue_url]
-        )
+        @queue.poll do |msg|
+          task = FutureTask.new(PublishTask.new(@opts, msg))
+          executor.execute(task)
+        end
+
+        executor.shutdown()
+      end
 
+    end
+
+    class PublishTask
+      include Callable
+
+      attr_reader :writer, :msg
+
+      def initialize(opts,msg)
+        @msg = msg
         @writer = Writer.new(
           opts.select do |k,v|
             [
+              :msg_vary_id_path,
+              :sequencer_table_name,
+              :sequence_id_path,
               :renderer_id,
               :s3_bucket_id,
               :s3_object_path,
@@ -56,21 +83,8 @@ module Alephant
         )
       end
 
-      def run!
-        Thread.new do
-          @queue.poll { |msg| receive(msg) }
-        end
-      end
-
-      def receive(msg)
-        write msg if sequencer.sequential?(msg)
-      end
-
-      private
-
-      def write(msg)
-        writer.write(parser.parse(msg), sequencer.sequence_id_from(msg))
-        sequencer.set_last_seen(msg)
+      def call
+        writer.write(msg)
       end
 
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: alephant-publisher
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.3
 platform: ruby
 authors:
 - Integralist
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2014-02-27 00:00:00.000000000 Z
+date: 2014-03-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -351,6 +351,7 @@ files:
 - lib/alephant/publisher.rb
 - lib/alephant/publisher/models/queue.rb
 - lib/alephant/publisher/models/render_mapper.rb
+- lib/alephant/publisher/models/write_operation.rb
 - lib/alephant/publisher/models/writer.rb
 - lib/alephant/publisher/version.rb
 - spec/fixtures/components/foo/models/bar.rb