alephant 0.0.9.8-java → 0.0.9.9-java

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9431f717738595e68a8a0c3b1bcd8f66cba57233
-  data.tar.gz: 1befa37fc85eb905ec085cdded713ce2192f1317
+  metadata.gz: 756986d1dd5f6db8d9dd0f75d26432af435bc915
+  data.tar.gz: 05a8a13c5c52efd12ae21744bbd2dd9aa1cc1f93
 SHA512:
-  metadata.gz: 2c7f72b93b915b929ffc5aaae66326d89e7cafbdf6f0ce48348ba5d7b8d1276d7e8448e87deb0d97dc5edcb7994136aa7a5d4c8c352d9729e4b44ef26d04e502
-  data.tar.gz: c8ac8b48eb497dad1d0b43eee8daec450a800ca87367c509abb5b502f3c6f7d8a4a2c2667f6b1647deb798f57f34fca52d3d62b53355463658c2c36675ebd7a1
+  metadata.gz: 53dbcfce2dfc411d514a519f1ef805aeccbb4cd8ec97f72c5c79b05a95872932779b7699dd208c0a5234c20747895d858dab3c74388a4b1a048088dd98e81a36
+  data.tar.gz: 8edf5c153d6da5691657ca275ce3b2b6e4960641be4e9e731edea571a60f88fd267b9dc25ef3e076930229eb414ed80b0e4ad391c474e31475b3411ac20284a0

data/lib/alephant.rb

@@ -1,5 +1,3 @@
-require 'aws-sdk'
-
 require_relative 'env'
 
 require 'alephant/models/logger'
@@ -7,6 +5,7 @@ require 'alephant/models/queue'
 require 'alephant/models/cache'
 require 'alephant/models/renderer'
 require 'alephant/models/multi_renderer'
+require 'alephant/models/sequence_table'
 require 'alephant/models/sequencer'
 require 'alephant/models/parser'
 
@@ -33,13 +32,7 @@ module Alephant
       set_opts(opts)
 
       @logger = ::Alephant.logger
-      @sequencer = Sequencer.new(
-        {
-          :table_name => @table_name
-        },
-        @sqs_queue_id
-      )
-
+      @sequencer = Sequencer.create(@table_name, @sqs_queue_id, @sequence_id)
       @queue = Queue.new(@sqs_queue_id)
       @cache = Cache.new(@s3_bucket_id, @s3_object_path)
       @multi_renderer = MultiRenderer.new(@component_id, @view_path)
@@ -59,9 +52,9 @@ module Alephant
     def receive(msg)
       @logger.info("Alephant.receive: with id #{msg.id} and body digest: #{msg.md5}")
 
-      if @sequencer.sequential?(msg, @sequence_id)
+      if @sequencer.sequential?(msg)
         write @parser.parse msg.body
-        @sequencer.set_last_seen(msg, @sequence_id)
+        @sequencer.set_last_seen(msg)
       else
         @logger.warn("Alephant.receive: out of sequence message received #{msg.id} (discarded)")
       end

data/lib/alephant/models/sequence_table.rb

@@ -0,0 +1,99 @@
+require 'aws-sdk'
+require 'thread'
+require 'timeout'
+
+module Alephant
+  module Sequencer
+    class SequenceTable
+      attr_reader :table_name
+
+      TIMEOUT = 120
+      DEFAULT_CONFIG = {
+        :write_units => 5,
+        :read_units => 10,
+      }
+      SCHEMA = {
+        :hash_key => {
+          :key => :string,
+          :value => :string
+        }
+      }
+
+      def initialize(table_name, config = DEFAULT_CONFIG)
+        @mutex = Mutex.new
+        @dynamo_db = AWS::DynamoDB.new
+        @table_name = table_name
+        @config = config
+      end
+
+      def create
+        @mutex.synchronize do
+          ensure_table_exists
+          ensure_table_active
+        end
+      end
+
+      def table
+        @table ||= @dynamo_db.tables[@table_name]
+      end
+
+      def sequence_for(ident)
+        rows = batch_get_value_for(ident)
+        rows.count >= 1 ? rows.first['value'].to_i : 0
+      end
+
+      def set_sequence_for(ident,value)
+        @mutex.synchronize do
+          AWS::DynamoDB::BatchWrite.new.tap { |batch|
+            batch.put(
+              table_name,
+              [:key => ident,:value => value]
+            )
+          }.process!
+        end
+      end
+
+      def delete_item!(ident)
+        table.items[ident].delete
+      end
+
+      private
+      def batch_get_value_for(ident)
+        table.batch_get(['value'],[ident],batch_get_opts)
+      end
+
+      def batch_get_opts
+        { :consistent_read => true }
+      end
+
+      def ensure_table_exists
+        create_dynamodb_table unless table.exists?
+      end
+
+      def ensure_table_active
+        sleep_until_table_active unless table_active?
+      end
+
+      def create_dynamodb_table
+        @table = @dynamo_db.tables.create(
+          @table_name,
+          @config[:read_units],
+          @config[:write_units],
+          SCHEMA
+        )
+      end
+
+      def table_active?
+        table.status == :active
+      end
+
+      def sleep_until_table_active
+        begin
+          Timeout::timeout(TIMEOUT) do
+            sleep 1 until table_active?
+          end
+        end
+      end
+    end
+  end
+end

data/lib/alephant/models/sequencer.rb

@@ -1,91 +1,61 @@
-require 'aws-sdk'
 require 'jsonpath'
 
 module Alephant
-  class Sequencer
-    attr_reader :id, :table_name, :table_conf
+  module Sequencer
+    @@sequence_tables = {}
 
-    def table_conf_defaults
-      {
-        :write_units => 5,
-        :read_units => 10,
-        :schema => {
-          :hash_key => {
-            :key => :string,
-            :value => :string
-          }
-        }
-      }
+    def self.create(table_name, ident, jsonpath = nil)
+      @@sequence_tables[table_name] ||= SequenceTable.new(table_name)
+      Sequencer.new(@@sequence_tables[table_name], ident, jsonpath)
     end
 
-    def initialize(opts, id)
-      @logger = ::Alephant.logger
+    class Sequencer
+      attr_reader :ident, :jsonpath
 
-      dynamo_db = AWS::DynamoDB.new
+      def initialize(sequence_table, id, sequence_path = nil)
+        @mutex = Mutex.new
+        @sequence_table = sequence_table
+        @jsonpath = sequence_path
+        @ident = id
 
-      @id = id
-      @table_name = opts[:table_name]
-      @table_conf = opts[:table_conf] || table_conf_defaults
-      @table = dynamo_db.tables[@table_name]
-
-      begin
-        sleep_until_table_active
-      rescue AWS::DynamoDB::Errors::ResourceNotFoundException
-        @logger.error("Sequencer.initialize: DynamoDB resource was not found.")
-
-        @table = dynamo_db.tables.create(
-          @table_name,
-          @table_conf[:read_units],
-          @table_conf[:write_units],
-          @table_conf[:schema]
-        )
+        ::Alephant.logger.info("Sequencer.initialize: with id #{@ident}")
+        @sequence_table.create
+      end
 
-        @logger.info("Sequencer.initialize: Creating table with name #{@table_name}, read units #{@table_conf[:read_units]}, write units #{@table_conf[:write_units]}, schema #{@table_conf[:schema]}")
+      def sequential?(data)
+        get_last_seen < sequence_id_from(data)
+      end
 
-        sleep_until_table_active
+      def delete!
+        @sequence_table.delete_item!(ident)
       end
 
-      @logger.info("Sequencer.initialize: end with id #{@id}")
-    end
+      def set_last_seen(data)
+        last_seen_id = sequence_id_from(data)
 
-    def sequential?(data, jsonpath = nil)
-      get_last_seen < get_sequence_id_from(data, jsonpath)
-    end
+        @sequence_table.set_sequence_for(ident, last_seen_id)
+        ::Alephant.logger.info("Sequencer.set_last_seen: #{ident}:#{last_seen_id}")
+      end
 
-    def set_last_seen(data, jsonpath)
-      last_seen_id = get_sequence_id_from(data, jsonpath)
+      def get_last_seen
+        @sequence_table.sequence_for(ident)
+      end
 
-      batch = AWS::DynamoDB::BatchWrite.new
-      batch.put(@table_name, [:key => @id,:value => last_seen_id])
-      batch.process!
-      @logger.info("Sequencer.set_last_seen: id #{id} and last_seen_id #{last_seen_id}")
-    end
+      private
+      def sequence_id_from(data)
+        jsonpath.nil? ?
+          default_sequence_id_for(data) :
+          sequence_from_jsonpath_for(data)
+      end
 
-    def get_sequence_id_from(data, jsonpath)
-      jsonpath.nil? ?
-        data.body['sequence_id'].to_i :
+      def sequence_from_jsonpath_for(data)
         JsonPath.on(data.body, jsonpath).first
-    end
+      end
 
-    def get_last_seen
-      begin
-        @table.batch_get(
-          ['value'],
-          [@id],
-          {
-            :consistent_read => true
-          }
-        ).first["value"].to_i
-      rescue Exception => e
-        trace = e.backtrace.join('\n')
-        @logger.error("Sequencer.get_last_seen: id #{id}\nmessage: #{e.message}\ntrace: #{trace}")
-        0
+      def default_sequence_id_for(data)
+        data.body['sequence_id'].to_i
       end
-    end
 
-    def sleep_until_table_active
-      sleep 1 until @table.status == :active
     end
-
   end
 end

data/lib/alephant/version.rb

@@ -1,3 +1,3 @@
 module Alephant
-  VERSION = "0.0.9.8"
+  VERSION = "0.0.9.9"
 end

data/lib/env.rb

@@ -1,3 +1,5 @@
+$LOAD_PATH << File.dirname(__FILE__)
+
 require 'aws-sdk'
 require 'yaml'
 

data/spec/alephant_spec.rb

@@ -10,7 +10,7 @@ describe Alephant::Alephant do
       cache          = double()
       multi_renderer = double()
 
-      Alephant::Sequencer.any_instance.stub(:initialize).and_return(sequencer)
+      Alephant::Sequencer.stub(:create).and_return(sequencer)
       Alephant::Queue.any_instance.stub(:initialize).and_return(queue)
       Alephant::Cache.any_instance.stub(:initialize).and_return(cache)
       Alephant::MultiRenderer.any_instance.stub(:initialize).and_return(multi_renderer)
@@ -49,14 +49,15 @@ describe Alephant::Alephant do
     end
 
     context "initializes @sequencer" do
-      it "with Sequencer.new({ :table_name => :table_name }, @sqs_queue_id)" do
+      it "with Sequencer.create(:table_name, @sqs_queue_id)" do
         Alephant::Sequencer
-          .should_receive(:new)
-          .with({ :table_name => :table_name }, :sqs_queue_id)
+          .should_receive(:create)
+          .with(:table_name, :sqs_queue_id, :sequence_id)
 
         instance = subject.new({
           :table_name   => :table_name,
-          :sqs_queue_id => :sqs_queue_id
+          :sqs_queue_id => :sqs_queue_id,
+          :sequence_id => :sequence_id
         })
       end
     end
@@ -107,7 +108,7 @@ describe Alephant::Alephant do
       cache          = double()
       multi_renderer = double()
 
-      Alephant::Sequencer.any_instance.stub(:initialize).and_return(sequencer)
+      Alephant::Sequencer.stub(:create).and_return(sequencer)
       Alephant::Queue.any_instance.stub(:initialize).and_return(queue)
       Alephant::Cache.any_instance.stub(:initialize).and_return(cache)
       Alephant::MultiRenderer.any_instance.stub(:initialize).and_return(multi_renderer)
@@ -144,22 +145,11 @@ describe Alephant::Alephant do
       cache          = double()
       multi_renderer = double()
 
-      Alephant::Sequencer.any_instance.stub(:initialize).and_return(sequencer)
       Alephant::Queue.any_instance.stub(:initialize).and_return(queue)
       Alephant::Cache.any_instance.stub(:initialize).and_return(cache)
       Alephant::MultiRenderer.any_instance.stub(:initialize).and_return(multi_renderer)
     end
 
-    it "takes json as an argument" do
-      instance = subject.new
-
-      msg = double()
-      msg.stub(:body).and_return('notjson')
-
-      expect { JsonPath.on(msg.body, '$.foo') }
-        .to raise_error(MultiJson::LoadError);
-    end
-
     it "writes data to cache if sequential order is true" do
       data = "{ \"foo\":\"bar\" }"
 
@@ -168,17 +158,22 @@ describe Alephant::Alephant do
       msg.stub(:md5).and_return(:md5)
       msg.stub(:body).and_return(data)
 
-      instance = subject.new
 
-      Alephant::Sequencer
+      Alephant::Sequencer::Sequencer
         .any_instance
         .stub(:sequential?)
         .and_return(true)
 
-      Alephant::Sequencer
+      Alephant::Sequencer::SequenceTable
+        .any_instance
+        .stub(:create)
+
+      Alephant::Sequencer::Sequencer
         .any_instance
         .stub(:set_last_seen)
 
+      instance = subject.new
+
       instance
         .should_receive(:write)
         .with(JSON.parse(data, :symbolize_names => true))
@@ -192,7 +187,7 @@ describe Alephant::Alephant do
       sequencer = double()
       queue     = double()
 
-      Alephant::Sequencer
+      Alephant::Sequencer::Sequencer
         .any_instance
         .stub(:initialize)
         .and_return(sequencer)

data/spec/logger_spec.rb

@@ -7,7 +7,7 @@ describe Alephant::LogSystem do
     cache     = double()
     renderer  = double()
 
-    Alephant::Sequencer.any_instance.stub(:initialize).and_return(sequencer)
+    Alephant::Sequencer.stub(:create).and_return(sequencer)
     Alephant::Queue.any_instance.stub(:initialize).and_return(queue)
     Alephant::Cache.any_instance.stub(:initialize).and_return(cache)
     Alephant::Renderer.any_instance.stub(:initialize).and_return(renderer)

data/spec/sequencer_spec.rb

@@ -1,124 +1,107 @@
 require 'spec_helper'
 
-describe Alephant::Sequencer do
-  subject { Alephant::Sequencer }
 
-  describe "initialize(opts, id)" do
+describe Alephant::Sequencer do
+  let(:ident) { :ident }
+  let(:jsonpath) { :jsonpath }
 
-    it "sets @id, @table_name and @table_conf" do
-      AWS::DynamoDB.any_instance.stub(:initialize).and_return({
-        :tables => {
-          :status => :active
-        }
-      })
+  describe ".create(table_name, ident, jsonpath)" do
+    it "should return a Sequencer" do
+      Alephant::Sequencer::SequenceTable.any_instance.stub(:create)
+      expect(subject.create(:table_name, ident, jsonpath)).to be_a Alephant::Sequencer::Sequencer
+    end
+  end
 
-      Alephant::Sequencer.any_instance.stub(:sleep_until_table_active)
+  describe Alephant::Sequencer::Sequencer do
+    let(:data)     { double() }
+    let(:last_seen) { 42 }
+    let(:sequence_table) { double().tap { |o| o.stub(:create) } }
+    subject { Alephant::Sequencer::Sequencer.new(sequence_table, ident, jsonpath) }
+
+    describe "#initialize(opts, id)" do
+      it "sets @jsonpath, @ident" do
+        expect(subject.jsonpath).to eq(jsonpath)
+        expect(subject.ident).to eq(ident)
+      end
 
-      instance = subject.new({
-        :table_name => :table_name,
-        :table_conf => :table_conf
-      }, :sqs_queue_id)
+      it "calls create on sequence_table" do
+        table = double()
+        table.should_receive(:create)
 
-      expect(instance.id).to eq(:sqs_queue_id)
-      expect(instance.table_name).to eq(:table_name)
-      expect(instance.table_conf).to eq(:table_conf)
+        Alephant::Sequencer::Sequencer.new(table, ident, jsonpath)
+      end
     end
 
-    context "sleep_until_table_active raises" do
-      context "AWS::DynamoDB::Errors::ResourceNotFoundException" do
-        it "dynamo_db.tables.create(@table_name, opts) then sleep_until_table_active" do
-          opts = {
-              :table_name => :table_name,
-              :table_conf => {
-                :read_units => :read_units,
-                :write_units => :write_units,
-                :schema => :schema
-              }
-          }
-
-          table_collection = double()
-          table_collection.should_receive(:create).with(
-            opts[:table_name],
-            opts[:table_conf][:read_units],
-            opts[:table_conf][:write_units],
-            opts[:table_conf][:schema]
-          )
-
-          AWS::DynamoDB.any_instance.stub(:tables).and_return({},table_collection)
-
-          Alephant::Sequencer
-            .any_instance
-            .stub(:sleep_until_table_active).and_yield do
-              @times_called ||= 0
-              raise AWS::DynamoDB::Errors::ResourceNotFoundException if @times_called == 0
-              @times_called += 1
-            end
-
-          subject.new(opts, :id)
-        end
+    describe "#get_last_seen" do
+      it "returns sequence_table.sequence_for(ident)" do
+        table = double()
+        table.stub(:create)
+        table.should_receive(:sequence_for).with(ident).and_return(:expected_value)
+
+        expect(
+          Alephant::Sequencer::Sequencer.new(table, ident).get_last_seen
+        ).to eq(:expected_value)
       end
     end
-  end
-
-  describe "sequential?(data, jsonpath = nil)" do
-    let(:jsonpath) { '$.sequence_id' }
-    let(:instance) { subject.new }
-    let(:id_value) { 0 }
-    let(:data)     { double() }
 
-    before(:each) do
-      Alephant::Sequencer
-        .any_instance.stub(:initialize)
-        .and_return(double())
+    describe "#set_last_seen(data)" do
+      before(:each) do
+        Alephant::Sequencer::Sequencer.any_instance.stub(:sequence_id_from).and_return(last_seen)
+      end
 
-      Alephant::Sequencer
-        .any_instance
-        .stub(:get_last_seen)
-        .and_return(1)
+      it "calls set_sequence_for(ident, last_seen)" do
+        table = double()
+        table.stub(:create)
+        table.should_receive(:set_sequence_for).with(ident, last_seen)
 
-      data.stub(:body).and_return('sequence_id' => id_value)
+        Alephant::Sequencer::Sequencer.new(table, ident).set_last_seen(data)
+      end
     end
 
-    context "jsonpath provided" do
-      context "in sequence" do
-        let(:id_value) { 2 }
+    describe "#sequential?(data, jsonpath)" do
 
-        it "looks up data using jsonpath (returns true)" do
-          in_sequence = instance.sequential?(data, jsonpath)
-          expect(in_sequence).to eq(true)
-        end
+      before(:each) do
+        Alephant::Sequencer::Sequencer.any_instance.stub(:get_last_seen).and_return(1)
+        data.stub(:body).and_return('sequence_id' => id_value)
       end
 
-      context "out of sequence" do
-        let(:id_value) { 0 }
+      context "jsonpath = '$.sequence_id'" do
+        let(:jsonpath) { '$.sequence_id' }
+        subject { Alephant::Sequencer::Sequencer.new(sequence_table, :ident, jsonpath) }
+        context "sequential" do
+          let(:id_value) { 2 }
+          it "is true" do
+            expect(subject.sequential?(data)).to be_true
+          end
+        end
 
-        it "looks up data using jsonpath (returns false)" do
-          in_sequence = instance.sequential?(data, jsonpath)
-          expect(in_sequence).to eq(false)
+        context "nonsequential" do
+          let(:id_value) { 0 }
+          it "is false" do
+            expect(subject.sequential?(data)).to be_false
+          end
         end
       end
-    end
 
-    context "jsonpath NOT provided" do
-      context "in sequence" do
-        let(:id_value) { 2 }
+      context "jsonpath = nil" do
+        let(:jsonpath) { nil }
+        subject { Alephant::Sequencer::Sequencer.new(sequence_table, :ident, jsonpath) }
 
-        it "looks up data using a fallback key (returns true)" do
-          in_sequence = instance.sequential?(data)
-          expect(in_sequence).to eq(true)
+        context "sequential" do
+          let(:id_value) { 2 }
+          it "is true" do
+            expect(subject.sequential?(data)).to be_true
+          end
         end
-      end
-
-      context "out of sequence" do
-        let(:id_value) { 0 }
 
-        it "looks up data using a fallback key (returns false)" do
-          in_sequence = instance.sequential?(data)
-          expect(in_sequence).to eq(false)
+        context "nonsequential" do
+          let(:id_value) { 0 }
+          it "is false" do
+            expect(subject.sequential?(data)).to be_false
+          end
         end
       end
-    end
 
+    end
   end
 end
-

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: alephant
 version: !ruby/object:Gem::Version
-  version: 0.0.9.8
+  version: 0.0.9.9
 platform: java
 authors:
 - Robert Kenny
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2014-02-10 00:00:00.000000000 Z
+date: 2014-02-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -234,6 +234,7 @@ files:
 - lib/alephant/models/parser.rb
 - lib/alephant/models/queue.rb
 - lib/alephant/models/renderer.rb
+- lib/alephant/models/sequence_table.rb
 - lib/alephant/models/sequencer.rb
 - lib/alephant/preview/server.rb
 - lib/alephant/preview/template.rb