coney_island 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c56c9ef5e0a701dbf1b5dc6d574158eb603c5b39
-  data.tar.gz: 78a9a731b07471b9591b43774805622601b6fc66
+  metadata.gz: fc75eee43a318be728acdf0b941206e100c76aa2
+  data.tar.gz: b94cbb25dcbfba834707f0a4fa1cc5e9056d435e
 SHA512:
-  metadata.gz: fcac6ce63736e212e822371306d7a4d57b5d0b5571f8716086d8d2d725f7846424b936e554593df95c56c2af1bd5912359ca6e5946ce40a905adae74ab44a083
-  data.tar.gz: f3fb209da6ffc4acec4ba8921b1523931f426e6bbdef62f9800562981bc14fecfde5ab7453da8f9bf0fdd295a45f5f42c7c72a7dea8e3f9e125cbac766435702
+  metadata.gz: 4eb709dc6cdf88655300a98fffec366b083756db5bb1cc4c60f268f8d1b444a7ba101dab5d0dfb4465cdb792ce65af9d287f4f7d7ef6ada3ae3facb3cea0f28a
+  data.tar.gz: 2bff91ac4362619b7eeb1e9111ae6c8983a1675199dd8b2c32e2c835fbc21e560233057d3c867dec5cf2a1c6324594a076124c36b314a4940f39bb8c1a913338

data/lib/coney_island.rb

@@ -16,8 +16,52 @@ module ConeyIsland
     @amqp_parameters
   end
 
-  def self.handle_connection
+  def self.tcp_connection_retries=(number)
+    @tcp_connection_retries = number
+  end
+
+  def self.tcp_connection_retries
+    @tcp_connection_retries
+  end
+
+  def self.tcp_connection_retry_limit=(limit)
+    @tcp_connection_retry_limit = limit
+  end
+  
+  def self.tcp_connection_retry_limit
+    @tcp_connection_retry_limit ||= 6
+  end
+
+  def self.tcp_connection_retry_interval=(interval)
+    @tcp_connection_retry_interval = interval
+  end
+  
+  def self.tcp_connection_retry_interval
+    @tcp_connection_retry_interval ||= 10
+  end
+
+  def self.handle_connection(log)
     @connection ||= AMQP.connect(self.amqp_parameters)
+  rescue AMQP::TCPConnectionFailed => e
+    self.tcp_connection_retries ||= 0
+      self.tcp_connection_retries += 1
+    if self.tcp_connection_retries >= self.tcp_connection_retry_limit
+      message = "Failed to connect to RabbitMQ #{self.tcp_connection_retry_limit} times, bailing out"
+      log.error(message)
+      self.poke_the_badger(e, {
+        code_source: 'ConeyIsland.handle_connection',
+        reason: message}
+      )
+    else
+      message = "Failed to connecto to RabbitMQ Attempt ##{self.tcp_connection_retries} time(s), trying again in #{self.tcp_connection_retry_interval} seconds..."
+      log.error(message)
+      self.poke_the_badger(e, {
+        code_source: 'ConeyIsland.handle_connection',
+        reason: message})
+      sleep(self.tcp_connection_retry_interval)
+      retry
+    end
+  else
     @channel  ||= AMQP::Channel.new(@connection)
     self.exchange = @channel.topic('coney_island')
   end
@@ -86,6 +130,18 @@ module ConeyIsland
   def self.submit(*args)
     ConeyIsland::Submitter.submit(*args)
   end
+
+  def self.poke_the_badger(message, context, attempts = 1)
+    Timeout::timeout(3) do
+      @notifier.notify(message, context)
+    end
+  rescue
+    if attempts <= 3
+      attempts += 1
+      retry
+    end
+  end
+
 end
 
 require 'coney_island/notifiers/honeybadger_notifier'

data/lib/coney_island/submitter.rb

@@ -18,10 +18,13 @@ module ConeyIsland
     end
 
     def self.submit!(args)
-      if @run_inline
-        self.handle_publish(args)
+      self.handle_connection unless @run_inline
+      if :all_cached_jobs == args
+        RequestStore.store[:jobs].delete_if do |job_args|
+          self.publish_job(job_args)
+        end
       else
-          self.handle_publish(args)
+        self.publish_job(args)
       end
     end
 
@@ -39,24 +42,38 @@ module ConeyIsland
 
     def self.handle_connection
       if ConeyIsland.single_amqp_connection?
-        ConeyIsland.handle_connection
+        Rails.logger.info("using single connection to RabbitMQ")
+        ConeyIsland.handle_connection(Rails.logger)
         @exchange = ConeyIsland.exchange
       else
-        @connection ||= AMQP.connect(self.amqp_parameters)
-        @channel  ||= AMQP::Channel.new(@connection)
-        @exchange = @channel.topic('coney_island')
+        self.submitter_connection
       end
     end
 
-    def self.handle_publish(args)
-      self.handle_connection unless @run_inline
-      if :all_cached_jobs == args
-        RequestStore.store[:jobs].delete_if do |job_args|
-          self.publish_job(job_args)
-        end
+    def self.submitter_connection
+      @connection ||= AMQP.connect(self.amqp_parameters)
+    rescue AMQP::TCPConnectionFailed => e
+      @tcp_connection_retries ||= 0
+        @tcp_connection_retries += 1
+      if @tcp_connection_retries >= 6
+        message = "Failed to connect to RabbitMQ 6 times, bailing out"
+        Rails.logger.error(message)
+        ConeyIsland.poke_the_badger(e, {
+          code_source: 'ConeyIsland::Submitter.submitter_connection',
+          reason: message}
+        )
       else
-        self.publish_job(args)
+        message = "Failed to connecto to RabbitMQ Attempt ##{@tcp_connection_retries} time(s), trying again in 10 seconds..."
+        Rails.logger.error(message)
+        ConeyIsland.poke_the_badger(e, {
+          code_source: 'ConeyIsland::Submitter.submitter_connection',
+          reason: message})
+        sleep(10)
+        retry
       end
+    else
+      @channel  ||= AMQP::Channel.new(@connection)
+      @exchange = @channel.topic('coney_island')
     end
 
     def self.publish_job(args)

data/lib/coney_island/version.rb

@@ -1,3 +1,3 @@
 module ConeyIsland
-  VERSION = "0.1.2"
+  VERSION = "0.1.3"
 end

data/lib/coney_island/worker.rb

@@ -66,17 +66,41 @@ module ConeyIsland
 
     def self.handle_connection
       if ConeyIsland.single_amqp_connection?
-        ConeyIsland.handle_connection
+        ConeyIsland.handle_connection(self.log)
         @exchange = ConeyIsland.exchange
         @channel = ConeyIsland.channel
       else
-        puts self.amqp_parameters
-        @connection ||= AMQP.connect(self.amqp_parameters)
-        @channel  ||= AMQP::Channel.new(@connection)
-        @exchange = @channel.topic('coney_island')
+        self.worker_connection
       end
     end
 
+    def self.worker_connection
+      @connection ||= AMQP.connect(self.amqp_parameters)
+    rescue AMQP::TCPConnectionFailed => e
+      @tcp_connection_retries ||= 0
+        @tcp_connection_retries += 1
+      if @tcp_connection_retries >= 6
+        message = "Failed to connect to RabbitMQ 6 times, bailing out"
+        self.log.error(message)
+        ConeyIsland.poke_the_badger(e, {
+          code_source: 'ConeyIsland::Worker.worker_connection',
+          reason: message}
+        )
+      else
+        message = "Failed to connecto to RabbitMQ Attempt ##{@tcp_connection_retries} time(s), trying again in 10 seconds..."
+        self.log.error(message)
+        ConeyIsland.poke_the_badger(e, {
+          code_source: 'ConeyIsland::Worker.worker_connection',
+          reason: message})
+        sleep(10)
+        retry
+      end
+    else
+      @channel  ||= AMQP::Channel.new(@connection)
+      @exchange = @channel.topic('coney_island')
+    end
+
+
     def self.start
       @child_count.times do
         child_pid = Process.fork
@@ -130,9 +154,9 @@ module ConeyIsland
           self.handle_job(metadata,args,job_id)
         end
       rescue Timeout::Error => e
-        self.poke_the_badger(e, {code_source: 'ConeyIsland', job_payload: args, reason: 'timeout in subscribe code before calling job method'})
+        ConeyIsland.poke_the_badger(e, {code_source: 'ConeyIsland', job_payload: args, reason: 'timeout in subscribe code before calling job method'})
       rescue Exception => e
-        self.poke_the_badger(e, {code_source: 'ConeyIsland', job_payload: args})
+        ConeyIsland.poke_the_badger(e, {code_source: 'ConeyIsland', job_payload: args})
         self.log.error("ConeyIsland code error, not application code:\n#{e.inspect}\nARGS: #{args}")
       end
     end
@@ -149,7 +173,7 @@ module ConeyIsland
           if self.job_attempts[job_id] >= 3
             self.log.error("Request #{job_id} timed out after #{timeout} seconds, bailing out after 3 attempts")
             self.finalize_job(metadata,job_id)
-            self.poke_the_badger(e, {work_queue: @ticket, job_payload: args, reason: 'Bailed out after 3 attempts'})
+            ConeyIsland.poke_the_badger(e, {work_queue: @ticket, job_payload: args, reason: 'Bailed out after 3 attempts'})
           else
             self.log.error("Request #{job_id} timed out after #{timeout} seconds on attempt number #{self.job_attempts[job_id]}, retrying...")
             self.job_attempts[job_id] += 1
@@ -157,7 +181,7 @@ module ConeyIsland
           end
         end
       rescue Exception => e
-        self.poke_the_badger(e, {work_queue: @ticket, job_payload: args})
+        ConeyIsland.poke_the_badger(e, {work_queue: @ticket, job_payload: args})
         self.log.error("Error executing #{args['klass']}##{args['method_name']} #{job_id} for id #{args['instance_id']} with args #{args}:")
         self.log.error(e.message)
         self.log.error(e.backtrace.join("\n"))
@@ -191,19 +215,6 @@ module ConeyIsland
       self.job_attempts.delete job_id
     end
 
-    def self.poke_the_badger(message, context, attempts = 1)
-      begin
-        Timeout::timeout(3) do
-          @notifier.notify(message, context)
-        end
-      rescue
-        if attempts <= 3
-          attempts += 1
-          self.poke_the_badger(message, context, attempts)
-        end
-      end
-    end
-
     def self.shutdown(signal)
       shutdown_time = Time.now
       @child_pids.each do |child_pid|

data/test/coney_island_test.rb

@@ -2,12 +2,26 @@ require 'test_helper'
 
 class ConeyIslandTest < MiniTest::Test
   describe "ConeyIsland running jobs" do
+
+    def force_tcp_error
+      lambda do |params|
+        @attempts ||= 0
+        @attempts += 1
+        if @attempts == 1
+          raise AMQP::TCPConnectionFailed.new({host: '127.0.0.1'})
+        else
+          return true
+        end
+      end
+    end
+
     it "runs inline" do
       ConeyIsland.run_inline
       my_array = []
       ConeyIsland.submit(TestModel, :add_to_list, args: [my_array])
       my_array.first.must_equal 'Added one!'
     end
+
     it "caches jobs" do
       ConeyIsland.run_inline
       my_array = []
@@ -19,5 +33,20 @@ class ConeyIslandTest < MiniTest::Test
       my_array.first.must_equal 'Added one!'
       ConeyIsland.stop_caching_jobs
     end
+
+    it "retries on TCP connection errors" do
+      ConeyIsland.stop_running_inline
+      ConeyIsland.tcp_connection_retry_interval = 0
+      @fake_channel = MiniTest::Mock.new
+      @fake_channel.expect :topic, nil, [String]
+      AMQP::Channel.stub(:new,@fake_channel) do
+        AMQP.stub(:connect, force_tcp_error) do
+          ConeyIsland.handle_connection(Logger.new(File.open(File::NULL, "w")))
+        end
+      end
+      @fake_channel.verify
+      ConeyIsland.tcp_connection_retries.must_equal 1
+    end
+
   end
 end

data/test/submitter_test.rb

@@ -17,17 +17,12 @@ class SubmitterTest < MiniTest::Test
       it "publishes the job to the message bus" do
         @exchange = Minitest::Mock.new
         @exchange.expect :publish, nil, [String,Hash]
-        @next_tick = Minitest::Mock.new
-        @next_tick.expect :call, nil, []
-        EventMachine.stub(:next_tick, @next_tick) do
-          ConeyIsland::Submitter.stub(:handle_connection, nil) do
-            ConeyIsland::Submitter.stub(:exchange, @exchange) do
-              ConeyIsland::Submitter.stop_running_inline
-              ConeyIsland::Submitter.submit(TestModel, :add_to_list, args: [[]])
-            end
+        ConeyIsland::Submitter.stub(:handle_connection, nil) do
+          ConeyIsland::Submitter.stub(:exchange, @exchange) do
+            ConeyIsland::Submitter.stop_running_inline
+            ConeyIsland::Submitter.submit(TestModel, :add_to_list, args: [[]])
           end
         end
-        @next_tick.verify
         @exchange.verify
       end
     end

data/test/worker_test.rb

@@ -13,7 +13,7 @@ class WorkerTest < MiniTest::Test
           ConeyIsland::Worker.handle_job(@metadata,{
             'klass' => 'TestModel',
             'method_name' => :take_too_long,
-            'timeout' => 0.01
+            'timeout' => 0.0001
             },'my_job_id')
         end
         ConeyIsland::Worker.job_attempts['my_job_id'].must_equal 3
@@ -21,7 +21,7 @@ class WorkerTest < MiniTest::Test
       it "sends other exeptions to a notification service" do
         @poke_the_badger = MiniTest::Mock.new
         @poke_the_badger.expect :call, nil, [Exception,Hash]
-        ConeyIsland::Worker.stub(:poke_the_badger,@poke_the_badger) do
+        ConeyIsland.stub(:poke_the_badger,@poke_the_badger) do
           ConeyIsland::Worker.handle_job(@metadata,{
             'klass' => 'TestModel',
             'method_name' => :throw_an_error

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: coney_island
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - Eric Draut
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-10-01 00:00:00.000000000 Z
+date: 2014-10-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -144,6 +144,8 @@ files:
 - test/dummy/config/locales/en.yml
 - test/dummy/config/routes.rb
 - test/dummy/config/secrets.yml
+- test/dummy/db/test.sqlite3
+- test/dummy/log/test.log
 - test/dummy/public/404.html
 - test/dummy/public/422.html
 - test/dummy/public/500.html
@@ -207,6 +209,8 @@ test_files:
 - test/dummy/config/routes.rb
 - test/dummy/config/secrets.yml
 - test/dummy/config.ru
+- test/dummy/db/test.sqlite3
+- test/dummy/log/test.log
 - test/dummy/public/404.html
 - test/dummy/public/422.html
 - test/dummy/public/500.html