coney_island 0.11.3 → 0.11.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: cb5f33066b9bc6d6eb023c3e2d9078badefe3847
-  data.tar.gz: 6360c5d5ee7865c643c000722de1848afc9c68db
+  metadata.gz: e89a4a37c2ee2c54d1971832c0a2116de95c3fe7
+  data.tar.gz: 695f0918f28bfb72651a51c0adde9981f87e5e36
 SHA512:
-  metadata.gz: 3379c592b226398c01c172ece55327704ef00efae5e8d0bd502073406ed241d9ef7273a4b5a22a0415c9292252d94baf247a492afff2c9921d8360b6915e03ce
-  data.tar.gz: dfb97960adf2014126aa7b66e5bf549d778e30d24a66706eaf357c5b5bfc936ea89eb9460af5fc0c7a47299fb47bb0ef1232c32d5135bbea766d2fdaf0f1fe0a
+  metadata.gz: fa2adee2194a6270b8b2e7d9cdfe37d6aa722e23e2ab7d7872d6702902d0120b4864b277928c7c1b969b97457ea59a96aff48d64affe557a21c81038346f9ee0
+  data.tar.gz: 6d4e7c6446c7c3addbbc9a8b4d6f6af741a2b2d21888544da8235e2d69a46b7e473363caf87f322780b7a67f458c8eb6f20b2539ff5820d93024a597bd851027

data/lib/coney_island.rb

@@ -30,12 +30,16 @@ module ConeyIsland
     @tcp_connection_retry_limit ||= 6
   end
 
-  def self.tcp_connection_retry_interval=(interval)
-    @tcp_connection_retry_interval = interval
+  def self.tcp_connection_retry_interval
+    self.tcp_connection_retry_seed ** self.tcp_connection_retries
   end
 
-  def self.tcp_connection_retry_interval
-    @tcp_connection_retry_interval ||= 10
+  def self.tcp_connection_retry_seed=(seed)
+    @tcp_connection_retry_seed = seed
+  end
+
+  def self.tcp_connection_retry_seed
+    @tcp_connection_retry_seed ||= 2
   end
 
   def self.notifier

data/lib/coney_island/version.rb

@@ -1,3 +1,3 @@
 module ConeyIsland
-  VERSION = "0.11.3"
+  VERSION = "0.11.4"
 end

data/lib/coney_island/worker.rb

@@ -117,59 +117,30 @@ module ConeyIsland
             self.log.info("Connected to AMQP broker. Running #{AMQP::VERSION}")
             connection.on_error do |conn, connection_close|
               self.log.error "Handling a connection-level exception."
+              self.log.error
               self.log.error "AMQP class id : #{connection_close.class_id}"
               self.log.error "AMQP method id: #{connection_close.method_id}"
               self.log.error "Status code   : #{connection_close.reply_code}"
               self.log.error "Error message : #{connection_close.reply_text}"
-              ConeyIsland.poke_the_badger({error_message: "ConeyIsland #{@ticket} Worker lost RabbitMQ Connection"}, connection_error: {
-                amqp_class: connection_close.class_id,
-                amqp_method: connection_close.method_id,
-                status_code: connection_close.reply_code,
-                message: connection_close.reply_text
-                })
             end
-            @channel = AMQP::Channel.new(connection)
-            @channel.on_error do |ch, channel_close|
-              self.log.error "Handling a channel-level exception."
-              self.log.error "AMQP class id : #{channel_close.class_id}"
-              self.log.error "AMQP method id: #{channel_close.method_id}"
-              self.log.error "Status code   : #{channel_close.reply_code}"
-              self.log.error "Error message : #{channel_close.reply_text}"
-              ConeyIsland.poke_the_badger({error_message: "ConeyIsland #{@ticket} Worker lost RabbitMQ Channel"}, connection_error: {
-                amqp_class: channel_close.class_id,
-                amqp_method: channel_close.method_id,
-                status_code: channel_close.reply_code,
-                message: channel_close.reply_text
-                })
-            end
-            @exchange = @channel.topic('coney_island')
             #Handle a lost connection to rabbitMQ
             connection.on_tcp_connection_loss do |connection, settings|
               # since we lost the connection, rabbitMQ will resend all jobs we didn't finish
               # so drop them and restart
-              unless @shutting_down
-                self.abandon_and_shutdown
-              end
-            end
-
-            #send a heartbeat every 15 seconds to avoid aggresive network configurations that close quiet connections
-            heartbeat_exchange = self.channel.fanout('coney_island_heartbeat')
-            EventMachine.add_periodic_timer(15) do
-              heartbeat_exchange.publish({:instance_name => @ticket})
-              self.handle_missing_children
+              self.log.warn("Lost rabbit connection, attempting to reconnect...")
+              connection.reconnect(true, 1)
+              self.initialize_rabbit(connection)
+              # unless @shutting_down
+              #   self.abandon_and_shutdown
+              # end
             end
 
-            self.channel.prefetch @prefetch_count
-            @queue = self.channel.queue(@full_instance_name, auto_delete: false, durable: true)
-            @queue.bind(self.exchange, routing_key: 'carousels.' + @ticket + '.#')
-            @queue.subscribe(:ack => true) do |metadata,payload|
-              self.handle_incoming_message(metadata,payload)
-            end
+            self.initialize_rabbit(connection)
           end
         end
-      rescue AMQP::TCPConnectionFailed => e
+      rescue AMQP::TCPConnectionFailed, AMQP::PossibleAuthenticationFailureError => e
         ConeyIsland.tcp_connection_retries ||= 0
-          ConeyIsland.tcp_connection_retries += 1
+        ConeyIsland.tcp_connection_retries += 1
         if ConeyIsland.tcp_connection_retries >= ConeyIsland.tcp_connection_retry_limit
           message = "Failed to connect to RabbitMQ #{ConeyIsland.tcp_connection_retry_limit} times, bailing out"
           self.log.error(message)
@@ -177,15 +148,43 @@ module ConeyIsland
             code_source: 'ConeyIsland::Worker.start',
             reason: message}
           )
+          self.abandon_and_shutdown
         else
           message = "Failed to connecto to RabbitMQ Attempt ##{ConeyIsland.tcp_connection_retries} time(s), trying again in #{ConeyIsland.tcp_connection_retry_interval} seconds..."
           self.log.error(message)
-          sleep(10)
+          sleep(ConeyIsland.tcp_connection_retry_interval)
           retry
         end
       end
     end
 
+    def self.initialize_rabbit(connection)
+      self.log.info('initializing rabbit connection with channel and queue...')
+      @channel = AMQP::Channel.new(connection)
+      channel.on_error do |ch, channel_close|
+        self.log.error "Handling a channel-level exception."
+        self.log.error
+        self.log.error "AMQP class id : #{channel_close.class_id}"
+        self.log.error "AMQP method id: #{channel_close.method_id}"
+        self.log.error "Status code   : #{channel_close.reply_code}"
+        self.log.error "Error message : #{channel_close.reply_text}"
+      end
+      @exchange = @channel.topic('coney_island')
+      #send a heartbeat every 15 seconds to avoid aggresive network configurations that close quiet connections
+      heartbeat_exchange = self.channel.fanout('coney_island_heartbeat')
+      EventMachine.add_periodic_timer(15) do
+        heartbeat_exchange.publish({:instance_name => @ticket})
+        self.handle_missing_children
+      end
+
+      self.channel.prefetch @prefetch_count
+      @queue = self.channel.queue(@full_instance_name, auto_delete: false, durable: true)
+      @queue.bind(self.exchange, routing_key: 'carousels.' + @ticket + '.#')
+      @queue.subscribe(:ack => true) do |metadata,payload|
+        self.handle_incoming_message(metadata,payload)
+      end
+    end
+
     def self.handle_incoming_message(metadata,payload)
       args = JSON.parse(payload)
       job = Job.new(metadata, args)
@@ -215,6 +214,7 @@ module ConeyIsland
     def self.shutdown(signal)
       @shutting_down = true
       @child_pids.each do |child_pid|
+        self.log("killing child #{child_pid}")
         Process.kill(signal, child_pid)
       end
       @queue.unsubscribe rescue nil

data/test/coney_island_test.rb

@@ -36,7 +36,7 @@ class ConeyIslandTest < MiniTest::Test
 
     it "retries on TCP connection errors" do
       ConeyIsland.stop_running_inline
-      ConeyIsland.tcp_connection_retry_interval = 0
+      ConeyIsland.tcp_connection_retry_seed = 0
       @fake_channel = MiniTest::Mock.new
       @fake_channel.expect :topic, nil, [String]
       @fake_channel.expect :topic, nil, [String]

data/test/dummy/log/test.log

@@ -32,3 +32,8 @@ ConeyIsland::Submitter.submit! about to iterate over this many jobs: 1
 Failed to connecto to RabbitMQ Attempt #1 time(s), trying again in 0 seconds...
 ConeyIsland::Submitter.submit! about to iterate over this many jobs: 1
 Failed to connecto to RabbitMQ Attempt #1 time(s), trying again in 0 seconds...
+ConeyIsland::Submitter.submit! about to iterate over this many jobs: 1
+ConeyIsland::Submitter.submit! about to iterate over this many jobs: 1
+Failed to connecto to RabbitMQ Attempt #1 time(s), trying again in 0 seconds...
+ConeyIsland::Submitter.submit! about to iterate over this many jobs: 1
+Failed to connecto to RabbitMQ Attempt #1 time(s), trying again in 0 seconds...

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: coney_island
 version: !ruby/object:Gem::Version
-  version: 0.11.3
+  version: 0.11.4
 platform: ruby
 authors:
 - Eric Draut
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-05-19 00:00:00.000000000 Z
+date: 2015-05-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails