coney_island 0.7.5 → 0.7.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f039e0c6dcd7965fd53e9563264440a0f2cfbd51
-  data.tar.gz: 2769bb8e3e684f0b0d3b5b0cce0fb2033dd1c116
+  metadata.gz: d091ef6d980cadba61712e9b2a597cae791df0e4
+  data.tar.gz: f6a0f197c6c441ebed031fae08ae35c6069a9129
 SHA512:
-  metadata.gz: b89c9a90de86274b2c006e95827de7e7c1909e02a5fae9620b2895d98bcdc69b0b8ce595dc3dd74acd5d1793e0f7a12cad28f92b41c7fb0c9b8c3bc26434ee94
-  data.tar.gz: 2cbb3dc9b4dba357951481935a13921c4b3e68d5c6b8d853f8a68140e023734996b0eccb832a45a52358f71097e95f80a8039483615e4be9b86c419f19633268
+  metadata.gz: 2176dd0cc32b96638bb949fe6572356fe4fdaa91a3bfc81dcf988f771445c2bed0cfa2ec7a6aaa6263dbb89923df3bdd74108455594c15c9a7d6254616032f3c
+  data.tar.gz: 50ffdea1c5394ee0738dd08b463b41022646e0317c6e0d4f43f6045de6020b43fc0a36cc930e33b464d1ee6ee464fb0b188750e6706aeb9e4d45c3cecb406b8b

data/lib/coney_island/version.rb

@@ -1,3 +1,3 @@
 module ConeyIsland
-  VERSION = "0.7.5"
+  VERSION = "0.7.7"
 end

data/lib/coney_island/worker.rb

@@ -21,6 +21,10 @@ module ConeyIsland
       @job_attempts ||= {}
     end
 
+    def self.clear_job_attempts
+      @job_attempts = {}
+    end
+
     def self.initialize_background
       ENV['NEW_RELIC_AGENT_ENABLED'] = 'false'
       ENV['NEWRELIC_ENABLE'] = 'false'
@@ -88,10 +92,11 @@ module ConeyIsland
             self.shutdown('TERM')
           end
 
-          AMQP.connect(self.amqp_parameters) do |connection|
+          AMQP.connect(self.amqp_parameters.merge(heartbeat: 15)) do |connection|
             connection.on_tcp_connection_loss do |connection, settings|
-              # reconnect in 10 seconds, without enforcement
-              connection.reconnect(false, 10)
+              # since we lost the connection, rabbitMQ will resend all jobs we didn't finish
+              # so drop them and restart
+              self.abandon_and_shutdown
             end
             self.log.info("Connected to AMQP broker. Running #{AMQP::VERSION}")
             @channel = AMQP::Channel.new(connection)
@@ -101,6 +106,7 @@ module ConeyIsland
             heartbeat_exchange = self.channel.fanout('coney_island_heartbeat')
             EventMachine.add_periodic_timer(15) do
               heartbeat_exchange.publish({:instance_name => @ticket})
+              self.handle_missing_children
             end
 
             self.channel.prefetch @prefetch_count
@@ -205,6 +211,22 @@ module ConeyIsland
       self.job_attempts.delete job_id
     end
 
+    def self.handle_missing_children
+      @child_pids.each do |child_pid|
+        begin
+          Process.kill 0, child_pid
+        rescue Errno::ESRCH => e
+          @child_pids.push Process.spawn("bundle exec coney_island #{@ticket}")
+        end
+      end
+    end
+
+    def self.abandon_and_shutdown
+      self.log.info("Lost RabbitMQ connection, abandoning current jobs and restarting")
+      self.clear_job_attempts
+      self.shutdown('TERM')
+    end
+
     def self.shutdown(signal)
       shutdown_time = Time.now
       @child_pids.each do |child_pid|

data/test/dummy/log/test.log

@@ -6,3 +6,7 @@ ConeyIsland::Submitter.submit! about to iterate over this many jobs: 1
 Failed to connecto to RabbitMQ Attempt #1 time(s), trying again in 0 seconds...
 ConeyIsland::Submitter.submit! about to iterate over this many jobs: 1
 Failed to connecto to RabbitMQ Attempt #1 time(s), trying again in 0 seconds...
+Failed to connecto to RabbitMQ Attempt #1 time(s), trying again in 0 seconds...
+ConeyIsland::Submitter.submit! about to iterate over this many jobs: 1
+Failed to connecto to RabbitMQ Attempt #1 time(s), trying again in 0 seconds...
+ConeyIsland::Submitter.submit! about to iterate over this many jobs: 1

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: coney_island
 version: !ruby/object:Gem::Version
-  version: 0.7.5
+  version: 0.7.7
 platform: ruby
 authors:
 - Eric Draut