rpush 2.2.0 → 2.3.0.rc1

data/lib/rpush/daemon/adm/delivery.rb

@@ -40,6 +40,9 @@ module Rpush
           handle_rate_limited(error)
         rescue Rpush::RetryableError => error
           handle_retryable(error)
+        rescue SocketError => error
+          mark_retryable(@notification, Time.now + 10.seconds, error)
+          raise
         rescue StandardError => error
           mark_failed(error)
           raise
@@ -176,7 +179,7 @@ module Rpush
         end
 
         def retry_message
-          "Notification #{@notification.id} will be retired after #{@notification.deliver_after.strftime("%Y-%m-%d %H:%M:%S")} (retry #{@notification.retries})."
+          "Notification #{@notification.id} will be retired after #{@notification.deliver_after.strftime('%Y-%m-%d %H:%M:%S')} (retry #{@notification.retries})."
         end
 
         def do_post(registration_id)

data/lib/rpush/daemon/apns/delivery.rb

@@ -12,6 +12,9 @@ module Rpush
           @connection.write(batch_to_binary)
           mark_batch_delivered
           describe_deliveries
+        rescue Rpush::Daemon::TcpConnectionError => error
+          mark_batch_retryable(Time.now + 10.seconds, error)
+          raise
         rescue StandardError => error
           mark_batch_failed(error)
           raise

data/lib/rpush/daemon/apns/feedback_receiver.rb

@@ -1,3 +1,5 @@
+# encoding: UTF-8
+
 module Rpush
   module Daemon
     module Apns
@@ -22,7 +24,7 @@ module Rpush
 
         def start
           return if Rpush.config.push
-          log_info("APNs Feedback Receiver started.")
+          Rpush.logger.info("[#{@app.name}] Starting feedback receiver... ", true)
           @interruptible_sleep.start
 
           @thread = Thread.new do
@@ -34,6 +36,8 @@ module Rpush
 
             Rpush::Daemon.store.release_connection
           end
+
+          puts ANSI.green { '✔' } if Rpush.config.foreground
         end
 
         def stop
@@ -70,7 +74,7 @@ module Rpush
         end
 
         def create_feedback(failed_at, device_token)
-          formatted_failed_at = failed_at.strftime("%Y-%m-%d %H:%M:%S UTC")
+          formatted_failed_at = failed_at.strftime('%Y-%m-%d %H:%M:%S UTC')
           log_info("[FeedbackReceiver] Delivery failed at #{formatted_failed_at} for #{device_token}.")
 
           feedback = Rpush::Daemon.store.create_apns_feedback(failed_at, device_token, @app)

data/lib/rpush/daemon/app_runner.rb

@@ -3,8 +3,6 @@
 module Rpush
   module Daemon
     class AppRunner
-      extend Term::ANSIColor
-
       extend Reflectable
       include Reflectable
       include Loggable
@@ -29,9 +27,10 @@ module Rpush
 
       def self.start_app(app)
         Rpush.logger.info("[#{app.name}] Starting #{pluralize(app.connections, 'dispatcher')}... ", true)
-        @runners[app.id] = new(app)
-        @runners[app.id].start
-        puts green('✔') if Rpush.config.foreground
+        runner = @runners[app.id] = new(app)
+        runner.start_dispatchers
+        puts ANSI.green { '✔' } if Rpush.config.foreground
+        runner.start_loops
       rescue StandardError => e
         @runners.delete(app.id)
         Rpush.logger.error("[#{app.name}] Exception raised during startup. Notifications will not be delivered for this app.")
@@ -90,6 +89,7 @@ module Rpush
       end
 
       attr_reader :app
+      delegate :size, to: :queue, prefix: true
 
       def initialize(app)
         @app = app
@@ -97,9 +97,13 @@ module Rpush
         @dispatcher_loops = []
       end
 
-      def start
+      def start_dispatchers
         app.connections.times { @dispatcher_loops.push(new_dispatcher_loop) }
-        start_loops
+      end
+
+      def start_loops
+        @loops = service.loop_instances(@app)
+        @loops.map(&:start)
       end
 
       def stop
@@ -151,21 +155,12 @@ module Rpush
         log_info(JSON.pretty_generate(runner_details))
       end
 
-      def queue_size
-        queue.size
-      end
-
       def num_dispatcher_loops
         @dispatcher_loops.size
       end
 
       private
 
-      def start_loops
-        @loops = service.loop_instances(@app)
-        @loops.map(&:start)
-      end
-
       def stop_loops
         @loops.map(&:stop)
         @loops = []

data/lib/rpush/daemon/batch.rb

@@ -34,6 +34,15 @@ module Rpush
         Rpush::Daemon.store.mark_retryable(notification, deliver_after, persist: false)
       end
 
+      def mark_all_retryable(deliver_after)
+        @mutex.synchronize do
+          @retryable[deliver_after] = @notifications
+        end
+        each_notification do |notification|
+          Rpush::Daemon.store.mark_retryable(notification, deliver_after, persist: false)
+        end
+      end
+
       def mark_delivered(notification)
         @mutex.synchronize do
           @delivered << notification

data/lib/rpush/daemon/delivery.rb

@@ -4,10 +4,13 @@ module Rpush
       include Reflectable
       include Loggable
 
-      def mark_retryable(notification, deliver_after)
+      def mark_retryable(notification, deliver_after, error = nil)
         if notification.fail_after && notification.fail_after < Time.now
-          @batch.mark_failed(notification, nil, "Notification failed to be delivered before #{notification.fail_after.strftime("%Y-%m-%d %H:%M:%S")}.")
+          @batch.mark_failed(notification, nil, "Notification failed to be delivered before #{notification.fail_after.strftime('%Y-%m-%d %H:%M:%S')}.")
         else
+          if error
+            log_warn("Will retry notification #{notification.id} after #{deliver_after.strftime('%Y-%m-%d %H:%M:%S')} due to error (#{error.class.name}, #{error.message})")
+          end
           @batch.mark_retryable(notification, deliver_after)
         end
       end
@@ -16,6 +19,11 @@ module Rpush
         mark_retryable(notification, Time.now + 2**(notification.retries + 1))
       end
 
+      def mark_batch_retryable(deliver_after, error)
+        log_warn("Will retry #{@batch.notifications.size} notifications after #{deliver_after.strftime('%Y-%m-%d %H:%M:%S')} due to error (#{error.class.name}, #{error.message})")
+        @batch.mark_all_retryable(deliver_after)
+      end
+
       def mark_delivered
         @batch.mark_delivered(@notification)
       end

data/lib/rpush/daemon/delivery_error.rb

@@ -19,9 +19,9 @@ module Rpush
 
     def ==(other)
       other.is_a?(DeliveryError) && \
-      other.code == code && \
-      other.notification_id == notification_id && \
-      other.to_s == to_s
+        other.code == code && \
+        other.notification_id == notification_id && \
+        other.to_s == to_s
     end
   end
 end

data/lib/rpush/daemon/dispatcher/apns_tcp.rb

@@ -28,7 +28,7 @@ module Rpush
 
         def dispatch(payload)
           @dispatch_mutex.synchronize do
-            @delivery_class.new(@app, connection, payload.batch).perform
+            @delivery_class.new(@app, @connection, payload.batch).perform
             record_batch(payload.batch)
           end
         end
@@ -60,13 +60,15 @@ module Rpush
 
         def check_for_error
           begin
-            return unless connection.select(SELECT_TIMEOUT)
-          rescue Errno::EBADF
-            # Connection closed, daemon is shutting down.
+            # On Linux, select returns nil from a dropped connection.
+            # On OS X, Errno::EBADF is raised following a Errno::EADDRNOTAVAIL from the write call.
+            return unless @connection.select(SELECT_TIMEOUT)
+          rescue SystemCallError
+            # Connection closed.
             return
           end
 
-          tuple = connection.read(ERROR_TUPLE_BYTES)
+          tuple = @connection.read(ERROR_TUPLE_BYTES)
           @dispatch_mutex.synchronize { handle_error_response(tuple) }
         end
 
@@ -78,22 +80,20 @@ module Rpush
             handle_disconnect
           end
 
-          log_error('Error received, reconnecting...')
-          connection.reconnect
+          log_error("Lost connection to #{@connection.host}:#{@connection.port}, reconnecting...")
+          @connection.reconnect_with_rescue
         ensure
           delivered_buffer.clear
         end
 
         def handle_disconnect
-          log_error('The APNs disconnected without returning an error. Marking all notifications delivered via this connection as failed.')
-          reason = 'The APNs disconnected without returning an error. This may indicate you are using an invalid certificate.'
-          Rpush::Daemon.store.mark_ids_failed(delivered_buffer, nil, reason, Time.now)
-          delivered_buffer.each { |id| reflect(:notification_id_failed, @app, id, nil, reason) }
+          log_error("The APNs disconnected before any notifications could be delivered. This usually indicates you are using an invalid certificate.") if delivered_buffer.size == 0
         end
 
         def handle_error(code, notification_id)
           failed_pos = delivered_buffer.index(notification_id)
           description = APNS_ERRORS[code.to_i] || "Unknown error code #{code.inspect}. Possible Rpush bug?"
+          log_error(description + " (#{code})")
           Rpush::Daemon.store.mark_ids_failed([notification_id], code, description, Time.now)
           reflect(:notification_id_failed, @app, notification_id, code, description)
 

data/lib/rpush/daemon/dispatcher/tcp.rb

@@ -6,24 +6,16 @@ module Rpush
           @app = app
           @delivery_class = delivery_class
           @host, @port = options[:host].call(@app)
+          @connection = Rpush::Daemon::TcpConnection.new(@app, @host, @port)
         end
 
         def dispatch(payload)
-          @delivery_class.new(@app, connection, payload.notification, payload.batch).perform
+          @delivery_class.new(@app, @connection, payload.notification, payload.batch).perform
         end
 
         def cleanup
           @connection.close if @connection
         end
-
-        protected
-
-        def connection
-          return @connection if defined? @connection
-          connection = Rpush::Daemon::TcpConnection.new(@app, @host, @port)
-          connection.connect
-          @connection = connection
-        end
       end
     end
   end

data/lib/rpush/daemon/gcm/delivery.rb

@@ -5,7 +5,7 @@ module Rpush
       class Delivery < Rpush::Daemon::Delivery
         include MultiJsonHelper
 
-        host = ENV["RPUSH_GCM_HOST"] || "https://android.googleapis.com"
+        host = 'https://android.googleapis.com'
         GCM_URI = URI.parse("#{host}/gcm/send")
         UNAVAILABLE_STATES = %w(Unavailable InternalServerError)
         INVALID_REGISTRATION_ID_STATES = %w(InvalidRegistration MismatchSenderId NotRegistered InvalidPackageName)
@@ -19,6 +19,9 @@ module Rpush
 
         def perform
           handle_response(do_post)
+        rescue SocketError => error
+          mark_retryable(@notification, Time.now + 10.seconds, error)
+          raise
         rescue StandardError => error
           mark_failed(error)
           raise
@@ -47,7 +50,6 @@ module Rpush
 
         def ok(response)
           results = process_response(response)
-
           handle_successes(results.successes)
 
           if results.failures.any?
@@ -136,7 +138,7 @@ module Rpush
         end
 
         def retry_message
-          "Notification #{@notification.id} will be retried after #{@notification.deliver_after.strftime("%Y-%m-%d %H:%M:%S")} (retry #{@notification.retries})."
+          "Notification #{@notification.id} will be retried after #{@notification.deliver_after.strftime('%Y-%m-%d %H:%M:%S')} (retry #{@notification.retries})."
         end
 
         def do_post
@@ -155,7 +157,7 @@ module Rpush
           @registration_ids = registration_ids
         end
 
-        def process(failure_partitions = {})
+        def process(failure_partitions = {}) # rubocop:disable Metrics/AbcSize
           @successes = []
           @failures = Failures.new
           failure_partitions.each_key do |category|
@@ -178,13 +180,13 @@ module Rpush
               end
             end
           end
-          failures.total_fail = failures.count == @registration_ids.count
+          failures.all_failed = failures.count == @registration_ids.count
         end
       end
 
       class Failures < Hash
         include Enumerable
-        attr_writer :total_fail, :description
+        attr_writer :all_failed, :description
 
         def initialize
           super[:all] = []
@@ -202,10 +204,14 @@ module Rpush
           @description ||= describe
         end
 
+        def any?
+          self[:all].any?
+        end
+
         private
 
         def describe
-          if @total_fail
+          if @all_failed
             error_description = "Failed to deliver to all recipients."
           else
             index_list = map { |item| item[:index] }

data/lib/rpush/daemon/signal_handler.rb

@@ -40,7 +40,7 @@ module Rpush
                 Rpush.logger.error("Unhandled signal: #{signal}")
               end
             rescue StandardError => e
-              Rpush.logger.error("Error raised when hndling signal '#{signal}'")
+              Rpush.logger.error("Error raised when handling signal '#{signal}'")
               Rpush.logger.error(e)
             end
           end
@@ -48,13 +48,15 @@ module Rpush
       end
 
       def self.handle_hup
-        Rpush.logger.info("Received HUP signal.")
+        Rpush.logger.reopen
+        Rpush.logger.info('Received HUP signal.')
+        Rpush::Daemon.store.reopen_log
         Synchronizer.sync
         Feeder.wakeup
       end
 
       def self.handle_usr2
-        Rpush.logger.info("Received USR2 signal.")
+        Rpush.logger.info('Received USR2 signal.')
         AppRunner.debug
       end
 

data/lib/rpush/daemon/store/active_record.rb

@@ -10,6 +10,14 @@ module Rpush
 
         DEFAULT_MARK_OPTIONS = { persist: true }
 
+        def initialize
+          reopen_log
+        end
+
+        def reopen_log
+          ::ActiveRecord::Base.logger = Rpush.logger.internal_logger
+        end
+
         def app(id)
           Rpush::Client::ActiveRecord::App.find(id)
         end
@@ -53,6 +61,8 @@ module Rpush
         end
 
         def mark_ids_retryable(ids, deliver_after)
+          return if ids.empty?
+
           with_database_reconnect_and_retry do
             Rpush::Client::ActiveRecord::Notification.where(id: ids).update_all(['processing = ?, delivered = ?, delivered_at = ?, failed = ?, failed_at = ?, retries = retries + 1, deliver_after = ?', false, false, nil, false, nil, deliver_after])
           end
@@ -72,6 +82,8 @@ module Rpush
         end
 
         def mark_batch_delivered(notifications)
+          return if notifications.empty?
+
           now = Time.now
           ids = []
           notifications.each do |n|
@@ -111,6 +123,8 @@ module Rpush
         end
 
         def mark_ids_failed(ids, code, description, time)
+          return if ids.empty?
+
           with_database_reconnect_and_retry do
             Rpush::Client::ActiveRecord::Notification.where(id: ids).update_all(['processing = ?, delivered = ?, delivered_at = NULL, failed = ?, failed_at = ?, error_code = ?, error_description = ?', false, false, true, time, code, description])
           end

data/lib/rpush/daemon/store/interface.rb

@@ -7,7 +7,8 @@ module Rpush
                           :mark_failed, :mark_batch_failed, :create_apns_feedback,
                           :create_gcm_notification, :create_adm_notification, :update_app,
                           :update_notification, :release_connection,
-                          :all_apps, :app, :mark_ids_failed, :mark_ids_retryable]
+                          :all_apps, :app, :mark_ids_failed, :mark_ids_retryable,
+                          :reopen_log]
 
         def self.check(klass)
           missing = PUBLIC_METHODS - klass.instance_methods.map(&:to_sym)

data/lib/rpush/daemon/store/redis.rb

@@ -103,6 +103,9 @@ module Rpush
         def release_connection
         end
 
+        def reopen_log
+        end
+
         private
 
         def create_gcm_like_notification(notification, attrs, data, registration_ids, deliver_after, app) # rubocop:disable ParameterLists

data/lib/rpush/daemon/tcp_connection.rb

@@ -6,7 +6,14 @@ module Rpush
       include Reflectable
       include Loggable
 
-      attr_accessor :last_write
+      OSX_TCP_KEEPALIVE = 0x10 # Defined in <netinet/tcp.h>
+      KEEPALIVE_INTERVAL = 5
+      KEEPALIVE_IDLE = 5
+      KEEPALIVE_MAX_FAIL_PROBES = 1
+      TCP_ERRORS = [SystemCallError, OpenSSL::OpenSSLError, IOError]
+
+      attr_accessor :last_touch
+      attr_reader :host, :port
 
       def self.idle_period
         30.minutes
@@ -18,12 +25,14 @@ module Rpush
         @port = port
         @certificate = app.certificate
         @password = app.password
-        written
+        @connected = false
+        touch
       end
 
       def connect
         @ssl_context = setup_ssl_context
         @tcp_socket, @ssl_socket = connect_socket
+        @connected = true
       end
 
       def close
@@ -33,38 +42,45 @@ module Rpush
       end
 
       def read(num_bytes)
-        @ssl_socket.read(num_bytes)
+        @ssl_socket.read(num_bytes) if @ssl_socket
       end
 
       def select(timeout)
-        IO.select([@ssl_socket], nil, nil, timeout)
+        IO.select([@ssl_socket], nil, nil, timeout) if @ssl_socket
       end
 
       def write(data)
+        connect unless @connected
         reconnect_idle if idle_period_exceeded?
 
         retry_count = 0
 
         begin
           write_data(data)
-        rescue Errno::EPIPE, Errno::ETIMEDOUT, OpenSSL::SSL::SSLError, IOError => e
+        rescue *TCP_ERRORS => e
           retry_count += 1
 
           if retry_count == 1
-            log_error("Lost connection to #{@host}:#{@port} (#{e.class.name}), reconnecting...")
+            log_error("Lost connection to #{@host}:#{@port} (#{e.class.name}, #{e.message}), reconnecting...")
             reflect(:tcp_connection_lost, @app, e)
           end
 
           if retry_count <= 3
-            reconnect
+            reconnect_with_rescue
             sleep 1
             retry
           else
-            raise TcpConnectionError, "#{@app.name} tried #{retry_count - 1} times to reconnect but failed (#{e.class.name})."
+            raise TcpConnectionError, "#{@app.name} tried #{retry_count - 1} times to reconnect but failed (#{e.class.name}, #{e.message})."
           end
         end
       end
 
+      def reconnect_with_rescue
+        reconnect
+      rescue StandardError => e
+        log_error(e)
+      end
+
       def reconnect
         close
         @tcp_socket, @ssl_socket = connect_socket
@@ -78,17 +94,17 @@ module Rpush
       end
 
       def idle_period_exceeded?
-        Time.now - last_write > self.class.idle_period
+        Time.now - last_touch > self.class.idle_period
       end
 
       def write_data(data)
         @ssl_socket.write(data)
         @ssl_socket.flush
-        written
+        touch
       end
 
-      def written
-        self.last_write = Time.now
+      def touch
+        self.last_touch = Time.now
       end
 
       def setup_ssl_context
@@ -99,21 +115,35 @@ module Rpush
       end
 
       def connect_socket
+        touch
         check_certificate_expiration
 
         tcp_socket = TCPSocket.new(@host, @port)
-        tcp_socket.setsockopt(Socket::SOL_SOCKET, Socket::SO_KEEPALIVE, 1)
-        tcp_socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1)
+        tcp_socket.setsockopt(Socket::SOL_SOCKET, Socket::SO_KEEPALIVE, true)
+        tcp_socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, true)
+
+        # Linux
+        if [:SOL_TCP, :TCP_KEEPIDLE, :TCP_KEEPINTVL, :TCP_KEEPCNT].all? { |c| Socket.const_defined?(c) }
+          tcp_socket.setsockopt(Socket::SOL_TCP, Socket::TCP_KEEPIDLE, KEEPALIVE_IDLE)
+          tcp_socket.setsockopt(Socket::SOL_TCP, Socket::TCP_KEEPINTVL, KEEPALIVE_INTERVAL)
+          tcp_socket.setsockopt(Socket::SOL_TCP, Socket::TCP_KEEPCNT, KEEPALIVE_MAX_FAIL_PROBES)
+        end
+
+        # OSX
+        if RUBY_PLATFORM =~ /darwin/
+          tcp_socket.setsockopt(Socket::IPPROTO_TCP, OSX_TCP_KEEPALIVE, KEEPALIVE_IDLE)
+        end
+
         ssl_socket = OpenSSL::SSL::SSLSocket.new(tcp_socket, @ssl_context)
         ssl_socket.sync = true
         ssl_socket.connect
         [tcp_socket, ssl_socket]
-      rescue StandardError => error
+      rescue *TCP_ERRORS => error
         if error.message =~ /certificate revoked/i
           log_warn('Certificate has been revoked.')
           reflect(:ssl_certificate_revoked, @app, error)
         end
-        raise
+        raise TcpConnectionError, "#{error.class.name}, #{error.message}"
       end
 
       def check_certificate_expiration
@@ -128,7 +158,7 @@ module Rpush
       end
 
       def certificate_msg(msg)
-        time = @ssl_context.cert.not_after.utc.strftime("%Y-%m-%d %H:%M:%S UTC")
+        time = @ssl_context.cert.not_after.utc.strftime('%Y-%m-%d %H:%M:%S UTC')
         "Certificate #{msg} at #{time}."
       end