google-cloud-pubsub 3.3.0 → 3.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 500ef51067933f91a1f6134ba3849fae4159f83d79e3f3dc6d6ef9237006e69e
-  data.tar.gz: 32be664ffdeed8864e5be601131fc784805fc82564cd4ca03f355894f7c2f588
+  metadata.gz: 2706c0520b9cd5aba014d233fddf25b3c5265cc35a30b65719ed91647a271bef
+  data.tar.gz: d6f9a3e8c9b74a08bd344582ce7c7e9b5a4bc03cebe0243ffb09ac17cdcc0afd
 SHA512:
-  metadata.gz: a75dd428cfa30499d42c0bd5b4a5726cd2c5029768adf0c027177c7e4ccbbfd982115b459bf310fe96cf2597fb90aae351a4a60157c0b436ea5253c5f3fa53cb
-  data.tar.gz: 7160f84ef94817b60a6e53d57e37c586dd44c5ecbb31b9f8e181be42a67f4b6d3c2a735ac0dc70631c4f340ba632748fbe7018c9bad0dec1acc0061fa9272018
+  metadata.gz: 426ac33d83d07732c22854216b673def03cf2b5813718a5e57428546f0e5499178d6848e75473da2e71169b79d76246879663d94f1ce8415758737efe6f5817a
+  data.tar.gz: b5def0275205fa9bd13ea1f463ce50992903ba2c423817f22401675ecfc1eb55e1ef57f9e97178d83659ee40fdc1e87a21075aa4180dfed177526e487d043327

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Release History
 
+### 3.4.0 (2026-07-29)
+
+#### Features
+
+* implement streaming keep-alive logic ([#34653](https://github.com/googleapis/google-cloud-ruby/issues/34653)) 
+
 ### 3.3.0 (2026-06-11)
 
 #### Features

data/lib/google/cloud/pubsub/message_listener/keepalive_monitor.rb

@@ -0,0 +1,159 @@
+# Copyright 2026 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "concurrent"
+
+module Google
+  module Cloud
+    module PubSub
+      class MessageListener
+        ##
+        # @private
+        # Monitors gRPC streaming connection health via periodic bi-directional keep-alive pings and pongs.
+        #
+        # Tracks ping and pong timestamps over the active gRPC stream to detect silent connection drops or freezes,
+        # and triggers a stream restart when server responses exceed the configured pong deadline.
+        # This prevents firewalls and load balancers from dropping idle connections during periods of low
+        # message publisher volume, which minimizes message delivery latency by maintaining a healthy lease.
+        #
+        # For users, this ensures highly reliable, resilient, and responsive message delivery operations.
+        # It directly prevents issues where Google Cloud Pub/Sub subscribers may unexpectedly stall
+        # or stop receiving messages for extended periods due to unnotified intermediate network disruptions
+        # (e.g., TCP halfway drops). By proactively maintaining the connection's liveness, subscribers
+        # avoid sudden latency spikes and lost leases.
+        class KeepaliveMonitor
+          # Default interval in seconds between keep-alive ping requests.
+          DEFAULT_INTERVAL = 30.0
+          # Default deadline in seconds to receive a keep-alive pong response.
+          DEFAULT_DEADLINE = 15.0
+          # Divisor applied to keep-alive interval to calculate monitor polling interval (1/5th of interval).
+          MONITOR_DIVISOR = 5.0
+          # Minimum floor in seconds (10ms) for the monitor polling interval to prevent CPU spinning.
+          MIN_MONITOR_INTERVAL = 0.01
+
+          # Initializes the KeepaliveMonitor.
+          #
+          # @param [Stream] stream The parent streaming pull connection to monitor.
+          # @param [Float] interval The interval in seconds between keep-alive pings.
+          # @param [Float] deadline The deadline in seconds for receiving a pong.
+          def initialize stream, interval: DEFAULT_INTERVAL, deadline: DEFAULT_DEADLINE
+            @stream       = stream
+            @interval     = interval
+            @deadline     = deadline
+            @last_ping_at = nil
+            @last_pong_at = nil
+            @ping_task    = nil
+            @monitor_task = nil
+          end
+
+          # Starts the background keep-alive ping and liveness monitor timer tasks.
+          #
+          # @return [KeepaliveMonitor] self for chaining.
+          def start
+            @stream.synchronize do
+              return self if @ping_task
+
+              @ping_task = Concurrent::TimerTask.new(execution_interval: [@interval, MIN_MONITOR_INTERVAL].max) do
+                send_ping!
+              end
+              @ping_task.execute
+
+              @monitor_task = Concurrent::TimerTask.new(
+                execution_interval: [@interval / MONITOR_DIVISOR, MIN_MONITOR_INTERVAL].max
+              ) do
+                check_liveness!
+              end
+              @monitor_task.execute
+            end
+            self
+          end
+
+          # Shuts down and cleans up the background keep-alive and monitor timer tasks.
+          #
+          # @return [KeepaliveMonitor] self for chaining.
+          def stop
+            @stream.synchronize do
+              @ping_task&.shutdown
+              @ping_task = nil
+              @monitor_task&.shutdown
+              @monitor_task = nil
+            end
+            self
+          end
+
+          # Records the initial stream handshake by setting ping and pong timestamps to the current monotonic time.
+          #
+          # @return [void]
+          def record_handshake!
+            @stream.synchronize do
+              now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+              @last_ping_at = now
+              @last_pong_at = now
+            end
+          end
+
+          # Records a received pong by updating the last pong timestamp to the current monotonic time.
+          #
+          # @return [void]
+          def record_pong!
+            @stream.synchronize do
+              @stream.log_info "received keepAlive pong from stream for subscription #{@stream.subscriber.subscription_name}"
+              @last_pong_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+            end
+          end
+
+          # Evaluates stream state and sends a bi-directional keep-alive ping if appropriate.
+          #
+          # @return [void]
+          def send_ping!
+            @stream.synchronize do
+              # Push unconditional keep-alive pings only when the stream is actively open and request queue is active.
+              # Note: ACKs are sent via unary RPCs (TimedUnaryBuffer), not over this stream.
+              return unless @stream.stream_open? && !@stream.stopped? && @stream.request_queue_active?
+
+              @stream.log_info "sending keepAlive to stream for subscription #{@stream.subscriber.subscription_name}"
+              # Only advance @last_ping_at if the previous ping was successfully ponged.
+              # If a pong is outstanding (@last_pong_at < @last_ping_at), freezing @last_ping_at preserves the start time
+              # of the un-ponged cycle so check_liveness! can accurately evaluate the elapsed deadline.
+              @last_ping_at = Process.clock_gettime(Process::CLOCK_MONOTONIC) if @last_pong_at.nil? || @last_ping_at.nil? || @last_pong_at >= @last_ping_at
+              @stream.send_ping_request!
+            end
+          end
+
+          # Checks whether a pong response has been received within the configured deadline,
+          # and triggers a stream restart if the deadline has been exceeded.
+          #
+          # @return [void]
+          def check_liveness!
+            @stream.synchronize do
+              # Do not check pong deadline if paused (client flow control inventory full).
+              # When paused, background_run waits on condition variable and stops calling enum.next,
+              # so incoming server pongs sit buffered in gRPC and last_pong_at stays un-updated.
+              return unless @stream.stream_open? && @last_ping_at && @last_pong_at && !@stream.stopped? && !@stream.paused?
+
+              now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+              if now - @last_ping_at >= @deadline && @last_pong_at < @last_ping_at
+                elapsed_pong = (now - @last_pong_at).round(2)
+                @stream.log_error "Keep-alive pong not received within #{@deadline}s (last pong #{elapsed_pong}s ago); restarting stream."
+                @stream.restart_stream_for_timeout!
+              end
+            end
+          end
+        end
+      end
+    end
+
+    Pubsub = PubSub unless const_defined? :Pubsub
+  end
+end

data/lib/google/cloud/pubsub/message_listener/stream.rb

@@ -15,6 +15,7 @@
 require "google/cloud/pubsub/message_listener/sequencer"
 require "google/cloud/pubsub/message_listener/enumerator_queue"
 require "google/cloud/pubsub/message_listener/inventory"
+require "google/cloud/pubsub/message_listener/keepalive_monitor"
 require "google/cloud/pubsub/service"
 require "google/cloud/errors"
 require "monitor"
@@ -49,6 +50,25 @@ module Google
           # @private exactly_once_delivery_enabled.
           attr_reader :exactly_once_delivery_enabled
 
+          ##
+          # @private KeepaliveMonitor.
+          attr_reader :keepalive_monitor
+
+          # Initial backoff delay in seconds when reconnecting after a transient stream disconnection.
+          INITIAL_RECONNECT_DELAY = 1.0
+
+          # Maximum backoff delay in seconds for stream reconnection attempts.
+          MAX_RECONNECT_DELAY = 60.0
+
+          # Exponential backoff multiplier applied to reconnect delay on successive retry attempts.
+          RECONNECT_BACKOFF_MULTIPLIER = 1.5
+
+          # Inventory capacity ratio (80%) below which a flow-control paused stream will unpause.
+          UNPAUSE_INVENTORY_RATIO = 0.8
+
+          # The keep-alive streaming pull protocol version sent in the initial request during handshake.
+          PROTOCOL_VERSION = 1
+
           ##
           # @private Create an empty Subscriber::Stream object.
           def initialize subscriber
@@ -59,7 +79,12 @@ module Google
             @request_queue = nil
             @stopped = nil
             @paused  = nil
+            # Condition variable used to cooperatively pause the background thread during flow control
+            # Governed by the Stream's MonitorMixin `synchronize` lock.
             @pause_cond = new_cond
+            # Condition variable used to cooperatively sleep the background thread during connection retries
+            # Governed by the Stream's MonitorMixin `synchronize` lock.
+            @backoff_cond = new_cond
             @exactly_once_delivery_enabled = false
 
             @inventory = Inventory.new self, **@subscriber.stream_inventory
@@ -68,17 +93,9 @@ module Google
 
             @callback_thread_pool = Concurrent::ThreadPoolExecutor.new max_threads: @subscriber.callback_threads
 
-            @stream_keepalive_task = Concurrent::TimerTask.new(
-              execution_interval: 30
-            ) do
-              # push empty request every 30 seconds to keep stream alive
-              unless inventory.empty?
-                subscriber.service.logger.log :info, "subscriber-streams" do
-                  "sending keepAlive to stream for subscription #{@subscriber.subscription_name}"
-                end
-                push Google::Cloud::PubSub::V1::StreamingPullRequest.new
-              end
-            end.execute
+            @keepalive_monitor = KeepaliveMonitor.new self
+            @stream_open = false
+            @reconnect_delay = nil
           end
 
           def start
@@ -86,6 +103,7 @@ module Google
               break if @background_thread
 
               @inventory.start
+              @keepalive_monitor.start
 
               start_streaming!
             end
@@ -107,6 +125,9 @@ module Google
               # Signal to the background thread that we are stopped.
               @stopped = true
               @pause_cond.broadcast
+              @backoff_cond.broadcast
+
+              @keepalive_monitor.stop
 
               # Now that the reception thread is stopped, immediately stop the
               # callback thread pool. All queued callbacks will see the stream
@@ -124,6 +145,10 @@ module Google
             synchronize { @stopped }
           end
 
+          def stream_open?
+            synchronize { @stream_open }
+          end
+
           def paused?
             synchronize { @paused }
           end
@@ -139,6 +164,38 @@ module Google
             self
           end
 
+
+          def request_queue_active?
+            !@request_queue.nil?
+          end
+
+          def send_ping_request!
+            push Google::Cloud::PubSub::V1::StreamingPullRequest.new
+          end
+
+          def restart_stream_for_timeout!
+            synchronize do
+              @stream_open = false
+              # Push self as a stream-closing sentinel to @request_queue.
+              # When EnumeratorQueue#each pops the sentinel object, it terminates the request enumerator,
+              # cleanly sending an HTTP/2 END_STREAM flag and unblocking the write-side gRPC C-core pipeline.
+              @request_queue&.push self
+              @background_thread&.raise RestartStream
+            end
+          end
+
+          def log_info msg
+            subscriber.service.logger.log :info, "subscriber-streams" do
+              msg
+            end
+          end
+
+          def log_error msg
+            subscriber.service.logger.log :error, "subscriber-streams" do
+              msg
+            end
+          end
+
           ##
           # @private
           def acknowledge *messages, &callback
@@ -219,6 +276,15 @@ module Google
 
           # rubocop:disable all
 
+          def backoff_and_wait!
+            @reconnect_delay = @reconnect_delay ? [@reconnect_delay * RECONNECT_BACKOFF_MULTIPLIER, MAX_RECONNECT_DELAY].min : INITIAL_RECONNECT_DELAY
+            synchronize do
+              # Disable liveness checker during backoff sleep to prevent monitor task from interrupting and double-backing off
+              @stream_open = false
+              @backoff_cond.wait(@reconnect_delay + rand(0.0..0.5)) unless @stopped
+            end
+          end
+
           def background_run
             synchronize do
               # Don't allow a stream to restart if already stopped
@@ -240,16 +306,32 @@ module Google
               # Always create a new request queue
               @request_queue = EnumeratorQueue.new self
               @request_queue.push initial_input_request
-              old_queue.each { |obj| @request_queue.push obj }
+              old_queue.each do |obj|
+                @request_queue.push obj unless obj == self
+              end
             end
 
             # Call the StreamingPull API to get the response enumerator
             options = { :"metadata" => { :"x-goog-request-params" =>  @subscriber.subscription_name } }
+            # Temporarily disable the liveness monitor while establishing the gRPC connection
+            # to prevent check_liveness! from evaluating stale timestamps during the handshake.
+            synchronize do
+              @stream_open = false
+            end
             enum = @subscriber.service.streaming_pull @request_queue.each, options
             subscriber.service.logger.log :info, "subscriber-streams" do
               "rpc: streamingPull, subscription: #{@subscriber.subscription_name}, stream opened"
             end
 
+            # Once the stream handshake completes, initialize ping/pong timestamps to monotonic now
+            # and mark @stream_open = true under synchronization.
+            # This ensures @keepalive_monitor evaluates liveness against a fresh window and prevents
+            # false-positive disconnect detections from stale timestamps prior to reconnecting.
+            synchronize do
+              @keepalive_monitor.record_handshake!
+              @stream_open = true
+            end
+
             loop do
               synchronize do
                 if @paused && !@stopped
@@ -264,8 +346,17 @@ module Google
               begin
                 # Cannot synchronize the enumerator, causes deadlock
                 response = enum.next
-                new_exactly_once_delivery_enabled = response&.subscription_properties&.exactly_once_delivery_enabled
+                synchronize do
+                  @keepalive_monitor.record_pong!
+                  # Reset backoff delay only after successfully reading a frame from enum.next.
+                  # If the connection drops immediately upon reading, @reconnect_delay is preserved.
+                  @reconnect_delay = nil
+                end
                 received_messages = response.received_messages
+                # Skip processing properties and inventory on Pong frames (empty received_messages).
+                # Subscription properties on keep-alive Pongs are not valid.
+                next if received_messages.empty?
+                new_exactly_once_delivery_enabled = response&.subscription_properties&.exactly_once_delivery_enabled
 
                 # Use synchronize so changes happen atomically
                 synchronize do
@@ -310,11 +401,13 @@ module Google
               "#{status_code}; will be retried."
             end
             # Restart the stream with an incremental back for a retriable error.
+            backoff_and_wait!
             retry
           rescue RestartStream
             subscriber.service.logger.log :info, "subscriber-streams" do
               "Subscriber stream for subscription #{@subscriber.subscription_name} has ended; will be retried."
             end
+            backoff_and_wait!
             retry
           rescue StandardError => e
             subscriber.service.logger.log :error, "subscriber-streams" do
@@ -322,6 +415,7 @@ module Google
             end
             @subscriber.error! e
 
+            backoff_and_wait!
             retry
           end
 
@@ -417,21 +511,27 @@ module Google
           end
 
           def unpause_streaming!
-            return unless unpause_streaming?
-
-            @paused = nil
-            subscriber.service.logger.log :info, "subscriber-flow-control" do
-              "subscriber for #{@subscriber.subscription_name} is unblocking client-side flow control"
+            synchronize do
+              return unless unpause_streaming?
+
+              @paused = nil
+              # Record pong when unpausing flow control. While paused, incoming server pongs sit buffered in gRPC,
+              # leaving last_pong_at stale. Updating timestamp guarantees the monitor thread will not trigger an immediate
+              # false-positive restart while the reader thread wakes up to drain buffered frames.
+              @keepalive_monitor.record_pong!
+              subscriber.service.logger.log :info, "subscriber-flow-control" do
+                "subscriber for #{@subscriber.subscription_name} is unblocking client-side flow control"
+              end
+              # Signal to the background thread that we are unpaused
+              @pause_cond.broadcast
             end
-            # signal to the background thread that we are unpaused
-            @pause_cond.broadcast
           end
 
           def unpause_streaming?
             return false if @stopped
             return false if @paused.nil?
 
-            @inventory.count < @inventory.limit * 0.8
+            @inventory.count < @inventory.limit * UNPAUSE_INVENTORY_RATIO
           end
 
           def initial_input_request
@@ -443,6 +543,7 @@ module Google
               req.client_id = @subscriber.service.client_id
               req.max_outstanding_messages = @inventory.limit
               req.max_outstanding_bytes = @inventory.bytesize
+              req.protocol_version = PROTOCOL_VERSION
             end
           end
 

data/lib/google/cloud/pubsub/version.rb

@@ -16,7 +16,7 @@
 module Google
   module Cloud
     module PubSub
-      VERSION = "3.3.0".freeze
+      VERSION = "3.4.0".freeze
     end
 
     Pubsub = PubSub unless const_defined? :Pubsub

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: google-cloud-pubsub
 version: !ruby/object:Gem::Version
-  version: 3.3.0
+  version: 3.4.0
 platform: ruby
 authors:
 - Mike Moore
@@ -100,6 +100,7 @@ files:
 - lib/google/cloud/pubsub/message_listener.rb
 - lib/google/cloud/pubsub/message_listener/enumerator_queue.rb
 - lib/google/cloud/pubsub/message_listener/inventory.rb
+- lib/google/cloud/pubsub/message_listener/keepalive_monitor.rb
 - lib/google/cloud/pubsub/message_listener/sequencer.rb
 - lib/google/cloud/pubsub/message_listener/stream.rb
 - lib/google/cloud/pubsub/message_listener/timed_unary_buffer.rb