google-cloud-pubsub 2.22.0 → 3.2.0

data/lib/google/cloud/pubsub/admin_clients.rb

@@ -0,0 +1,116 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "google/cloud/pubsub/v1/topic_admin/client"
+require "google/cloud/pubsub/v1/subscription_admin/client"
+
+module Google
+  module Cloud
+    module PubSub
+      module TopicAdmin
+        ##
+        # The TopicAdmin client is used to manage topics.
+        #
+        # This client is a subclass of the auto-generated TopicAdmin client, and provides
+        # the same methods. However, it raises errors on data plane operations
+        # to prevent misuse.
+        #
+        class Client < Google::Cloud::PubSub::V1::TopicAdmin::Client
+          # @private
+          alias publish_internal publish
+
+          ##
+          # The `publish` method is a data plane operation.
+          #
+          # @raise [NotImplementedError] This method is not implemented on the
+          #   admin client. Use {Google::Cloud::PubSub::Publisher} instead.
+          #
+          def publish *args, **kwargs
+            raise NotImplementedError,
+                  "The `publish` method is a data plane operation. " \
+                  "Use Google::Cloud::PubSub::Publisher instead."
+          end
+        end
+      end
+
+      module SubscriptionAdmin
+        ##
+        # The SubscriptionAdmin client is used to manage subscriptions.
+        #
+        # This client is a subclass of the auto-generated SubscriptionAdmin client, and
+        # provides the same methods. However, it raises errors on data plane
+        # operations to prevent misuse.
+        #
+        class Client < Google::Cloud::PubSub::V1::SubscriptionAdmin::Client
+          # @private
+          alias modify_ack_deadline_internal modify_ack_deadline
+          # @private
+          alias acknowledge_internal acknowledge
+          # @private
+          alias pull_internal pull
+          # @private
+          alias streaming_pull_internal streaming_pull
+
+          ##
+          # The `modify_ack_deadline` method is a data plane operation.
+          #
+          # @raise [NotImplementedError] This method is not implemented on the
+          #   admin client. Use {Google::Cloud::PubSub::Subscriber} instead.
+          #
+          def modify_ack_deadline *args, **kwargs
+            raise NotImplementedError,
+                  "The `modify_ack_deadline` method is a data plane operation. " \
+                  "Use Google::Cloud::PubSub::Subscriber instead."
+          end
+
+          ##
+          # The `acknowledge` method is a data plane operation.
+          #
+          # @raise [NotImplementedError] This method is not implemented on the
+          #   admin client. Use {Google::Cloud::PubSub::Subscriber} instead.
+          #
+          def acknowledge *args, **kwargs
+            raise NotImplementedError,
+                  "The `acknowledge` method is a data plane operation. " \
+                  "Use Google::Cloud::PubSub::Subscriber instead."
+          end
+
+          ##
+          # The `pull` method is a data plane operation.
+          #
+          # @raise [NotImplementedError] This method is not implemented on the
+          #   admin client. Use {Google::Cloud::PubSub::Subscriber} instead.
+          #
+          def pull *args, **kwargs
+            raise NotImplementedError,
+                  "The `pull` method is a data plane operation. " \
+                  "Use Google::Cloud::PubSub::Subscriber instead."
+          end
+
+          ##
+          # The `streaming_pull` method is a data plane operation.
+          #
+          # @raise [NotImplementedError] This method is not implemented on the
+          #   admin client. Use {Google::Cloud::PubSub::Subscriber} instead.
+          #
+          def streaming_pull *args, **kwargs
+            raise NotImplementedError,
+                  "The `streaming_pull` method is a data plane operation. " \
+                  "Use Google::Cloud::PubSub::Subscriber instead."
+          end
+        end
+      end
+    end
+  end
+end

data/lib/google/cloud/pubsub/async_publisher.rb

@@ -27,15 +27,15 @@ module Google
     module PubSub
       ##
       # Used to publish multiple messages in batches to a topic. See
-      # {Google::Cloud::PubSub::Topic#async_publisher}
+      # {Google::Cloud::PubSub::Publisher#async_publisher}
       #
       # @example
       #   require "google/cloud/pubsub"
       #
       #   pubsub = Google::Cloud::PubSub.new
       #
-      #   topic = pubsub.topic "my-topic"
-      #   topic.publish_async "task completed" do |result|
+      #   publisher = pubsub.publisher "my-topic"
+      #   publisher.publish_async "task completed" do |result|
       #     if result.succeeded?
       #       log_publish_success result.data
       #     else
@@ -43,7 +43,7 @@ module Google
       #     end
       #   end
       #
-      #   topic.async_publisher.stop!
+      #   publisher.async_publisher.stop!
       #
       # @attr_reader [String] topic_name The name of the topic the messages are published to. The value is a
       #   fully-qualified topic name in the form `projects/{project_id}/topics/{topic_id}`.
@@ -116,7 +116,7 @@ module Google
         ##
         # Add a message to the async publisher to be published to the topic.
         # Messages will be collected in batches and published together.
-        # See {Google::Cloud::PubSub::Topic#publish_async}
+        # See {Google::Cloud::PubSub::Publisher#publish_async}
         #
         # @param [String, File] data The message payload. This will be converted
         #   to bytes encoded as ASCII-8BIT.
@@ -157,7 +157,7 @@ module Google
             end
             batch_action = batch.add msg, callback
             if batch_action == :full
-              publish_batches!
+              publish_batches! reason: "batch full"
             elsif @published_at.nil?
               # Set initial time to now to start the background counter
               @published_at = Time.now
@@ -180,7 +180,7 @@ module Google
             break if @stopped
 
             @stopped = true
-            publish_batches! stop: true
+            publish_batches! stop: true, reason: "shutdown"
             @cond.signal
             @publish_thread_pool.shutdown
           end
@@ -234,7 +234,7 @@ module Google
         # @return [AsyncPublisher] returns self so calls can be chained.
         def flush
           synchronize do
-            publish_batches!
+            publish_batches! reason: "manual flush"
             @cond.signal
           end
 
@@ -262,8 +262,8 @@ module Google
         # enabled, messages published with the same `ordering_key` will be
         # delivered in the order they were published.
         #
-        # See {#message_ordering?}. See {Topic#publish_async},
-        # {Subscription#listen}, and {Message#ordering_key}.
+        # See {#message_ordering?}. See {Publisher#publish_async},
+        # {Subscriber#listen}, and {Message#ordering_key}.
         #
         def enable_message_ordering!
           synchronize { @ordered = true }
@@ -275,8 +275,8 @@ module Google
         # will be delivered in the order they were published. When disabled,
         # messages may be delivered in any order.
         #
-        # See {#enable_message_ordering!}. See {Topic#publish_async},
-        # {Subscription#listen}, and {Message#ordering_key}.
+        # See {#enable_message_ordering!}. See {Publisher#publish_async},
+        # {Subscriber#listen}, and {Message#ordering_key}.
         #
         # @return [Boolean]
         #
@@ -313,7 +313,7 @@ module Google
               time_since_first_publish = Time.now - @published_at
               if time_since_first_publish > @interval
                 # interval met, flush the batches...
-                publish_batches!
+                publish_batches! reason: "interval timeout"
                 @cond.wait
               else
                 # still waiting for the interval to publish the batch...
@@ -347,28 +347,28 @@ module Google
           end
         end
 
-        def publish_batches! stop: nil
+        def publish_batches! stop: nil, reason: "unknown"
           @batches.reject! { |_ordering_key, batch| batch.empty? }
           @batches.each_value do |batch|
             ready = batch.publish! stop: stop
-            publish_batch_async @topic_name, batch if ready
+            publish_batch_async @topic_name, batch, reason: reason if ready
           end
           # Set published_at to nil to wait indefinitely
           @published_at = nil
         end
 
-        def publish_batch_async topic_name, batch
+        def publish_batch_async topic_name, batch, reason: "unknown"
           # TODO: raise unless @publish_thread_pool.running?
           return unless @publish_thread_pool.running?
 
           Concurrent::Promises.future_on(
-            @publish_thread_pool, topic_name, batch
-          ) { |t, b| publish_batch_sync t, b }
+            @publish_thread_pool, topic_name, batch, reason
+          ) { |t, b, r| publish_batch_sync t, b, reason: r }
         end
 
         # rubocop:disable Metrics/AbcSize
 
-        def publish_batch_sync topic_name, batch
+        def publish_batch_sync topic_name, batch, reason: "unknown"
           # The only batch methods that are safe to call from the loop are
           # rebalance! and reset! because they are the only methods that are
           # synchronized.
@@ -379,6 +379,7 @@ module Google
               grpc = @service.publish topic_name,
                                       items.map(&:msg),
                                       compress: compress && batch.total_message_bytes >= compression_bytes_threshold
+              service.logger.log_batch "publish-batch", reason, "publish", items.count, items.sum(&:bytesize)
               items.zip Array(grpc.message_ids) do |item, id|
                 @flow_controller.release item.bytesize
                 next unless item.callback

data/lib/google/cloud/pubsub/batch_publisher.rb

@@ -27,14 +27,15 @@ module Google
       #
       #   pubsub = Google::Cloud::PubSub.new
       #
-      #   topic = pubsub.topic "my-topic"
-      #   msgs = topic.publish do |batch_publisher|
+      #   publisher = pubsub.publisher "my-topic"
+      #   msgs = publisher.publish do |batch_publisher|
       #     batch_publisher.publish "task 1 completed", foo: :bar
       #     batch_publisher.publish "task 2 completed", foo: :baz
       #     batch_publisher.publish "task 3 completed", foo: :bif
       #   end
       #
       class BatchPublisher
+
         ##
         # @private The messages to publish
         attr_reader :messages
@@ -86,8 +87,8 @@ module Google
         #
         #   pubsub = Google::Cloud::PubSub.new
         #
-        #   topic = pubsub.topic "my-topic"
-        #   msgs = topic.publish do |batch_publisher|
+        #   publisher = pubsub.publisher "my-topic"
+        #   msgs = publisher.publish do |batch_publisher|
         #     batch_publisher.publish "task 1 completed", foo: :bar
         #     batch_publisher.publish "task 2 completed", foo: :baz
         #     batch_publisher.publish "task 3 completed", foo: :bif
@@ -117,10 +118,11 @@ module Google
 
         ##
         # @private  Call the publish API with arrays of data and attrs.
-        def publish_batch_messages topic_name, service
+        def publish_batch_messages topic_name, service, reason: "unknown"
           grpc = service.publish topic_name,
                                  messages,
                                  compress: compress && total_message_bytes >= compression_bytes_threshold
+          service.logger.log_batch "publish-batch", reason, "publish", messages.count, @total_message_bytes
           to_gcloud_messages Array(grpc.message_ids)
         end
       end

data/lib/google/cloud/pubsub/errors.rb

@@ -41,7 +41,7 @@ module Google
       end
 
       ##
-      # Indicates that the {Subscriber} for a {Subscription} with message
+      # Indicates that the {MessageListener} for a {Subscriber} with message
       # ordering enabled has observed that a message has been delivered out of
       # order.
       #
@@ -57,11 +57,11 @@ module Google
 
       ##
       # Indicates that messages using the {#ordering_key} are not being
-      # published due to error. Future calls to {Topic#publish_async} with the
+      # published due to error. Future calls to {Publisher#publish_async} with the
       # {#ordering_key} will fail with this error.
       #
       # To allow future messages with the {#ordering_key} to be published, the
-      # {#ordering_key} must be passed to {Topic#resume_publish}.
+      # {#ordering_key} must be passed to {Publisher#resume_publish}.
       #
       # If this error is retrieved from {PublishResult#error}, inspect `cause`
       # for the error raised while publishing.

data/lib/google/cloud/pubsub/internal_logger.rb

@@ -0,0 +1,76 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+require "logger"
+
+require "google/cloud/config"
+
+module Google
+  module Cloud
+    module PubSub
+      ##
+      # @private
+      class InternalLogger
+        LOG_NAME = "pubsub".freeze
+        VALID_LOG_LEVELS = [:debug, :info, :warn, :error, :fatal].freeze
+        private_constant :VALID_LOG_LEVELS, :LOG_NAME
+
+        ##
+        # @private
+        # rubocop:disable Naming/BlockForwarding
+        def log level, subtag, &message_block
+          return unless VALID_LOG_LEVELS.include?(level) && block_given?
+          # Only log if the logger is explicitly tagged for 'pubsub'.
+          return unless @logger && @logger.progname == LOG_NAME
+
+          @logger.public_send(level, "#{LOG_NAME}:#{subtag}", &message_block)
+        end
+        # rubocop:enable Naming/BlockForwarding
+
+        ##
+        # @private
+        def log_batch logger_name, reason, type, num_messages, total_bytes
+          log :info, logger_name do
+            "#{reason} triggered #{type} batch of #{num_messages} messages, a total of #{total_bytes} bytes"
+          end
+        end
+
+        ##
+        # @private
+        def log_ack_nack ack_ids, type
+          ack_ids.each do |ack_id|
+            log :info, "ack-nack" do
+              "message (ackID #{ack_id}) #{type}"
+            end
+          end
+        end
+
+        ##
+        # @private
+        def log_expiry expired
+          expired.each do |ack_id, item|
+            log :info, "expiry" do
+              "message (ID #{item.message_id}, ackID #{ack_id}) has been dropped from leasing due to a timeout"
+            end
+          end
+        end
+
+        private
+
+        def initialize logger
+          @logger = logger || Logger.new(nil)
+        end
+      end
+    end
+  end
+end

data/lib/google/cloud/pubsub/message.rb

@@ -24,7 +24,7 @@ module Google
       #
       # Represents a Pub/Sub Message.
       #
-      # Message objects are created by {Topic#publish}. {Subscription#pull}
+      # Message objects are created by {Publisher#publish}. {Subscriber#pull}
       # returns an array of {ReceivedMessage} objects, each of which contains a
       # Message object. Each {ReceivedMessage} object can be acknowledged and/or
       # delayed.
@@ -35,22 +35,22 @@ module Google
       #   pubsub = Google::Cloud::PubSub.new
       #
       #   # Publish a message
-      #   topic = pubsub.topic "my-topic"
-      #   message = topic.publish "task completed"
+      #   publisher = pubsub.publisher "my-topic"
+      #   message = publisher.publish "task completed"
       #   message.data #=> "task completed"
       #
       #   # Listen for messages
-      #   sub = pubsub.subscription "my-topic-sub"
-      #   subscriber = sub.listen do |received_message|
+      #   subscriber = pubsub.subscriber "my-topic-sub"
+      #   listener = subscriber.listen do |received_message|
       #     # process message
       #     received_message.acknowledge!
       #   end
       #
       #   # Start background threads that will call the block passed to listen.
-      #   subscriber.start
+      #   listener.start
       #
       #   # Shut down the subscriber when ready to stop receiving messages.
-      #   subscriber.stop!
+      #   listener.stop!
       #
       class Message
         ##
@@ -113,7 +113,7 @@ module Google
         # or scalability, as the service automatically distributes messages for
         # different ordering keys across subscribers.
         #
-        # See {Topic#publish_async} and {Subscription#listen}.
+        # See {Publisher#publish_async} and {Subscriber#listen}.
         #
         # @return [String]
         #

data/lib/google/cloud/pubsub/{subscriber → message_listener}/enumerator_queue.rb

@@ -16,7 +16,7 @@
 module Google
   module Cloud
     module PubSub
-      class Subscriber
+      class MessageListener
         # @private
         class EnumeratorQueue
           def initialize sentinel = nil

data/lib/google/cloud/pubsub/{subscriber → message_listener}/inventory.rb

@@ -12,19 +12,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
 require "monitor"
 
 module Google
   module Cloud
     module PubSub
-      class Subscriber
+      class MessageListener
         ##
         # @private
         class Inventory
-          InventoryItem = Struct.new :bytesize, :pulled_at do
+          InventoryItem = Struct.new :message_id, :bytesize, :pulled_at do
             def self.from rec_msg
-              new rec_msg.to_proto.bytesize, Time.now
+              new rec_msg.message.message_id, rec_msg.to_proto.bytesize, Time.now
             end
           end
 
@@ -36,10 +35,9 @@ module Google
           attr_reader :extension
           attr_reader :max_duration_per_lease_extension
           attr_accessor :min_duration_per_lease_extension
-          attr_reader :use_legacy_flow_control
 
           def initialize stream, limit:, bytesize:, extension:, max_duration_per_lease_extension:,
-                         min_duration_per_lease_extension:, use_legacy_flow_control:
+                         min_duration_per_lease_extension:
             super()
             @stream = stream
             @limit = limit
@@ -47,7 +45,6 @@ module Google
             @extension = extension
             @max_duration_per_lease_extension = max_duration_per_lease_extension
             @min_duration_per_lease_extension = min_duration_per_lease_extension
-            @use_legacy_flow_control = use_legacy_flow_control
             @inventory = {}
             @wait_cond = new_cond
           end
@@ -72,18 +69,24 @@ module Google
           def remove *ack_ids
             ack_ids.flatten!
             ack_ids.compact!
-            return if ack_ids.empty?
+            return {} if ack_ids.empty?
 
+            removed_items = {}
             synchronize do
-              @inventory.delete_if { |ack_id, _| ack_ids.include? ack_id }
+              removed, keep = @inventory.partition { |ack_id, _| ack_ids.include? ack_id }
+              @inventory = keep.to_h
+              removed_items = removed.to_h
               @wait_cond.broadcast
             end
+            removed_items
           end
 
           def remove_expired!
             synchronize do
               extension_time = Time.new - extension
-              @inventory.delete_if { |_ack_id, item| item.pulled_at < extension_time }
+              expired, keep = @inventory.partition { |_ack_id, item| item.pulled_at < extension_time }
+              @inventory = keep.to_h
+              stream.subscriber.service.logger.log_expiry expired
               @wait_cond.broadcast
             end
           end

data/lib/google/cloud/pubsub/{subscriber → message_listener}/sequencer.rb

@@ -18,7 +18,7 @@ require "monitor"
 module Google
   module Cloud
     module PubSub
-      class Subscriber
+      class MessageListener
         ##
         # @private The sequencer's job is simple, keep track of all the
         # streams's recieved message and deliver the messages with an