google-cloud-pubsub 2.14.0 → 2.15.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c4aa541774d81af56f0d85bea8428b5dfa7f348194faccd34924a0bc60e76eb5
-  data.tar.gz: 342948cbfc7001c74cc8df748a1fdf0964b4611495c710cd7e8d24a6e25291ae
+  metadata.gz: 487a660ea90e29fd8dcde076a9baf33aabefdba1d06267160b56254129b2af1a
+  data.tar.gz: 21138b050ced799b7a16cd2048a321d6be6f488457474c313a9976236d4599ab
 SHA512:
-  metadata.gz: bf8df4d2a93ee5133e982158e754f5f21f8f5a2e4066430232c139fa80194d11e85657ecb7b12c573d8777b5434e113e82922ce84d2d841f25a51ae67b7587b7
-  data.tar.gz: 1af4583788b6e79a93b854bd941fed593407afd6c0be1a771a2016a78629a88853f994e0e0e9188f1dc7c56393d1d7b75acf97ca1337cf3b3cd254df3f4e40c6
+  metadata.gz: 6a4aded9ee5ca61d0496ec669d6fdf1ac842eab912409cf2c2b2b52059d0151cfd7ff847c0d64b0be32946fb5484d8bfaf9499f8e061cee386868969179e11fd
+  data.tar.gz: 2964d7408a959048069ee489f22b72427ba6713453518b0a79cb660fef25ca38951f9d5ad5d903494757ed325932238309ff271e6f460718f6b337e294738eb5

data/CHANGELOG.md

@@ -1,5 +1,11 @@
 # Release History
 
+### 2.15.0 (2023-01-22)
+
+#### Features
+
+* support publisher compression ([#19910](https://github.com/googleapis/google-cloud-ruby/issues/19910)) 
+
 ### 2.14.0 (2023-01-12)
 
 #### Features

data/lib/google/cloud/pubsub/async_publisher/batch.rb

@@ -255,6 +255,10 @@ module Google
             end
           end
 
+          def total_message_bytes
+            @total_message_bytes
+          end
+
           protected
 
           def items_add msg, callback
@@ -288,10 +292,6 @@ module Google
             @items.count
           end
 
-          def total_message_bytes
-            @total_message_bytes
-          end
-
           Item = Struct.new :msg, :callback do
             def bytesize
               msg.to_proto.bytesize

data/lib/google/cloud/pubsub/async_publisher.rb

@@ -70,7 +70,8 @@ module Google
         ##
         # @private Implementation accessors
         attr_reader :service, :batch, :publish_thread_pool,
-                    :callback_thread_pool, :flow_controller
+                    :callback_thread_pool, :flow_controller,
+                    :compress, :compression_bytes_threshold
 
         ##
         # @private Create a new instance of the object.
@@ -80,7 +81,9 @@ module Google
                        max_messages: 100,
                        interval: 0.01,
                        threads: {},
-                       flow_control: {}
+                       flow_control: {},
+                       compress: nil,
+                       compression_bytes_threshold: nil
           # init MonitorMixin
           super()
           @topic_name = service.topic_path topic_name
@@ -105,6 +108,9 @@ module Google
           @cond = new_cond
           @flow_controller = FlowController.new(**@flow_control)
           @thread = Thread.new { run_background }
+          @compress = compress || Google::Cloud::PubSub::DEFAULT_COMPRESS
+          @compression_bytes_threshold = compression_bytes_threshold ||
+                                         Google::Cloud::PubSub::DEFAULT_COMPRESSION_BYTES_THRESHOLD
         end
 
         ##
@@ -370,7 +376,9 @@ module Google
             items = batch.rebalance!
 
             unless items.empty?
-              grpc = @service.publish topic_name, items.map(&:msg)
+              grpc = @service.publish topic_name,
+                                      items.map(&:msg),
+                                      compress: compress && batch.total_message_bytes >= compression_bytes_threshold
               items.zip Array(grpc.message_ids) do |item, id|
                 @flow_controller.release item.bytesize
                 next unless item.callback

data/lib/google/cloud/pubsub/batch_publisher.rb

@@ -39,11 +39,32 @@ module Google
         # @private The messages to publish
         attr_reader :messages
 
+        ##
+        # @private Enables publisher compression
+        attr_reader :compress
+
+        ##
+        # @private The threshold bytes size for compression
+        attr_reader :compression_bytes_threshold
+
+        ##
+        # @private The total bytes size of messages data.
+        attr_reader :total_message_bytes
+
         ##
         # @private Create a new instance of the object.
-        def initialize data, attributes, ordering_key, extra_attrs
+        def initialize data,
+                       attributes,
+                       ordering_key,
+                       extra_attrs,
+                       compress: nil,
+                       compression_bytes_threshold: nil
           @messages = []
           @mode = :batch
+          @compress = compress || Google::Cloud::PubSub::DEFAULT_COMPRESS
+          @compression_bytes_threshold = compression_bytes_threshold ||
+                                         Google::Cloud::PubSub::DEFAULT_COMPRESSION_BYTES_THRESHOLD
+          @total_message_bytes = 0
           return if data.nil?
           @mode = :single
           publish data, attributes, ordering_key: ordering_key, **extra_attrs
@@ -74,6 +95,7 @@ module Google
         #
         def publish data, attributes = nil, ordering_key: nil, **extra_attrs
           msg = Convert.pubsub_message data, attributes, ordering_key, extra_attrs
+          @total_message_bytes += msg.data.bytesize + 2
           @messages << msg
         end
 
@@ -92,6 +114,15 @@ module Google
             msgs
           end
         end
+
+        ##
+        # @private  Call the publish API with arrays of data and attrs.
+        def publish_batch_messages topic_name, service
+          grpc = service.publish topic_name,
+                                 messages,
+                                 compress: compress && total_message_bytes >= compression_bytes_threshold
+          to_gcloud_messages Array(grpc.message_ids)
+        end
       end
     end
 

data/lib/google/cloud/pubsub/project.rb

@@ -24,6 +24,9 @@ require "google/cloud/pubsub/snapshot"
 module Google
   module Cloud
     module PubSub
+      DEFAULT_COMPRESS = false
+      DEFAULT_COMPRESSION_BYTES_THRESHOLD = 240
+
       ##
       # # Project
       #
@@ -103,6 +106,9 @@ module Google
         #   * `:threads` (Hash) The number of threads to create to handle concurrent calls by the publisher:
         #     * `:publish` (Integer) The number of threads used to publish messages. Default is 2.
         #     * `:callback` (Integer) The number of threads to handle the published messages' callbacks. Default is 4.
+        #   * `:compress` (Boolean) The flag that enables publisher compression. Default is false
+        #   * `:compression_bytes_threshold` (Integer) The number of bytes above which compress should be enabled.
+        #       Default is 240.
         #   * `:flow_control` (Hash) The client flow control settings for message publishing:
         #     * `:message_limit` (Integer) The maximum number of messages allowed to wait to be published. Default is
         #       `10 * max_messages`.
@@ -211,11 +217,13 @@ module Google
         #     the batch is published. Default is 0.01.
         #   * `:threads` (Hash) The number of threads to create to handle concurrent
         #     calls by the publisher:
-        #
         #     * `:publish` (Integer) The number of threads used to publish messages.
         #       Default is 2.
         #     * `:callback` (Integer) The number of threads to handle the published
         #       messages' callbacks. Default is 4.
+        #   * `:compress` (Boolean) The flag that enables publisher compression. Default is false
+        #   * `:compression_bytes_threshold` (Integer) The number of bytes above which compress should be enabled.
+        #       Default is 240.
         #   * `:flow_control` (Hash) The client flow control settings for message publishing:
         #     * `:message_limit` (Integer) The maximum number of messages allowed to wait to be published. Default is
         #       `10 * max_messages`.
@@ -642,13 +650,6 @@ module Google
         def ensure_service!
           raise "Must have active connection to service" unless service
         end
-
-        ##
-        # Call the publish API with arrays of data data and attrs.
-        def publish_batch_messages topic_name, batch
-          grpc = service.publish topic_name, batch.messages
-          batch.to_gcloud_messages Array(grpc.message_ids)
-        end
       end
     end
 

data/lib/google/cloud/pubsub/service.rb

@@ -173,8 +173,10 @@ module Google
         # Raises GRPC status code 5 if the topic does not exist.
         # The messages parameter is an array of arrays.
         # The first element is the data, second is attributes hash.
-        def publish topic, messages
-          publisher.publish topic: topic_path(topic), messages: messages
+        def publish topic, messages, compress: false
+          request = { topic: topic_path(topic), messages: messages }
+          compress_options = ::Gapic::CallOptions.new metadata: { "grpc-internal-encoding-request": "gzip" }
+          compress ? (publisher.publish request, compress_options) : (publisher.publish request)
         end
 
         ##

data/lib/google/cloud/pubsub/topic.rb

@@ -676,12 +676,19 @@ module Google
         #   topic.publish "task completed",
         #                 ordering_key: "task-key"
         #
-        def publish data = nil, attributes = nil, ordering_key: nil, **extra_attrs, &block
+        def publish data = nil, attributes = nil, ordering_key: nil, compress: nil, compression_bytes_threshold: nil,
+                    **extra_attrs, &block
           ensure_service!
-          batch = BatchPublisher.new data, attributes, ordering_key, extra_attrs
+          batch = BatchPublisher.new data,
+                                     attributes,
+                                     ordering_key,
+                                     extra_attrs,
+                                     compress: compress,
+                                     compression_bytes_threshold: compression_bytes_threshold
+
           block&.call batch
           return nil if batch.messages.count.zero?
-          publish_batch_messages batch
+          batch.publish_batch_messages name, service
         end
 
         ##
@@ -1085,13 +1092,6 @@ module Google
           ensure_service!
           reload! if reference?
         end
-
-        ##
-        # Call the publish API with arrays of data data and attrs.
-        def publish_batch_messages batch
-          grpc = service.publish name, batch.messages
-          batch.to_gcloud_messages Array(grpc.message_ids)
-        end
       end
     end
 

data/lib/google/cloud/pubsub/version.rb

@@ -16,7 +16,7 @@
 module Google
   module Cloud
     module PubSub
-      VERSION = "2.14.0".freeze
+      VERSION = "2.15.0".freeze
     end
 
     Pubsub = PubSub unless const_defined? :Pubsub

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: google-cloud-pubsub
 version: !ruby/object:Gem::Version
-  version: 2.14.0
+  version: 2.15.0
 platform: ruby
 authors:
 - Mike Moore
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-01-12 00:00:00.000000000 Z
+date: 2023-01-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby