google-cloud-pubsub 2.1.1 → 2.7.0

data/lib/google/cloud/pubsub/service.rb

@@ -26,7 +26,10 @@ module Google
       ##
       # @private Represents the Pub/Sub service API, including IAM mixins.
       class Service
-        attr_accessor :project, :credentials, :host, :timeout
+        attr_accessor :project
+        attr_accessor :credentials
+        attr_accessor :host
+        attr_accessor :timeout
         ###
         # The same client_id is used across all streaming pull connections that are created by this client. This is
         # intentional, as it indicates to the server that any guarantees, such as message ordering, made for a stream
@@ -46,49 +49,56 @@ module Google
 
         def subscriber
           return mocked_subscriber if mocked_subscriber
-          @subscriber ||= begin
-            V1::Subscriber::Client.new do |config|
-              config.credentials = credentials if credentials
-              config.timeout = timeout if timeout
-              config.endpoint = host if host
-              config.lib_name = "gccl"
-              config.lib_version = Google::Cloud::PubSub::VERSION
-              config.metadata = { "google-cloud-resource-prefix": "projects/#{@project}" }
-            end
+          @subscriber ||= V1::Subscriber::Client.new do |config|
+            config.credentials = credentials if credentials
+            config.timeout = timeout if timeout
+            config.endpoint = host if host
+            config.lib_name = "gccl"
+            config.lib_version = Google::Cloud::PubSub::VERSION
+            config.metadata = { "google-cloud-resource-prefix": "projects/#{@project}" }
           end
         end
         attr_accessor :mocked_subscriber
 
         def publisher
           return mocked_publisher if mocked_publisher
-          @publisher ||= begin
-            V1::Publisher::Client.new do |config|
-              config.credentials = credentials if credentials
-              config.timeout = timeout if timeout
-              config.endpoint = host if host
-              config.lib_name = "gccl"
-              config.lib_version = Google::Cloud::PubSub::VERSION
-              config.metadata = { "google-cloud-resource-prefix": "projects/#{@project}" }
-            end
+          @publisher ||= V1::Publisher::Client.new do |config|
+            config.credentials = credentials if credentials
+            config.timeout = timeout if timeout
+            config.endpoint = host if host
+            config.lib_name = "gccl"
+            config.lib_version = Google::Cloud::PubSub::VERSION
+            config.metadata = { "google-cloud-resource-prefix": "projects/#{@project}" }
           end
         end
         attr_accessor :mocked_publisher
 
         def iam
           return mocked_iam if mocked_iam
-          @iam ||= begin
-            V1::IAMPolicy::Client.new do |config|
-              config.credentials = credentials if credentials
-              config.timeout = timeout if timeout
-              config.endpoint = host if host
-              config.lib_name = "gccl"
-              config.lib_version = Google::Cloud::PubSub::VERSION
-              config.metadata = { "google-cloud-resource-prefix" => "projects/#{@project}" }
-            end
+          @iam ||= V1::IAMPolicy::Client.new do |config|
+            config.credentials = credentials if credentials
+            config.timeout = timeout if timeout
+            config.endpoint = host if host
+            config.lib_name = "gccl"
+            config.lib_version = Google::Cloud::PubSub::VERSION
+            config.metadata = { "google-cloud-resource-prefix" => "projects/#{@project}" }
           end
         end
         attr_accessor :mocked_iam
 
+        def schemas
+          return mocked_schemas if mocked_schemas
+          @schemas ||= V1::SchemaService::Client.new do |config|
+            config.credentials = credentials if credentials
+            config.timeout = timeout if timeout
+            config.endpoint = host if host
+            config.lib_name = "gccl"
+            config.lib_version = Google::Cloud::PubSub::VERSION
+            config.metadata = { "google-cloud-resource-prefix" => "projects/#{@project}" }
+          end
+        end
+        attr_accessor :mocked_schemas
+
         ##
         # Gets the configuration of a topic.
         # Since the topic only has the name attribute,
@@ -111,18 +121,35 @@ module Google
 
         ##
         # Creates the given topic with the given name.
-        def create_topic topic_name, labels: nil, kms_key_name: nil, persistence_regions: nil, options: {}
+        def create_topic topic_name,
+                         labels: nil,
+                         kms_key_name: nil,
+                         persistence_regions: nil,
+                         schema_name: nil,
+                         message_encoding: nil,
+                         options: {}
           if persistence_regions
-            message_storage_policy = {
+            message_storage_policy = Google::Cloud::PubSub::V1::MessageStoragePolicy.new(
               allowed_persistence_regions: Array(persistence_regions)
-            }
+            )
+          end
+
+          if schema_name || message_encoding
+            unless schema_name && message_encoding
+              raise ArgumentError, "Schema settings must include both schema_name and message_encoding."
+            end
+            schema_settings = Google::Cloud::PubSub::V1::SchemaSettings.new(
+              schema:   schema_path(schema_name),
+              encoding: message_encoding.to_s.upcase
+            )
           end
 
           publisher.create_topic \
             name:                   topic_path(topic_name, options),
             labels:                 labels,
             kms_key_name:           kms_key_name,
-            message_storage_policy: message_storage_policy
+            message_storage_policy: message_storage_policy,
+            schema_settings:        schema_settings
         end
 
         def update_topic topic_obj, *fields
@@ -294,6 +321,89 @@ module Google
           end
         end
 
+        ##
+        # Lists schemas in the current (or given) project.
+        # @param view [String, Symbol, nil] Possible values:
+        #   * `BASIC` - Include the name and type of the schema, but not the definition.
+        #   * `FULL` - Include all Schema object fields.
+        #
+        def list_schemas view, options = {}
+          schema_view = Google::Cloud::PubSub::V1::SchemaView.const_get view.to_s.upcase
+          paged_enum = schemas.list_schemas parent:     project_path(options),
+                                            view:       schema_view,
+                                            page_size:  options[:max],
+                                            page_token: options[:token]
+
+          paged_enum.response
+        end
+
+        ##
+        # Creates a schema in the current (or given) project.
+        def create_schema schema_id, type, definition, options = {}
+          schema = Google::Cloud::PubSub::V1::Schema.new(
+            type:       type,
+            definition: definition
+          )
+          schemas.create_schema parent:    project_path(options),
+                                schema:    schema,
+                                schema_id: schema_id
+        end
+
+        ##
+        # Gets the details of a schema.
+        # @param view [String, Symbol, nil] The set of fields to return in the response. Possible values:
+        #   * `BASIC` - Include the name and type of the schema, but not the definition.
+        #   * `FULL` - Include all Schema object fields.
+        #
+        def get_schema schema_name, view, options = {}
+          schema_view = Google::Cloud::PubSub::V1::SchemaView.const_get view.to_s.upcase
+          schemas.get_schema name: schema_path(schema_name, options),
+                             view: schema_view
+        end
+
+        ##
+        # Delete a schema.
+        def delete_schema schema_name
+          schemas.delete_schema name: schema_path(schema_name)
+        end
+
+        ##
+        # Validate the definition string intended for a schema.
+        def validate_schema type, definition, options = {}
+          schema = Google::Cloud::PubSub::V1::Schema.new(
+            type:       type,
+            definition: definition
+          )
+          schemas.validate_schema parent: project_path(options),
+                                  schema: schema
+        end
+
+        ##
+        # Validates a message against a schema.
+        #
+        # @param message_data [String] Message to validate against the provided `schema_spec`.
+        # @param message_encoding [Google::Cloud::PubSub::V1::Encoding] The encoding expected for messages.
+        # @param schema_name [String] Name of the schema against which to validate.
+        # @param project [String] Name of the project if not the default project.
+        # @param type [String] Ad-hoc schema type against which to validate.
+        # @param definition [String] Ad-hoc schema definition against which to validate.
+        #
+        def validate_message message_data, message_encoding, schema_name: nil, project: nil, type: nil, definition: nil
+          if type && definition
+            schema = Google::Cloud::PubSub::V1::Schema.new(
+              type:       type,
+              definition: definition
+            )
+          end
+          schemas.validate_message parent:   project_path(project: project),
+                                   name:     schema_path(schema_name),
+                                   schema:   schema,
+                                   message:  message_data,
+                                   encoding: message_encoding
+        end
+
+        # Helper methods
+
         def get_topic_policy topic_name, options = {}
           iam.get_iam_policy resource: topic_path(topic_name, options)
         end
@@ -338,6 +448,11 @@ module Google
           "#{project_path options}/snapshots/#{snapshot_name}"
         end
 
+        def schema_path schema_name, options = {}
+          return schema_name if schema_name.nil? || schema_name.to_s.include?("/")
+          "#{project_path options}/schemas/#{schema_name}"
+        end
+
         def inspect
           "#<#{self.class.name} (#{@project})>"
         end

data/lib/google/cloud/pubsub/snapshot.rb

@@ -15,6 +15,7 @@
 
 require "google/cloud/errors"
 require "google/cloud/pubsub/snapshot/list"
+require "google/cloud/pubsub/v1"
 
 module Google
   module Cloud
@@ -58,8 +59,10 @@ module Google
         end
 
         ##
-        # The name of the snapshot. Format is
-        # `projects/{project}/snapshots/{snap}`.
+        # The name of the snapshot.
+        #
+        # @return [String] A fully-qualified snapshot name in the form
+        #   `projects/{project_id}/snapshots/{snapshot_id}`.
         def name
           @grpc.name
         end

data/lib/google/cloud/pubsub/snapshot/list.rb

@@ -125,12 +125,12 @@ module Google
           #     puts snapshot.name
           #   end
           #
-          def all request_limit: nil
+          def all request_limit: nil, &block
             request_limit = request_limit.to_i if request_limit
             return enum_for :all, request_limit: request_limit unless block_given?
             results = self
             loop do
-              results.each { |r| yield r }
+              results.each(&block)
               if request_limit
                 request_limit -= 1
                 break if request_limit.negative?

data/lib/google/cloud/pubsub/subscriber.rb

@@ -64,8 +64,13 @@ module Google
       class Subscriber
         include MonitorMixin
 
-        attr_reader :subscription_name, :callback, :deadline, :streams, :message_ordering, :callback_threads,
-                    :push_threads
+        attr_reader :subscription_name
+        attr_reader :callback
+        attr_reader :deadline
+        attr_reader :streams
+        attr_reader :message_ordering
+        attr_reader :callback_threads
+        attr_reader :push_threads
 
         ##
         # @private Implementation attributes.
@@ -123,28 +128,21 @@ module Google
 
         ##
         # Immediately stops the subscriber. No new messages will be pulled from
-        # the subscription. All actions taken on received messages that have not
-        # yet been sent to the API will be sent to the API. All received but
-        # unprocessed messages will be released back to the API and redelivered.
-        # Use {#wait!} to block until the subscriber is fully stopped and all
-        # received messages have been processed or released.
+        # the subscription. Use {#wait!} to block until all received messages have
+        # been processed or released: All actions taken on received messages that
+        # have not yet been sent to the API will be sent to the API. All received
+        # but unprocessed messages will be released back to the API and redelivered.
         #
         # @return [Subscriber] returns self so calls can be chained.
         #
         def stop
-          stop_pool = synchronize do
+          synchronize do
             @started = false
             @stopped = true
-
-            @stream_pool.map do |stream|
-              Thread.new { stream.stop }
-            end
+            @stream_pool.map(&:stop)
+            wait_stop_buffer_thread!
+            self
           end
-          stop_pool.map(&:join)
-          # Stop the buffer after the streams are all stopped
-          synchronize { @buffer.stop }
-
-          self
         end
 
         ##
@@ -162,13 +160,8 @@ module Google
         # @return [Subscriber] returns self so calls can be chained.
         #
         def wait! timeout = nil
-          wait_pool = synchronize do
-            @stream_pool.map do |stream|
-              Thread.new { stream.wait! timeout }
-            end
-          end
-          wait_pool.map(&:join)
-
+          wait_stop_buffer_thread!
+          @wait_stop_buffer_thread.join timeout
           self
         end
 
@@ -306,6 +299,17 @@ module Google
         # @deprecated Use {#max_outstanding_bytes}.
         alias inventory_bytesize max_outstanding_bytes
 
+        ##
+        # Whether to enforce flow control at the client side only or to enforce it at both the client and
+        # the server. For more details about flow control see https://cloud.google.com/pubsub/docs/pull#config.
+        #
+        # @return [Boolean] `true` when only client side flow control is enforced, `false` when both client and
+        # server side flow control are enforced.
+        #
+        def use_legacy_flow_control?
+          @inventory[:use_legacy_flow_control]
+        end
+
         ##
         # The number of seconds that received messages can be held awaiting processing. Default is 3,600 (1 hour).
         #
@@ -334,7 +338,8 @@ module Google
             limit:                            @inventory[:max_outstanding_messages].fdiv(@streams).ceil,
             bytesize:                         @inventory[:max_outstanding_bytes].fdiv(@streams).ceil,
             extension:                        @inventory[:max_total_lease_duration],
-            max_duration_per_lease_extension: @inventory[:max_duration_per_lease_extension]
+            max_duration_per_lease_extension: @inventory[:max_duration_per_lease_extension],
+            use_legacy_flow_control:          @inventory[:use_legacy_flow_control]
           }
         end
 
@@ -362,6 +367,18 @@ module Google
 
         protected
 
+        ##
+        # Starts a new thread to call wait! (blocking) on each Stream and then stop the TimedUnaryBuffer.
+        def wait_stop_buffer_thread!
+          synchronize do
+            @wait_stop_buffer_thread ||= Thread.new do
+              @stream_pool.map(&:wait!)
+              # Shutdown the buffer TimerTask (and flush the buffer) after the streams are all stopped.
+              @buffer.stop
+            end
+          end
+        end
+
         def coerce_inventory inventory
           @inventory = inventory
           if @inventory.is_a? Hash
@@ -377,6 +394,7 @@ module Google
           @inventory[:max_outstanding_bytes] = Integer(@inventory[:max_outstanding_bytes] || 100_000_000)
           @inventory[:max_total_lease_duration] = Integer(@inventory[:max_total_lease_duration] || 3600)
           @inventory[:max_duration_per_lease_extension] = Integer(@inventory[:max_duration_per_lease_extension] || 0)
+          @inventory[:use_legacy_flow_control] = @inventory[:use_legacy_flow_control] || false
         end
 
         def default_error_callbacks

data/lib/google/cloud/pubsub/subscriber/inventory.rb

@@ -30,15 +30,22 @@ module Google
 
           include MonitorMixin
 
-          attr_reader :stream, :limit, :bytesize, :extension, :max_duration_per_lease_extension
-
-          def initialize stream, limit:, bytesize:, extension:, max_duration_per_lease_extension:
+          attr_reader :stream
+          attr_reader :limit
+          attr_reader :bytesize
+          attr_reader :extension
+          attr_reader :max_duration_per_lease_extension
+          attr_reader :use_legacy_flow_control
+
+          def initialize stream, limit:, bytesize:, extension:, max_duration_per_lease_extension:,
+                         use_legacy_flow_control:
             super()
             @stream = stream
             @limit = limit
             @bytesize = bytesize
             @extension = extension
             @max_duration_per_lease_extension = max_duration_per_lease_extension
+            @use_legacy_flow_control = use_legacy_flow_control
             @inventory = {}
             @wait_cond = new_cond
           end

data/lib/google/cloud/pubsub/subscriber/stream.rb

@@ -58,7 +58,7 @@ module Google
             @paused  = nil
             @pause_cond = new_cond
 
-            @inventory = Inventory.new self, @subscriber.stream_inventory
+            @inventory = Inventory.new self, **@subscriber.stream_inventory
 
             @sequencer = Sequencer.new(&method(:perform_callback_async)) if subscriber.message_ordering
 
@@ -363,8 +363,8 @@ module Google
               req.modify_deadline_ack_ids += @inventory.ack_ids
               req.modify_deadline_seconds += @inventory.ack_ids.map { @subscriber.deadline }
               req.client_id = @subscriber.service.client_id
-              req.max_outstanding_messages = @inventory.limit
-              req.max_outstanding_bytes = @inventory.bytesize
+              req.max_outstanding_messages = @inventory.use_legacy_flow_control ? 0 : @inventory.limit
+              req.max_outstanding_bytes = @inventory.use_legacy_flow_control ? 0 : @inventory.bytesize
             end
           end
 

data/lib/google/cloud/pubsub/subscriber/timed_unary_buffer.rb

@@ -25,7 +25,8 @@ module Google
         class TimedUnaryBuffer
           include MonitorMixin
 
-          attr_reader :max_bytes, :interval
+          attr_reader :max_bytes
+          attr_reader :interval
 
           def initialize subscriber, max_bytes: 500_000, interval: 1.0
             super() # to init MonitorMixin
@@ -138,7 +139,7 @@ module Google
               end
 
             groups = prev_reg.each_pair.group_by { |_ack_id, delay| delay }
-            req_hash = Hash[groups.map { |k, v| [k, v.map(&:first)] }]
+            req_hash = groups.transform_values { |v| v.map(&:first) }
 
             requests = { acknowledge: [] }
             ack_ids = Array(req_hash.delete(:ack)) # ack has no deadline set
@@ -215,11 +216,9 @@ module Google
 
           def add_future pool
             Concurrent::Promises.future_on pool do
-              begin
-                yield
-              rescue StandardError => e
-                error! e
-              end
+              yield
+            rescue StandardError => e
+              error! e
             end
           end
         end