google-cloud-pubsub 2.1.1 → 2.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -26,7 +26,10 @@ module Google
26
26
  ##
27
27
  # @private Represents the Pub/Sub service API, including IAM mixins.
28
28
  class Service
29
- attr_accessor :project, :credentials, :host, :timeout
29
+ attr_accessor :project
30
+ attr_accessor :credentials
31
+ attr_accessor :host
32
+ attr_accessor :timeout
30
33
  ###
31
34
  # The same client_id is used across all streaming pull connections that are created by this client. This is
32
35
  # intentional, as it indicates to the server that any guarantees, such as message ordering, made for a stream
@@ -46,49 +49,56 @@ module Google
46
49
 
47
50
  def subscriber
48
51
  return mocked_subscriber if mocked_subscriber
49
- @subscriber ||= begin
50
- V1::Subscriber::Client.new do |config|
51
- config.credentials = credentials if credentials
52
- config.timeout = timeout if timeout
53
- config.endpoint = host if host
54
- config.lib_name = "gccl"
55
- config.lib_version = Google::Cloud::PubSub::VERSION
56
- config.metadata = { "google-cloud-resource-prefix": "projects/#{@project}" }
57
- end
52
+ @subscriber ||= V1::Subscriber::Client.new do |config|
53
+ config.credentials = credentials if credentials
54
+ config.timeout = timeout if timeout
55
+ config.endpoint = host if host
56
+ config.lib_name = "gccl"
57
+ config.lib_version = Google::Cloud::PubSub::VERSION
58
+ config.metadata = { "google-cloud-resource-prefix": "projects/#{@project}" }
58
59
  end
59
60
  end
60
61
  attr_accessor :mocked_subscriber
61
62
 
62
63
  def publisher
63
64
  return mocked_publisher if mocked_publisher
64
- @publisher ||= begin
65
- V1::Publisher::Client.new do |config|
66
- config.credentials = credentials if credentials
67
- config.timeout = timeout if timeout
68
- config.endpoint = host if host
69
- config.lib_name = "gccl"
70
- config.lib_version = Google::Cloud::PubSub::VERSION
71
- config.metadata = { "google-cloud-resource-prefix": "projects/#{@project}" }
72
- end
65
+ @publisher ||= V1::Publisher::Client.new do |config|
66
+ config.credentials = credentials if credentials
67
+ config.timeout = timeout if timeout
68
+ config.endpoint = host if host
69
+ config.lib_name = "gccl"
70
+ config.lib_version = Google::Cloud::PubSub::VERSION
71
+ config.metadata = { "google-cloud-resource-prefix": "projects/#{@project}" }
73
72
  end
74
73
  end
75
74
  attr_accessor :mocked_publisher
76
75
 
77
76
  def iam
78
77
  return mocked_iam if mocked_iam
79
- @iam ||= begin
80
- V1::IAMPolicy::Client.new do |config|
81
- config.credentials = credentials if credentials
82
- config.timeout = timeout if timeout
83
- config.endpoint = host if host
84
- config.lib_name = "gccl"
85
- config.lib_version = Google::Cloud::PubSub::VERSION
86
- config.metadata = { "google-cloud-resource-prefix" => "projects/#{@project}" }
87
- end
78
+ @iam ||= V1::IAMPolicy::Client.new do |config|
79
+ config.credentials = credentials if credentials
80
+ config.timeout = timeout if timeout
81
+ config.endpoint = host if host
82
+ config.lib_name = "gccl"
83
+ config.lib_version = Google::Cloud::PubSub::VERSION
84
+ config.metadata = { "google-cloud-resource-prefix" => "projects/#{@project}" }
88
85
  end
89
86
  end
90
87
  attr_accessor :mocked_iam
91
88
 
89
+ def schemas
90
+ return mocked_schemas if mocked_schemas
91
+ @schemas ||= V1::SchemaService::Client.new do |config|
92
+ config.credentials = credentials if credentials
93
+ config.timeout = timeout if timeout
94
+ config.endpoint = host if host
95
+ config.lib_name = "gccl"
96
+ config.lib_version = Google::Cloud::PubSub::VERSION
97
+ config.metadata = { "google-cloud-resource-prefix" => "projects/#{@project}" }
98
+ end
99
+ end
100
+ attr_accessor :mocked_schemas
101
+
92
102
  ##
93
103
  # Gets the configuration of a topic.
94
104
  # Since the topic only has the name attribute,
@@ -111,18 +121,35 @@ module Google
111
121
 
112
122
  ##
113
123
  # Creates the given topic with the given name.
114
- def create_topic topic_name, labels: nil, kms_key_name: nil, persistence_regions: nil, options: {}
124
+ def create_topic topic_name,
125
+ labels: nil,
126
+ kms_key_name: nil,
127
+ persistence_regions: nil,
128
+ schema_name: nil,
129
+ message_encoding: nil,
130
+ options: {}
115
131
  if persistence_regions
116
- message_storage_policy = {
132
+ message_storage_policy = Google::Cloud::PubSub::V1::MessageStoragePolicy.new(
117
133
  allowed_persistence_regions: Array(persistence_regions)
118
- }
134
+ )
135
+ end
136
+
137
+ if schema_name || message_encoding
138
+ unless schema_name && message_encoding
139
+ raise ArgumentError, "Schema settings must include both schema_name and message_encoding."
140
+ end
141
+ schema_settings = Google::Cloud::PubSub::V1::SchemaSettings.new(
142
+ schema: schema_path(schema_name),
143
+ encoding: message_encoding.to_s.upcase
144
+ )
119
145
  end
120
146
 
121
147
  publisher.create_topic \
122
148
  name: topic_path(topic_name, options),
123
149
  labels: labels,
124
150
  kms_key_name: kms_key_name,
125
- message_storage_policy: message_storage_policy
151
+ message_storage_policy: message_storage_policy,
152
+ schema_settings: schema_settings
126
153
  end
127
154
 
128
155
  def update_topic topic_obj, *fields
@@ -294,6 +321,89 @@ module Google
294
321
  end
295
322
  end
296
323
 
324
+ ##
325
+ # Lists schemas in the current (or given) project.
326
+ # @param view [String, Symbol, nil] Possible values:
327
+ # * `BASIC` - Include the name and type of the schema, but not the definition.
328
+ # * `FULL` - Include all Schema object fields.
329
+ #
330
+ def list_schemas view, options = {}
331
+ schema_view = Google::Cloud::PubSub::V1::SchemaView.const_get view.to_s.upcase
332
+ paged_enum = schemas.list_schemas parent: project_path(options),
333
+ view: schema_view,
334
+ page_size: options[:max],
335
+ page_token: options[:token]
336
+
337
+ paged_enum.response
338
+ end
339
+
340
+ ##
341
+ # Creates a schema in the current (or given) project.
342
+ def create_schema schema_id, type, definition, options = {}
343
+ schema = Google::Cloud::PubSub::V1::Schema.new(
344
+ type: type,
345
+ definition: definition
346
+ )
347
+ schemas.create_schema parent: project_path(options),
348
+ schema: schema,
349
+ schema_id: schema_id
350
+ end
351
+
352
+ ##
353
+ # Gets the details of a schema.
354
+ # @param view [String, Symbol, nil] The set of fields to return in the response. Possible values:
355
+ # * `BASIC` - Include the name and type of the schema, but not the definition.
356
+ # * `FULL` - Include all Schema object fields.
357
+ #
358
+ def get_schema schema_name, view, options = {}
359
+ schema_view = Google::Cloud::PubSub::V1::SchemaView.const_get view.to_s.upcase
360
+ schemas.get_schema name: schema_path(schema_name, options),
361
+ view: schema_view
362
+ end
363
+
364
+ ##
365
+ # Delete a schema.
366
+ def delete_schema schema_name
367
+ schemas.delete_schema name: schema_path(schema_name)
368
+ end
369
+
370
+ ##
371
+ # Validate the definition string intended for a schema.
372
+ def validate_schema type, definition, options = {}
373
+ schema = Google::Cloud::PubSub::V1::Schema.new(
374
+ type: type,
375
+ definition: definition
376
+ )
377
+ schemas.validate_schema parent: project_path(options),
378
+ schema: schema
379
+ end
380
+
381
+ ##
382
+ # Validates a message against a schema.
383
+ #
384
+ # @param message_data [String] Message to validate against the provided `schema_spec`.
385
+ # @param message_encoding [Google::Cloud::PubSub::V1::Encoding] The encoding expected for messages.
386
+ # @param schema_name [String] Name of the schema against which to validate.
387
+ # @param project [String] Name of the project if not the default project.
388
+ # @param type [String] Ad-hoc schema type against which to validate.
389
+ # @param definition [String] Ad-hoc schema definition against which to validate.
390
+ #
391
+ def validate_message message_data, message_encoding, schema_name: nil, project: nil, type: nil, definition: nil
392
+ if type && definition
393
+ schema = Google::Cloud::PubSub::V1::Schema.new(
394
+ type: type,
395
+ definition: definition
396
+ )
397
+ end
398
+ schemas.validate_message parent: project_path(project: project),
399
+ name: schema_path(schema_name),
400
+ schema: schema,
401
+ message: message_data,
402
+ encoding: message_encoding
403
+ end
404
+
405
+ # Helper methods
406
+
297
407
  def get_topic_policy topic_name, options = {}
298
408
  iam.get_iam_policy resource: topic_path(topic_name, options)
299
409
  end
@@ -338,6 +448,11 @@ module Google
338
448
  "#{project_path options}/snapshots/#{snapshot_name}"
339
449
  end
340
450
 
451
+ def schema_path schema_name, options = {}
452
+ return schema_name if schema_name.nil? || schema_name.to_s.include?("/")
453
+ "#{project_path options}/schemas/#{schema_name}"
454
+ end
455
+
341
456
  def inspect
342
457
  "#<#{self.class.name} (#{@project})>"
343
458
  end
@@ -15,6 +15,7 @@
15
15
 
16
16
  require "google/cloud/errors"
17
17
  require "google/cloud/pubsub/snapshot/list"
18
+ require "google/cloud/pubsub/v1"
18
19
 
19
20
  module Google
20
21
  module Cloud
@@ -58,8 +59,10 @@ module Google
58
59
  end
59
60
 
60
61
  ##
61
- # The name of the snapshot. Format is
62
- # `projects/{project}/snapshots/{snap}`.
62
+ # The name of the snapshot.
63
+ #
64
+ # @return [String] A fully-qualified snapshot name in the form
65
+ # `projects/{project_id}/snapshots/{snapshot_id}`.
63
66
  def name
64
67
  @grpc.name
65
68
  end
@@ -125,12 +125,12 @@ module Google
125
125
  # puts snapshot.name
126
126
  # end
127
127
  #
128
- def all request_limit: nil
128
+ def all request_limit: nil, &block
129
129
  request_limit = request_limit.to_i if request_limit
130
130
  return enum_for :all, request_limit: request_limit unless block_given?
131
131
  results = self
132
132
  loop do
133
- results.each { |r| yield r }
133
+ results.each(&block)
134
134
  if request_limit
135
135
  request_limit -= 1
136
136
  break if request_limit.negative?
@@ -64,8 +64,13 @@ module Google
64
64
  class Subscriber
65
65
  include MonitorMixin
66
66
 
67
- attr_reader :subscription_name, :callback, :deadline, :streams, :message_ordering, :callback_threads,
68
- :push_threads
67
+ attr_reader :subscription_name
68
+ attr_reader :callback
69
+ attr_reader :deadline
70
+ attr_reader :streams
71
+ attr_reader :message_ordering
72
+ attr_reader :callback_threads
73
+ attr_reader :push_threads
69
74
 
70
75
  ##
71
76
  # @private Implementation attributes.
@@ -123,28 +128,21 @@ module Google
123
128
 
124
129
  ##
125
130
  # Immediately stops the subscriber. No new messages will be pulled from
126
- # the subscription. All actions taken on received messages that have not
127
- # yet been sent to the API will be sent to the API. All received but
128
- # unprocessed messages will be released back to the API and redelivered.
129
- # Use {#wait!} to block until the subscriber is fully stopped and all
130
- # received messages have been processed or released.
131
+ # the subscription. Use {#wait!} to block until all received messages have
132
+ # been processed or released: All actions taken on received messages that
133
+ # have not yet been sent to the API will be sent to the API. All received
134
+ # but unprocessed messages will be released back to the API and redelivered.
131
135
  #
132
136
  # @return [Subscriber] returns self so calls can be chained.
133
137
  #
134
138
  def stop
135
- stop_pool = synchronize do
139
+ synchronize do
136
140
  @started = false
137
141
  @stopped = true
138
-
139
- @stream_pool.map do |stream|
140
- Thread.new { stream.stop }
141
- end
142
+ @stream_pool.map(&:stop)
143
+ wait_stop_buffer_thread!
144
+ self
142
145
  end
143
- stop_pool.map(&:join)
144
- # Stop the buffer after the streams are all stopped
145
- synchronize { @buffer.stop }
146
-
147
- self
148
146
  end
149
147
 
150
148
  ##
@@ -162,13 +160,8 @@ module Google
162
160
  # @return [Subscriber] returns self so calls can be chained.
163
161
  #
164
162
  def wait! timeout = nil
165
- wait_pool = synchronize do
166
- @stream_pool.map do |stream|
167
- Thread.new { stream.wait! timeout }
168
- end
169
- end
170
- wait_pool.map(&:join)
171
-
163
+ wait_stop_buffer_thread!
164
+ @wait_stop_buffer_thread.join timeout
172
165
  self
173
166
  end
174
167
 
@@ -306,6 +299,17 @@ module Google
306
299
  # @deprecated Use {#max_outstanding_bytes}.
307
300
  alias inventory_bytesize max_outstanding_bytes
308
301
 
302
+ ##
303
+ # Whether to enforce flow control at the client side only or to enforce it at both the client and
304
+ # the server. For more details about flow control see https://cloud.google.com/pubsub/docs/pull#config.
305
+ #
306
+ # @return [Boolean] `true` when only client side flow control is enforced, `false` when both client and
307
+ # server side flow control are enforced.
308
+ #
309
+ def use_legacy_flow_control?
310
+ @inventory[:use_legacy_flow_control]
311
+ end
312
+
309
313
  ##
310
314
  # The number of seconds that received messages can be held awaiting processing. Default is 3,600 (1 hour).
311
315
  #
@@ -334,7 +338,8 @@ module Google
334
338
  limit: @inventory[:max_outstanding_messages].fdiv(@streams).ceil,
335
339
  bytesize: @inventory[:max_outstanding_bytes].fdiv(@streams).ceil,
336
340
  extension: @inventory[:max_total_lease_duration],
337
- max_duration_per_lease_extension: @inventory[:max_duration_per_lease_extension]
341
+ max_duration_per_lease_extension: @inventory[:max_duration_per_lease_extension],
342
+ use_legacy_flow_control: @inventory[:use_legacy_flow_control]
338
343
  }
339
344
  end
340
345
 
@@ -362,6 +367,18 @@ module Google
362
367
 
363
368
  protected
364
369
 
370
+ ##
371
+ # Starts a new thread to call wait! (blocking) on each Stream and then stop the TimedUnaryBuffer.
372
+ def wait_stop_buffer_thread!
373
+ synchronize do
374
+ @wait_stop_buffer_thread ||= Thread.new do
375
+ @stream_pool.map(&:wait!)
376
+ # Shutdown the buffer TimerTask (and flush the buffer) after the streams are all stopped.
377
+ @buffer.stop
378
+ end
379
+ end
380
+ end
381
+
365
382
  def coerce_inventory inventory
366
383
  @inventory = inventory
367
384
  if @inventory.is_a? Hash
@@ -377,6 +394,7 @@ module Google
377
394
  @inventory[:max_outstanding_bytes] = Integer(@inventory[:max_outstanding_bytes] || 100_000_000)
378
395
  @inventory[:max_total_lease_duration] = Integer(@inventory[:max_total_lease_duration] || 3600)
379
396
  @inventory[:max_duration_per_lease_extension] = Integer(@inventory[:max_duration_per_lease_extension] || 0)
397
+ @inventory[:use_legacy_flow_control] = @inventory[:use_legacy_flow_control] || false
380
398
  end
381
399
 
382
400
  def default_error_callbacks
@@ -30,15 +30,22 @@ module Google
30
30
 
31
31
  include MonitorMixin
32
32
 
33
- attr_reader :stream, :limit, :bytesize, :extension, :max_duration_per_lease_extension
34
-
35
- def initialize stream, limit:, bytesize:, extension:, max_duration_per_lease_extension:
33
+ attr_reader :stream
34
+ attr_reader :limit
35
+ attr_reader :bytesize
36
+ attr_reader :extension
37
+ attr_reader :max_duration_per_lease_extension
38
+ attr_reader :use_legacy_flow_control
39
+
40
+ def initialize stream, limit:, bytesize:, extension:, max_duration_per_lease_extension:,
41
+ use_legacy_flow_control:
36
42
  super()
37
43
  @stream = stream
38
44
  @limit = limit
39
45
  @bytesize = bytesize
40
46
  @extension = extension
41
47
  @max_duration_per_lease_extension = max_duration_per_lease_extension
48
+ @use_legacy_flow_control = use_legacy_flow_control
42
49
  @inventory = {}
43
50
  @wait_cond = new_cond
44
51
  end
@@ -58,7 +58,7 @@ module Google
58
58
  @paused = nil
59
59
  @pause_cond = new_cond
60
60
 
61
- @inventory = Inventory.new self, @subscriber.stream_inventory
61
+ @inventory = Inventory.new self, **@subscriber.stream_inventory
62
62
 
63
63
  @sequencer = Sequencer.new(&method(:perform_callback_async)) if subscriber.message_ordering
64
64
 
@@ -363,8 +363,8 @@ module Google
363
363
  req.modify_deadline_ack_ids += @inventory.ack_ids
364
364
  req.modify_deadline_seconds += @inventory.ack_ids.map { @subscriber.deadline }
365
365
  req.client_id = @subscriber.service.client_id
366
- req.max_outstanding_messages = @inventory.limit
367
- req.max_outstanding_bytes = @inventory.bytesize
366
+ req.max_outstanding_messages = @inventory.use_legacy_flow_control ? 0 : @inventory.limit
367
+ req.max_outstanding_bytes = @inventory.use_legacy_flow_control ? 0 : @inventory.bytesize
368
368
  end
369
369
  end
370
370
 
@@ -25,7 +25,8 @@ module Google
25
25
  class TimedUnaryBuffer
26
26
  include MonitorMixin
27
27
 
28
- attr_reader :max_bytes, :interval
28
+ attr_reader :max_bytes
29
+ attr_reader :interval
29
30
 
30
31
  def initialize subscriber, max_bytes: 500_000, interval: 1.0
31
32
  super() # to init MonitorMixin
@@ -138,7 +139,7 @@ module Google
138
139
  end
139
140
 
140
141
  groups = prev_reg.each_pair.group_by { |_ack_id, delay| delay }
141
- req_hash = Hash[groups.map { |k, v| [k, v.map(&:first)] }]
142
+ req_hash = groups.transform_values { |v| v.map(&:first) }
142
143
 
143
144
  requests = { acknowledge: [] }
144
145
  ack_ids = Array(req_hash.delete(:ack)) # ack has no deadline set
@@ -215,11 +216,9 @@ module Google
215
216
 
216
217
  def add_future pool
217
218
  Concurrent::Promises.future_on pool do
218
- begin
219
- yield
220
- rescue StandardError => e
221
- error! e
222
- end
219
+ yield
220
+ rescue StandardError => e
221
+ error! e
223
222
  end
224
223
  end
225
224
  end