google-cloud-pubsub 2.1.1 → 2.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -14,7 +14,7 @@
14
14
 
15
15
 
16
16
  require "googleauth"
17
- require "google/cloud/pubsub/v1/publisher/credentials.rb"
17
+ require "google/cloud/pubsub/v1/publisher/credentials"
18
18
 
19
19
  module Google
20
20
  module Cloud
@@ -78,6 +78,14 @@ module Google
78
78
  super "Can't publish message using #{ordering_key}."
79
79
  end
80
80
  end
81
+
82
+ ##
83
+ # Raised when the desired action is `error` and the message would exceed
84
+ # flow control limits, or when the desired action is `block` and the
85
+ # message would block forever against the flow control limits.
86
+ #
87
+ class FlowControlLimitError < Google::Cloud::Error
88
+ end
81
89
  end
82
90
 
83
91
  Pubsub = PubSub unless const_defined? :Pubsub
@@ -0,0 +1,139 @@
1
+ # Copyright 2021 Google LLC
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+
16
+ require "google/cloud/pubsub/errors"
17
+ require "concurrent/atomics"
18
+
19
+ module Google
20
+ module Cloud
21
+ module PubSub
22
+ ##
23
+ # @private
24
+ #
25
+ # Used to control the flow of messages passing through it.
26
+ #
27
+ class FlowController
28
+ attr_reader :message_limit
29
+ attr_reader :byte_limit
30
+ attr_reader :limit_exceeded_behavior
31
+ ##
32
+ # @private Implementation accessors
33
+ attr_reader :outstanding_messages, :outstanding_bytes, :awaiting
34
+
35
+ def initialize message_limit: 1000, byte_limit: 10_000_000, limit_exceeded_behavior: :ignore
36
+ unless [:ignore, :error, :block].include? limit_exceeded_behavior
37
+ raise ArgumentError, "limit_exceeded_behavior must be one of :ignore, :error, :block"
38
+ end
39
+ if [:error, :block].include?(limit_exceeded_behavior) && message_limit < 1
40
+ raise ArgumentError,
41
+ "Flow control message limit (#{message_limit}) exceeded by a single message, would block forever"
42
+ end
43
+ @mutex = Mutex.new
44
+ @message_limit = message_limit
45
+ @byte_limit = byte_limit
46
+ @limit_exceeded_behavior = limit_exceeded_behavior
47
+ @outstanding_messages = 0
48
+ @outstanding_bytes = 0
49
+
50
+ @awaiting = []
51
+ end
52
+
53
+ def acquire message_size
54
+ return if limit_exceeded_behavior == :ignore
55
+ @mutex.lock
56
+ if limit_exceeded_behavior == :error && would_exceed_message_limit?
57
+ raise FlowControlLimitError, "Flow control message limit (#{message_limit}) would be exceeded"
58
+ end
59
+ if limit_exceeded_behavior == :error && would_exceed_byte_limit?(message_size)
60
+ raise FlowControlLimitError,
61
+ "Flow control byte limit (#{byte_limit}) would be exceeded, message_size: #{message_size}"
62
+ end
63
+ if limit_exceeded_behavior == :block && message_size > byte_limit
64
+ raise FlowControlLimitError,
65
+ "Flow control byte limit (#{byte_limit}) exceeded by a single message, would block forever"
66
+ end
67
+
68
+ acquire_or_wait message_size
69
+ ensure
70
+ @mutex.unlock if @mutex.owned?
71
+ end
72
+
73
+ def release message_size
74
+ return if limit_exceeded_behavior == :ignore
75
+ @mutex.synchronize do
76
+ raise "Flow control messages count would be negative" if (@outstanding_messages - 1).negative?
77
+ raise "Flow control bytes count would be negative" if (@outstanding_bytes - message_size).negative?
78
+
79
+ @outstanding_messages -= 1
80
+ @outstanding_bytes -= message_size
81
+ @awaiting.first.set unless @awaiting.empty?
82
+ end
83
+ end
84
+
85
+ protected
86
+
87
+ # rubocop:disable Style/IdenticalConditionalBranches
88
+ # rubocop:disable Style/GuardClause
89
+
90
+ def acquire_or_wait message_size
91
+ waiter = nil
92
+ while is_new_and_others_wait?(waiter) ||
93
+ would_exceed_byte_limit?(message_size) ||
94
+ would_exceed_message_limit?
95
+
96
+ if waiter.nil?
97
+ waiter = Concurrent::Event.new
98
+ # This waiter gets added to the back of the line.
99
+ @awaiting << waiter
100
+ else
101
+ waiter = Concurrent::Event.new
102
+ # This waiter already in line stays at the head of the line.
103
+ @awaiting[0] = waiter
104
+ end
105
+ @mutex.unlock
106
+ waiter.wait
107
+ @mutex.lock
108
+ end
109
+ @outstanding_messages += 1
110
+ @outstanding_bytes += message_size
111
+
112
+ @awaiting.shift if waiter # Remove the newly released waiter from the head of the queue.
113
+
114
+ # There may be some surplus left; let the next message waiting try to acquire a permit.
115
+ if !@awaiting.empty? && @outstanding_bytes < byte_limit && @outstanding_messages < message_limit
116
+ @awaiting.first.set
117
+ end
118
+ end
119
+
120
+ # rubocop:enable Style/IdenticalConditionalBranches
121
+ # rubocop:enable Style/GuardClause
122
+
123
+ def is_new_and_others_wait? waiter
124
+ waiter.nil? && !@awaiting.empty?
125
+ end
126
+
127
+ def would_exceed_message_limit?
128
+ @outstanding_messages + 1 > message_limit
129
+ end
130
+
131
+ def would_exceed_byte_limit? bytes_requested
132
+ @outstanding_bytes + bytes_requested > byte_limit
133
+ end
134
+ end
135
+ end
136
+
137
+ Pubsub = PubSub unless const_defined? :Pubsub
138
+ end
139
+ end
@@ -68,7 +68,8 @@ module Google
68
68
  # end
69
69
  #
70
70
  class Policy
71
- attr_reader :etag, :roles
71
+ attr_reader :etag
72
+ attr_reader :roles
72
73
 
73
74
  ##
74
75
  # @private Creates a Policy object.
@@ -167,7 +168,7 @@ module Google
167
168
  role: role_name,
168
169
  members: roles[role_name]
169
170
  )
170
- end
171
+ end.compact
171
172
  )
172
173
  end
173
174
 
@@ -18,6 +18,7 @@ require "google/cloud/pubsub/service"
18
18
  require "google/cloud/pubsub/credentials"
19
19
  require "google/cloud/pubsub/topic"
20
20
  require "google/cloud/pubsub/batch_publisher"
21
+ require "google/cloud/pubsub/schema"
21
22
  require "google/cloud/pubsub/snapshot"
22
23
 
23
24
  module Google
@@ -75,10 +76,14 @@ module Google
75
76
  ##
76
77
  # Retrieves topic by name.
77
78
  #
78
- # @param [String] topic_name Name of a topic.
79
+ # @param [String] topic_name Name of a topic. The value can be a simple
80
+ # topic ID (relative name), in which case the current project ID will
81
+ # be supplied, or a fully-qualified topic name in the form
82
+ # `projects/{project_id}/topics/{topic_id}`.
79
83
  # @param [String] project If the topic belongs to a project other than
80
84
  # the one currently connected to, the alternate project ID can be
81
- # specified here. Optional.
85
+ # specified here. Optional. Not used if a fully-qualified topic name
86
+ # is provided for `topic_name`.
82
87
  # @param [Boolean] skip_lookup Optionally create a {Topic} object
83
88
  # without verifying the topic resource exists on the Pub/Sub service.
84
89
  # Calls made on this object will raise errors if the topic resource
@@ -98,6 +103,16 @@ module Google
98
103
  # * `:threads` (Hash) The number of threads to create to handle concurrent calls by the publisher:
99
104
  # * `:publish` (Integer) The number of threads used to publish messages. Default is 2.
100
105
  # * `:callback` (Integer) The number of threads to handle the published messages' callbacks. Default is 4.
106
+ # * `:flow_control` (Hash) The client flow control settings for message publishing:
107
+ # * `:message_limit` (Integer) The maximum number of messages allowed to wait to be published. Default is
108
+ # `10 * max_messages`.
109
+ # * `:byte_limit` (Integer) The maximum total size of messages allowed to wait to be published. Default is
110
+ # `10 * max_bytes`.
111
+ # * `:limit_exceeded_behavior` (Symbol) The action to take when publish flow control limits are exceeded.
112
+ # Possible values include: `:ignore` - Flow control is disabled. `:error` - Calls to {Topic#publish_async}
113
+ # will raise {FlowControlLimitError} when publish flow control limits are exceeded. `:block` - Calls to
114
+ # {Topic#publish_async} will block until capacity is available when publish flow control limits are
115
+ # exceeded. The default value is `:ignore`.
101
116
  #
102
117
  # @return [Google::Cloud::PubSub::Topic, nil] Returns `nil` if topic
103
118
  # does not exist.
@@ -147,7 +162,7 @@ module Google
147
162
  ensure_service!
148
163
  options = { project: project }
149
164
  return Topic.from_name topic_name, service, options if skip_lookup
150
- grpc = service.get_topic topic_name
165
+ grpc = service.get_topic topic_name, options
151
166
  Topic.from_grpc grpc, service, async: async
152
167
  rescue Google::Cloud::NotFoundError
153
168
  nil
@@ -158,7 +173,16 @@ module Google
158
173
  ##
159
174
  # Creates a new topic.
160
175
  #
161
- # @param [String] topic_name Name of a topic.
176
+ # @param [String] topic_name Name of a topic. Required.
177
+ # The value can be a simple topic ID (relative name), in which
178
+ # case the current project ID will be supplied, or a fully-qualified
179
+ # topic name in the form `projects/{project_id}/topics/{topic_id}`.
180
+ #
181
+ # The topic ID (relative name) must start with a letter, and
182
+ # contain only letters (`[A-Za-z]`), numbers (`[0-9]`), dashes (`-`),
183
+ # underscores (`_`), periods (`.`), tildes (`~`), plus (`+`) or percent
184
+ # signs (`%`). It must be between 3 and 255 characters in length, and
185
+ # it must not start with `goog`.
162
186
  # @param [Hash] labels A hash of user-provided labels associated with
163
187
  # the topic. You can use these to organize and group your topics.
164
188
  # Label keys and values can be no longer than 63 characters, can only
@@ -179,15 +203,41 @@ module Google
179
203
  #
180
204
  # Hash keys and values may include the following:
181
205
  #
182
- # * `:max_bytes` (Integer) The maximum size of messages to be collected before the batch is published. Default
183
- # is 1,000,000 (1MB).
184
- # * `:max_messages` (Integer) The maximum number of messages to be collected before the batch is published.
185
- # Default is 100.
186
- # * `:interval` (Numeric) The number of seconds to collect messages before the batch is published. Default is
187
- # 0.01.
188
- # * `:threads` (Hash) The number of threads to create to handle concurrent calls by the publisher:
189
- # * `:publish` (Integer) The number of threads used to publish messages. Default is 2.
190
- # * `:callback` (Integer) The number of threads to handle the published messages' callbacks. Default is 4.
206
+ # * `:max_bytes` (Integer) The maximum size of messages to be collected
207
+ # before the batch is published. Default is 1,000,000 (1MB).
208
+ # * `:max_messages` (Integer) The maximum number of messages to be
209
+ # collected before the batch is published. Default is 100.
210
+ # * `:interval` (Numeric) The number of seconds to collect messages before
211
+ # the batch is published. Default is 0.01.
212
+ # * `:threads` (Hash) The number of threads to create to handle concurrent
213
+ # calls by the publisher:
214
+ #
215
+ # * `:publish` (Integer) The number of threads used to publish messages.
216
+ # Default is 2.
217
+ # * `:callback` (Integer) The number of threads to handle the published
218
+ # messages' callbacks. Default is 4.
219
+ # * `:flow_control` (Hash) The client flow control settings for message publishing:
220
+ # * `:message_limit` (Integer) The maximum number of messages allowed to wait to be published. Default is
221
+ # `10 * max_messages`.
222
+ # * `:byte_limit` (Integer) The maximum total size of messages allowed to wait to be published. Default is
223
+ # `10 * max_bytes`.
224
+ # * `:limit_exceeded_behavior` (Symbol) The action to take when publish flow control limits are exceeded.
225
+ # Possible values include: `:ignore` - Flow control is disabled. `:error` - Calls to {Topic#publish_async}
226
+ # will raise {FlowControlLimitError} when publish flow control limits are exceeded. `:block` - Calls to
227
+ # {Topic#publish_async} will block until capacity is available when publish flow control limits are
228
+ # exceeded. The default value is `:ignore`.
229
+ # @param [String] schema_name The name of the schema that messages
230
+ # published should be validated against. Optional. The value can be a
231
+ # simple schema ID (relative name), in which case the current project
232
+ # ID will be supplied, or a fully-qualified schema name in the form
233
+ # `projects/{project_id}/schemas/{schema_id}`. If provided,
234
+ # `message_encoding` must also be provided.
235
+ # @param [String, Symbol] message_encoding The encoding of messages validated
236
+ # against the schema identified by `schema_name`. Optional. Values include:
237
+ #
238
+ # * `JSON` - JSON encoding.
239
+ # * `BINARY` - Binary encoding, as defined by the schema type. For some
240
+ # schema types, binary encoding may not be available.
191
241
  #
192
242
  # @return [Google::Cloud::PubSub::Topic]
193
243
  #
@@ -197,12 +247,20 @@ module Google
197
247
  # pubsub = Google::Cloud::PubSub.new
198
248
  # topic = pubsub.create_topic "my-topic"
199
249
  #
200
- def create_topic topic_name, labels: nil, kms_key: nil, persistence_regions: nil, async: nil
250
+ def create_topic topic_name,
251
+ labels: nil,
252
+ kms_key: nil,
253
+ persistence_regions: nil,
254
+ async: nil,
255
+ schema_name: nil,
256
+ message_encoding: nil
201
257
  ensure_service!
202
258
  grpc = service.create_topic topic_name,
203
259
  labels: labels,
204
260
  kms_key_name: kms_key,
205
- persistence_regions: persistence_regions
261
+ persistence_regions: persistence_regions,
262
+ schema_name: schema_name,
263
+ message_encoding: message_encoding
206
264
  Topic.from_grpc grpc, service, async: async
207
265
  end
208
266
  alias new_topic create_topic
@@ -250,10 +308,14 @@ module Google
250
308
  ##
251
309
  # Retrieves subscription by name.
252
310
  #
253
- # @param [String] subscription_name Name of a subscription.
311
+ # @param [String] subscription_name Name of a subscription. The value can
312
+ # be a simple subscription ID, in which case the current project ID
313
+ # will be supplied, or a fully-qualified subscription name in the form
314
+ # `projects/{project_id}/subscriptions/{subscription_id}`.
254
315
  # @param [String] project If the subscription belongs to a project other
255
316
  # than the one currently connected to, the alternate project ID can be
256
- # specified here.
317
+ # specified here. Not used if a fully-qualified subscription name is
318
+ # provided for `subscription_name`.
257
319
  # @param [Boolean] skip_lookup Optionally create a {Subscription} object
258
320
  # without verifying the subscription resource exists on the Pub/Sub
259
321
  # service. Calls made on this object will raise errors if the service
@@ -283,7 +345,7 @@ module Google
283
345
  ensure_service!
284
346
  options = { project: project }
285
347
  return Subscription.from_name subscription_name, service, options if skip_lookup
286
- grpc = service.get_subscription subscription_name
348
+ grpc = service.get_subscription subscription_name, options
287
349
  Subscription.from_grpc grpc, service
288
350
  rescue Google::Cloud::NotFoundError
289
351
  nil
@@ -370,6 +432,198 @@ module Google
370
432
  alias find_snapshots snapshots
371
433
  alias list_snapshots snapshots
372
434
 
435
+ ##
436
+ # Retrieves schema by name.
437
+ #
438
+ # @param [String] schema_name Name of a schema. The value can
439
+ # be a simple schema ID, in which case the current project ID
440
+ # will be supplied, or a fully-qualified schema name in the form
441
+ # `projects/{project_id}/schemas/{schema_id}`.
442
+ # @param view [Symbol, String, nil] Possible values:
443
+ # * `BASIC` - Include the `name` and `type` of the schema, but not the `definition`.
444
+ # * `FULL` - Include all Schema object fields.
445
+ #
446
+ # The default value is `FULL`.
447
+ # @param [String] project If the schema belongs to a project other
448
+ # than the one currently connected to, the alternate project ID can be
449
+ # specified here. Not used if a fully-qualified schema name is
450
+ # provided for `schema_name`.
451
+ # @param [Boolean] skip_lookup Optionally create a {Schema} object
452
+ # without verifying the schema resource exists on the Pub/Sub
453
+ # service. Calls made on this object will raise errors if the service
454
+ # resource does not exist. Default is `false`.
455
+ #
456
+ # @return [Google::Cloud::PubSub::Schema, nil] Returns `nil` if
457
+ # the schema does not exist.
458
+ #
459
+ # @example
460
+ # require "google/cloud/pubsub"
461
+ #
462
+ # pubsub = Google::Cloud::PubSub.new
463
+ #
464
+ # schema = pubsub.schema "my-schema"
465
+ # schema.name #=> "projects/my-project/schemas/my-schema"
466
+ # schema.type #=> :PROTOCOL_BUFFER
467
+ # schema.definition # The schema definition
468
+ #
469
+ # @example Skip the lookup against the service with `skip_lookup`:
470
+ # require "google/cloud/pubsub"
471
+ #
472
+ # pubsub = Google::Cloud::PubSub.new
473
+ #
474
+ # # No API call is made to retrieve the schema information.
475
+ # # The default project is used in the name.
476
+ # schema = pubsub.schema "my-schema", skip_lookup: true
477
+ # schema.name #=> "projects/my-project/schemas/my-schema"
478
+ # schema.type #=> nil
479
+ # schema.definition #=> nil
480
+ #
481
+ # @example Omit the schema definition with `view: :basic`:
482
+ # require "google/cloud/pubsub"
483
+ #
484
+ # pubsub = Google::Cloud::PubSub.new
485
+ #
486
+ # schema = pubsub.schema "my-schema", view: :basic
487
+ # schema.name #=> "projects/my-project/schemas/my-schema"
488
+ # schema.type #=> :PROTOCOL_BUFFER
489
+ # schema.definition #=> nil
490
+ #
491
+ def schema schema_name, view: nil, project: nil, skip_lookup: nil
492
+ ensure_service!
493
+ options = { project: project }
494
+ return Schema.from_name schema_name, view, service, options if skip_lookup
495
+ view ||= :FULL
496
+ grpc = service.get_schema schema_name, view, options
497
+ Schema.from_grpc grpc, service
498
+ rescue Google::Cloud::NotFoundError
499
+ nil
500
+ end
501
+ alias get_schema schema
502
+ alias find_schema schema
503
+
504
+ ##
505
+ # Creates a new schema.
506
+ #
507
+ # @param [String] schema_id The ID to use for the schema, which will
508
+ # become the final component of the schema's resource name. Required.
509
+ #
510
+ # The schema ID (relative name) must start with a letter, and
511
+ # contain only letters (`[A-Za-z]`), numbers (`[0-9]`), dashes (`-`),
512
+ # underscores (`_`), periods (`.`), tildes (`~`), plus (`+`) or percent
513
+ # signs (`%`). It must be between 3 and 255 characters in length, and
514
+ # it must not start with `goog`.
515
+ # @param [String, Symbol] type The type of the schema. Required. Possible
516
+ # values are case-insensitive and include:
517
+ #
518
+ # * `PROTOCOL_BUFFER` - A Protocol Buffer schema definition.
519
+ # * `AVRO` - An Avro schema definition.
520
+ # @param [String] definition The definition of the schema. Required. This
521
+ # should be a string representing the full definition of the schema that
522
+ # is a valid schema definition of the type specified in `type`.
523
+ # @param [String] project If the schema belongs to a project other
524
+ # than the one currently connected to, the alternate project ID can be
525
+ # specified here. Optional.
526
+ #
527
+ # @return [Google::Cloud::PubSub::Schema]
528
+ #
529
+ # @example
530
+ # require "google/cloud/pubsub"
531
+ #
532
+ # pubsub = Google::Cloud::PubSub.new
533
+ #
534
+ # definition = "..."
535
+ # schema = pubsub.create_schema "my-schema", :avro, definition
536
+ # schema.name #=> "projects/my-project/schemas/my-schema"
537
+ #
538
+ def create_schema schema_id, type, definition, project: nil
539
+ ensure_service!
540
+ type = type.to_s.upcase
541
+ grpc = service.create_schema schema_id, type, definition, project: project
542
+ Schema.from_grpc grpc, service
543
+ end
544
+ alias new_schema create_schema
545
+
546
+ ##
547
+ # Retrieves a list of schemas for the given project.
548
+ #
549
+ # @param view [String, Symbol, nil] The set of fields to return in the response. Possible values:
550
+ #
551
+ # * `BASIC` - Include the `name` and `type` of the schema, but not the `definition`.
552
+ # * `FULL` - Include all Schema object fields.
553
+ #
554
+ # The default value is `FULL`.
555
+ # @param [String] token A previously-returned page token representing
556
+ # part of the larger set of results to view.
557
+ # @param [Integer] max Maximum number of schemas to return.
558
+ #
559
+ # @return [Array<Google::Cloud::PubSub::Schema>] (See
560
+ # {Google::Cloud::PubSub::Schema::List})
561
+ #
562
+ # @example
563
+ # require "google/cloud/pubsub"
564
+ #
565
+ # pubsub = Google::Cloud::PubSub.new
566
+ #
567
+ # schemas = pubsub.schemas
568
+ # schemas.each do |schema|
569
+ # puts schema.name
570
+ # end
571
+ #
572
+ # @example Retrieve all schemas: (See {Schema::List#all})
573
+ # require "google/cloud/pubsub"
574
+ #
575
+ # pubsub = Google::Cloud::PubSub.new
576
+ #
577
+ # schemas = pubsub.schemas
578
+ # schemas.all do |schema|
579
+ # puts schema.name
580
+ # end
581
+ #
582
+ def schemas view: nil, token: nil, max: nil
583
+ ensure_service!
584
+ view ||= :FULL
585
+ options = { token: token, max: max }
586
+ grpc = service.list_schemas view, options
587
+ Schema::List.from_grpc grpc, service, view, max
588
+ end
589
+ alias find_schemas schemas
590
+ alias list_schemas schemas
591
+
592
+ ##
593
+ # Validates a schema type and definition.
594
+ #
595
+ # @param [String, Symbol] type The type of the schema. Required. Possible
596
+ # values are case-insensitive and include:
597
+ #
598
+ # * `PROTOCOL_BUFFER` - A Protocol Buffer schema definition.
599
+ # * `AVRO` - An Avro schema definition.
600
+ # @param [String] definition The definition of the schema. Required. This
601
+ # should be a string representing the full definition of the schema that
602
+ # is a valid schema definition of the type specified in `type`.
603
+ # @param [String] project If the schema belongs to a project other
604
+ # than the one currently connected to, the alternate project ID can be
605
+ # specified here. Optional.
606
+ #
607
+ # @return [Boolean] `true` if the schema is valid, `false` otherwise.
608
+ #
609
+ # @example
610
+ # require "google/cloud/pubsub"
611
+ #
612
+ # pubsub = Google::Cloud::PubSub.new
613
+ #
614
+ # definition = "..."
615
+ # pubsub.validate_schema :avro, definition #=> true
616
+ #
617
+ def valid_schema? type, definition, project: nil
618
+ ensure_service!
619
+ type = type.to_s.upcase
620
+ service.validate_schema type, definition, project: project # return type is empty
621
+ true
622
+ rescue Google::Cloud::InvalidArgumentError
623
+ false
624
+ end
625
+ alias validate_schema valid_schema?
626
+
373
627
  protected
374
628
 
375
629
  ##