google-cloud-pubsub 1.0.2 → 2.19.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (58) hide show
  1. checksums.yaml +4 -4
  2. data/AUTHENTICATION.md +16 -54
  3. data/CHANGELOG.md +464 -0
  4. data/CONTRIBUTING.md +328 -116
  5. data/EMULATOR.md +1 -1
  6. data/LOGGING.md +94 -2
  7. data/OVERVIEW.md +121 -68
  8. data/TROUBLESHOOTING.md +2 -8
  9. data/lib/google/cloud/pubsub/acknowledge_result.rb +79 -0
  10. data/lib/google/cloud/pubsub/async_publisher/batch.rb +319 -0
  11. data/lib/google/cloud/pubsub/async_publisher.rb +231 -156
  12. data/lib/google/cloud/pubsub/batch_publisher.rb +60 -30
  13. data/lib/google/cloud/pubsub/convert.rb +33 -7
  14. data/lib/google/cloud/pubsub/credentials.rb +2 -2
  15. data/lib/google/cloud/pubsub/errors.rb +93 -0
  16. data/lib/google/cloud/pubsub/flow_controller.rb +137 -0
  17. data/lib/google/cloud/pubsub/message.rb +45 -4
  18. data/lib/google/cloud/pubsub/policy.rb +3 -2
  19. data/lib/google/cloud/pubsub/project.rb +316 -49
  20. data/lib/google/cloud/pubsub/publish_result.rb +6 -1
  21. data/lib/google/cloud/pubsub/received_message.rb +171 -10
  22. data/lib/google/cloud/pubsub/retry_policy.rb +88 -0
  23. data/lib/google/cloud/pubsub/schema/list.rb +180 -0
  24. data/lib/google/cloud/pubsub/schema.rb +310 -0
  25. data/lib/google/cloud/pubsub/service.rb +285 -269
  26. data/lib/google/cloud/pubsub/snapshot/list.rb +4 -6
  27. data/lib/google/cloud/pubsub/snapshot.rb +5 -2
  28. data/lib/google/cloud/pubsub/subscriber/inventory.rb +69 -32
  29. data/lib/google/cloud/pubsub/subscriber/sequencer.rb +115 -0
  30. data/lib/google/cloud/pubsub/subscriber/stream.rb +108 -49
  31. data/lib/google/cloud/pubsub/subscriber/timed_unary_buffer.rb +191 -30
  32. data/lib/google/cloud/pubsub/subscriber.rb +155 -45
  33. data/lib/google/cloud/pubsub/subscription/list.rb +4 -6
  34. data/lib/google/cloud/pubsub/subscription/push_config.rb +55 -31
  35. data/lib/google/cloud/pubsub/subscription.rb +561 -77
  36. data/lib/google/cloud/pubsub/topic/list.rb +4 -6
  37. data/lib/google/cloud/pubsub/topic.rb +372 -52
  38. data/lib/google/cloud/pubsub/version.rb +1 -1
  39. data/lib/google/cloud/pubsub.rb +35 -46
  40. data/lib/google-cloud-pubsub.rb +21 -27
  41. metadata +26 -189
  42. data/lib/google/cloud/pubsub/v1/credentials.rb +0 -41
  43. data/lib/google/cloud/pubsub/v1/doc/google/iam/v1/iam_policy.rb +0 -21
  44. data/lib/google/cloud/pubsub/v1/doc/google/iam/v1/options.rb +0 -21
  45. data/lib/google/cloud/pubsub/v1/doc/google/iam/v1/policy.rb +0 -21
  46. data/lib/google/cloud/pubsub/v1/doc/google/protobuf/duration.rb +0 -91
  47. data/lib/google/cloud/pubsub/v1/doc/google/protobuf/empty.rb +0 -29
  48. data/lib/google/cloud/pubsub/v1/doc/google/protobuf/field_mask.rb +0 -222
  49. data/lib/google/cloud/pubsub/v1/doc/google/protobuf/timestamp.rb +0 -113
  50. data/lib/google/cloud/pubsub/v1/doc/google/pubsub/v1/pubsub.rb +0 -744
  51. data/lib/google/cloud/pubsub/v1/doc/google/type/expr.rb +0 -19
  52. data/lib/google/cloud/pubsub/v1/publisher_client.rb +0 -786
  53. data/lib/google/cloud/pubsub/v1/publisher_client_config.json +0 -105
  54. data/lib/google/cloud/pubsub/v1/subscriber_client.rb +0 -1385
  55. data/lib/google/cloud/pubsub/v1/subscriber_client_config.json +0 -138
  56. data/lib/google/cloud/pubsub/v1.rb +0 -17
  57. data/lib/google/pubsub/v1/pubsub_pb.rb +0 -249
  58. data/lib/google/pubsub/v1/pubsub_services_pb.rb +0 -211
@@ -28,9 +28,7 @@ module Google
28
28
  # Force the object to be a Time object.
29
29
  time = time.to_time
30
30
 
31
- Google::Protobuf::Timestamp.new \
32
- seconds: time.to_i,
33
- nanos: time.nsec
31
+ Google::Protobuf::Timestamp.new seconds: time.to_i, nanos: time.nsec
34
32
  end
35
33
 
36
34
  def timestamp_to_time timestamp
@@ -42,9 +40,7 @@ module Google
42
40
  def number_to_duration number
43
41
  return nil if number.nil?
44
42
 
45
- Google::Protobuf::Duration.new \
46
- seconds: number.to_i,
47
- nanos: (number.remainder(1) * 1000000000).round
43
+ Google::Protobuf::Duration.new seconds: number.to_i, nanos: (number.remainder(1) * 1_000_000_000).round
48
44
  end
49
45
 
50
46
  def duration_to_number duration
@@ -52,7 +48,37 @@ module Google
52
48
 
53
49
  return duration.seconds if duration.nanos.zero?
54
50
 
55
- duration.seconds + (duration.nanos / 1000000000.0)
51
+ duration.seconds + (duration.nanos / 1_000_000_000.0)
52
+ end
53
+
54
+ def pubsub_message data, attributes, ordering_key, extra_attrs
55
+ # TODO: allow data to be a Message object,
56
+ # then ensure attributes and ordering_key are nil
57
+ if data.is_a?(::Hash) && (attributes.nil? || attributes.empty?)
58
+ attributes = data.merge extra_attrs
59
+ data = nil
60
+ else
61
+ attributes = Hash(attributes).merge extra_attrs
62
+ end
63
+ # Convert IO-ish objects to strings
64
+ if data.respond_to?(:read) && data.respond_to?(:rewind)
65
+ data.rewind
66
+ data = data.read
67
+ end
68
+ # Convert data to encoded byte array to match the protobuf defn
69
+ data_bytes = String(data).dup.force_encoding(Encoding::ASCII_8BIT).freeze
70
+
71
+ # Convert attributes to strings to match the protobuf definition
72
+ attributes = attributes.to_h { |k, v| [String(k), String(v)] }
73
+
74
+ # Ordering Key must always be a string
75
+ ordering_key = String(ordering_key).freeze
76
+
77
+ Google::Cloud::PubSub::V1::PubsubMessage.new(
78
+ data: data_bytes,
79
+ attributes: attributes,
80
+ ordering_key: ordering_key
81
+ )
56
82
  end
57
83
  end
58
84
 
@@ -14,7 +14,7 @@
14
14
 
15
15
 
16
16
  require "googleauth"
17
- require "google/cloud/pubsub/v1/credentials.rb"
17
+ require "google/cloud/pubsub/v1/publisher/credentials"
18
18
 
19
19
  module Google
20
20
  module Cloud
@@ -38,7 +38,7 @@ module Google
38
38
  #
39
39
  # pubsub.project_id #=> "my-project"
40
40
  #
41
- class Credentials < Google::Cloud::PubSub::V1::Credentials
41
+ class Credentials < Google::Cloud::PubSub::V1::Publisher::Credentials
42
42
  end
43
43
  end
44
44
 
@@ -0,0 +1,93 @@
1
+ # Copyright 2019 Google LLC
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+
16
+ require "google/cloud/errors"
17
+
18
+ module Google
19
+ module Cloud
20
+ module PubSub
21
+ ##
22
+ # Indicates that the {AsyncPublisher} has been stopped and cannot accept
23
+ # messages to publish.
24
+ #
25
+ class AsyncPublisherStopped < Google::Cloud::Error
26
+ def initialize message = "Can't publish when stopped."
27
+ super message
28
+ end
29
+ end
30
+
31
+ ##
32
+ # Indicates that the {AsyncPublisher} has not been enabled to publish
33
+ # messages with an ordering key. Use
34
+ # {AsyncPublisher#enable_message_ordering!} to enable publishing ordered
35
+ # messages.
36
+ #
37
+ class OrderedMessagesDisabled < Google::Cloud::Error
38
+ def initialize message = "Ordered messages are disabled."
39
+ super message
40
+ end
41
+ end
42
+
43
+ ##
44
+ # Indicates that the {Subscriber} for a {Subscription} with message
45
+ # ordering enabled has observed that a message has been delivered out of
46
+ # order.
47
+ #
48
+ class OrderedMessageDeliveryError < Google::Cloud::Error
49
+ attr_reader :ordered_message
50
+
51
+ def initialize ordered_message
52
+ @ordered_message = ordered_message
53
+
54
+ super "Ordered message delivered out of order."
55
+ end
56
+ end
57
+
58
+ ##
59
+ # Indicates that messages using the {#ordering_key} are not being
60
+ # published due to error. Future calls to {Topic#publish_async} with the
61
+ # {#ordering_key} will fail with this error.
62
+ #
63
+ # To allow future messages with the {#ordering_key} to be published, the
64
+ # {#ordering_key} must be passed to {Topic#resume_publish}.
65
+ #
66
+ # If this error is retrieved from {PublishResult#error}, inspect `cause`
67
+ # for the error raised while publishing.
68
+ #
69
+ # @!attribute [r] ordering_key
70
+ # @return [String] The ordering key that is in a failed state.
71
+ #
72
+ class OrderingKeyError < Google::Cloud::Error
73
+ attr_reader :ordering_key
74
+
75
+ def initialize ordering_key
76
+ @ordering_key = ordering_key
77
+
78
+ super "Can't publish message using #{ordering_key}."
79
+ end
80
+ end
81
+
82
+ ##
83
+ # Raised when the desired action is `error` and the message would exceed
84
+ # flow control limits, or when the desired action is `block` and the
85
+ # message would block forever against the flow control limits.
86
+ #
87
+ class FlowControlLimitError < Google::Cloud::Error
88
+ end
89
+ end
90
+
91
+ Pubsub = PubSub unless const_defined? :Pubsub
92
+ end
93
+ end
@@ -0,0 +1,137 @@
1
+ # Copyright 2021 Google LLC
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+
16
+ require "google/cloud/pubsub/errors"
17
+ require "concurrent/atomics"
18
+
19
+ module Google
20
+ module Cloud
21
+ module PubSub
22
+ ##
23
+ # @private
24
+ #
25
+ # Used to control the flow of messages passing through it.
26
+ #
27
+ class FlowController
28
+ attr_reader :message_limit
29
+ attr_reader :byte_limit
30
+ attr_reader :limit_exceeded_behavior
31
+ ##
32
+ # @private Implementation accessors
33
+ attr_reader :outstanding_messages, :outstanding_bytes, :awaiting
34
+
35
+ def initialize message_limit: 1000, byte_limit: 10_000_000, limit_exceeded_behavior: :ignore
36
+ unless [:ignore, :error, :block].include? limit_exceeded_behavior
37
+ raise ArgumentError, "limit_exceeded_behavior must be one of :ignore, :error, :block"
38
+ end
39
+ if [:error, :block].include?(limit_exceeded_behavior) && message_limit < 1
40
+ raise ArgumentError,
41
+ "Flow control message limit (#{message_limit}) exceeded by a single message, would block forever"
42
+ end
43
+ @mutex = Mutex.new
44
+ @message_limit = message_limit
45
+ @byte_limit = byte_limit
46
+ @limit_exceeded_behavior = limit_exceeded_behavior
47
+ @outstanding_messages = 0
48
+ @outstanding_bytes = 0
49
+
50
+ @awaiting = []
51
+ end
52
+
53
+ def acquire message_size
54
+ return if limit_exceeded_behavior == :ignore
55
+ @mutex.lock
56
+ if limit_exceeded_behavior == :error && would_exceed_message_limit?
57
+ raise FlowControlLimitError, "Flow control message limit (#{message_limit}) would be exceeded"
58
+ end
59
+ if limit_exceeded_behavior == :error && would_exceed_byte_limit?(message_size)
60
+ raise FlowControlLimitError,
61
+ "Flow control byte limit (#{byte_limit}) would be exceeded, message_size: #{message_size}"
62
+ end
63
+ if limit_exceeded_behavior == :block && message_size > byte_limit
64
+ raise FlowControlLimitError,
65
+ "Flow control byte limit (#{byte_limit}) exceeded by a single message, would block forever"
66
+ end
67
+
68
+ acquire_or_wait message_size
69
+ ensure
70
+ @mutex.unlock if @mutex.owned?
71
+ end
72
+
73
+ def release message_size
74
+ return if limit_exceeded_behavior == :ignore
75
+ @mutex.synchronize do
76
+ raise "Flow control messages count would be negative" if (@outstanding_messages - 1).negative?
77
+ raise "Flow control bytes count would be negative" if (@outstanding_bytes - message_size).negative?
78
+
79
+ @outstanding_messages -= 1
80
+ @outstanding_bytes -= message_size
81
+ @awaiting.first.set unless @awaiting.empty?
82
+ end
83
+ end
84
+
85
+ protected
86
+
87
+ # rubocop:disable Style/GuardClause
88
+
89
+ def acquire_or_wait message_size
90
+ waiter = nil
91
+ while is_new_and_others_wait?(waiter) ||
92
+ would_exceed_byte_limit?(message_size) ||
93
+ would_exceed_message_limit?
94
+
95
+ if waiter.nil?
96
+ waiter = Concurrent::Event.new
97
+ # This waiter gets added to the back of the line.
98
+ @awaiting << waiter
99
+ else
100
+ waiter = Concurrent::Event.new
101
+ # This waiter already in line stays at the head of the line.
102
+ @awaiting[0] = waiter
103
+ end
104
+ @mutex.unlock
105
+ waiter.wait
106
+ @mutex.lock
107
+ end
108
+ @outstanding_messages += 1
109
+ @outstanding_bytes += message_size
110
+
111
+ @awaiting.shift if waiter # Remove the newly released waiter from the head of the queue.
112
+
113
+ # There may be some surplus left; let the next message waiting try to acquire a permit.
114
+ if !@awaiting.empty? && @outstanding_bytes < byte_limit && @outstanding_messages < message_limit
115
+ @awaiting.first.set
116
+ end
117
+ end
118
+
119
+ # rubocop:enable Style/GuardClause
120
+
121
+ def is_new_and_others_wait? waiter
122
+ waiter.nil? && !@awaiting.empty?
123
+ end
124
+
125
+ def would_exceed_message_limit?
126
+ @outstanding_messages + 1 > message_limit
127
+ end
128
+
129
+ def would_exceed_byte_limit? bytes_requested
130
+ @outstanding_bytes + bytes_requested > byte_limit
131
+ end
132
+ end
133
+ end
134
+
135
+ Pubsub = PubSub unless const_defined? :Pubsub
136
+ end
137
+ end
@@ -50,7 +50,7 @@ module Google
50
50
  # subscriber.start
51
51
  #
52
52
  # # Shut down the subscriber when ready to stop receiving messages.
53
- # subscriber.stop.wait!
53
+ # subscriber.stop!
54
54
  #
55
55
  class Message
56
56
  ##
@@ -62,7 +62,7 @@ module Google
62
62
  # This can be used to publish several messages in bulk.
63
63
  def initialize data = nil, attributes = {}
64
64
  # Convert attributes to strings to match the protobuf definition
65
- attributes = Hash[attributes.map { |k, v| [String(k), String(v)] }]
65
+ attributes = attributes.to_h { |k, v| [String(k), String(v)] }
66
66
 
67
67
  @grpc = Google::Cloud::PubSub::V1::PubsubMessage.new(
68
68
  data: String(data).dup.force_encoding(Encoding::ASCII_8BIT),
@@ -81,8 +81,7 @@ module Google
81
81
  # Optional attributes for the message.
82
82
  def attributes
83
83
  return @grpc.attributes.to_h if @grpc.attributes.respond_to? :to_h
84
- # Enumerable doesn't have to_h on Ruby 2.0, so fallback to this
85
- Hash[@grpc.attributes.to_a]
84
+ @grpc.attributes.to_a.to_h
86
85
  end
87
86
 
88
87
  ##
@@ -100,6 +99,48 @@ module Google
100
99
  end
101
100
  alias publish_time published_at
102
101
 
102
+ ##
103
+ # Identifies related messages for which publish order should be
104
+ # respected.
105
+ #
106
+ # Google Cloud Pub/Sub ordering keys provide the ability to ensure
107
+ # related messages are sent to subscribers in the order in which they
108
+ # were published. Messages can be tagged with an ordering key, a string
109
+ # that identifies related messages for which publish order should be
110
+ # respected. The service guarantees that, for a given ordering key and
111
+ # publisher, messages are sent to subscribers in the order in which they
112
+ # were published. Ordering does not require sacrificing high throughput
113
+ # or scalability, as the service automatically distributes messages for
114
+ # different ordering keys across subscribers.
115
+ #
116
+ # See {Topic#publish_async} and {Subscription#listen}.
117
+ #
118
+ # @return [String]
119
+ #
120
+ def ordering_key
121
+ @grpc.ordering_key
122
+ end
123
+
124
+ # @private
125
+ def hash
126
+ @grpc.hash
127
+ end
128
+
129
+ # @private
130
+ def eql? other
131
+ return false unless other.is_a? self.class
132
+ @grpc.hash == other.hash
133
+ end
134
+ # @private
135
+ alias == eql?
136
+
137
+ # @private
138
+ def <=> other
139
+ return nil unless other.is_a? self.class
140
+ other_grpc = other.instance_variable_get :@grpc
141
+ @grpc <=> other_grpc
142
+ end
143
+
103
144
  ##
104
145
  # @private New Message from a Google::Cloud::PubSub::V1::PubsubMessage
105
146
  # object.
@@ -68,7 +68,8 @@ module Google
68
68
  # end
69
69
  #
70
70
  class Policy
71
- attr_reader :etag, :roles
71
+ attr_reader :etag
72
+ attr_reader :roles
72
73
 
73
74
  ##
74
75
  # @private Creates a Policy object.
@@ -167,7 +168,7 @@ module Google
167
168
  role: role_name,
168
169
  members: roles[role_name]
169
170
  )
170
- end
171
+ end.compact
171
172
  )
172
173
  end
173
174