google-cloud-pubsub 1.0.2 → 2.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/AUTHENTICATION.md +16 -54
- data/CHANGELOG.md +464 -0
- data/CONTRIBUTING.md +328 -116
- data/EMULATOR.md +1 -1
- data/LOGGING.md +94 -2
- data/OVERVIEW.md +121 -68
- data/TROUBLESHOOTING.md +2 -8
- data/lib/google/cloud/pubsub/acknowledge_result.rb +79 -0
- data/lib/google/cloud/pubsub/async_publisher/batch.rb +319 -0
- data/lib/google/cloud/pubsub/async_publisher.rb +231 -156
- data/lib/google/cloud/pubsub/batch_publisher.rb +60 -30
- data/lib/google/cloud/pubsub/convert.rb +33 -7
- data/lib/google/cloud/pubsub/credentials.rb +2 -2
- data/lib/google/cloud/pubsub/errors.rb +93 -0
- data/lib/google/cloud/pubsub/flow_controller.rb +137 -0
- data/lib/google/cloud/pubsub/message.rb +45 -4
- data/lib/google/cloud/pubsub/policy.rb +3 -2
- data/lib/google/cloud/pubsub/project.rb +316 -49
- data/lib/google/cloud/pubsub/publish_result.rb +6 -1
- data/lib/google/cloud/pubsub/received_message.rb +171 -10
- data/lib/google/cloud/pubsub/retry_policy.rb +88 -0
- data/lib/google/cloud/pubsub/schema/list.rb +180 -0
- data/lib/google/cloud/pubsub/schema.rb +310 -0
- data/lib/google/cloud/pubsub/service.rb +285 -269
- data/lib/google/cloud/pubsub/snapshot/list.rb +4 -6
- data/lib/google/cloud/pubsub/snapshot.rb +5 -2
- data/lib/google/cloud/pubsub/subscriber/inventory.rb +69 -32
- data/lib/google/cloud/pubsub/subscriber/sequencer.rb +115 -0
- data/lib/google/cloud/pubsub/subscriber/stream.rb +108 -49
- data/lib/google/cloud/pubsub/subscriber/timed_unary_buffer.rb +191 -30
- data/lib/google/cloud/pubsub/subscriber.rb +155 -45
- data/lib/google/cloud/pubsub/subscription/list.rb +4 -6
- data/lib/google/cloud/pubsub/subscription/push_config.rb +55 -31
- data/lib/google/cloud/pubsub/subscription.rb +561 -77
- data/lib/google/cloud/pubsub/topic/list.rb +4 -6
- data/lib/google/cloud/pubsub/topic.rb +372 -52
- data/lib/google/cloud/pubsub/version.rb +1 -1
- data/lib/google/cloud/pubsub.rb +35 -46
- data/lib/google-cloud-pubsub.rb +21 -27
- metadata +26 -189
- data/lib/google/cloud/pubsub/v1/credentials.rb +0 -41
- data/lib/google/cloud/pubsub/v1/doc/google/iam/v1/iam_policy.rb +0 -21
- data/lib/google/cloud/pubsub/v1/doc/google/iam/v1/options.rb +0 -21
- data/lib/google/cloud/pubsub/v1/doc/google/iam/v1/policy.rb +0 -21
- data/lib/google/cloud/pubsub/v1/doc/google/protobuf/duration.rb +0 -91
- data/lib/google/cloud/pubsub/v1/doc/google/protobuf/empty.rb +0 -29
- data/lib/google/cloud/pubsub/v1/doc/google/protobuf/field_mask.rb +0 -222
- data/lib/google/cloud/pubsub/v1/doc/google/protobuf/timestamp.rb +0 -113
- data/lib/google/cloud/pubsub/v1/doc/google/pubsub/v1/pubsub.rb +0 -744
- data/lib/google/cloud/pubsub/v1/doc/google/type/expr.rb +0 -19
- data/lib/google/cloud/pubsub/v1/publisher_client.rb +0 -786
- data/lib/google/cloud/pubsub/v1/publisher_client_config.json +0 -105
- data/lib/google/cloud/pubsub/v1/subscriber_client.rb +0 -1385
- data/lib/google/cloud/pubsub/v1/subscriber_client_config.json +0 -138
- data/lib/google/cloud/pubsub/v1.rb +0 -17
- data/lib/google/pubsub/v1/pubsub_pb.rb +0 -249
- data/lib/google/pubsub/v1/pubsub_services_pb.rb +0 -211
@@ -28,9 +28,7 @@ module Google
|
|
28
28
|
# Force the object to be a Time object.
|
29
29
|
time = time.to_time
|
30
30
|
|
31
|
-
Google::Protobuf::Timestamp.new
|
32
|
-
seconds: time.to_i,
|
33
|
-
nanos: time.nsec
|
31
|
+
Google::Protobuf::Timestamp.new seconds: time.to_i, nanos: time.nsec
|
34
32
|
end
|
35
33
|
|
36
34
|
def timestamp_to_time timestamp
|
@@ -42,9 +40,7 @@ module Google
|
|
42
40
|
def number_to_duration number
|
43
41
|
return nil if number.nil?
|
44
42
|
|
45
|
-
Google::Protobuf::Duration.new
|
46
|
-
seconds: number.to_i,
|
47
|
-
nanos: (number.remainder(1) * 1000000000).round
|
43
|
+
Google::Protobuf::Duration.new seconds: number.to_i, nanos: (number.remainder(1) * 1_000_000_000).round
|
48
44
|
end
|
49
45
|
|
50
46
|
def duration_to_number duration
|
@@ -52,7 +48,37 @@ module Google
|
|
52
48
|
|
53
49
|
return duration.seconds if duration.nanos.zero?
|
54
50
|
|
55
|
-
duration.seconds + (duration.nanos /
|
51
|
+
duration.seconds + (duration.nanos / 1_000_000_000.0)
|
52
|
+
end
|
53
|
+
|
54
|
+
def pubsub_message data, attributes, ordering_key, extra_attrs
|
55
|
+
# TODO: allow data to be a Message object,
|
56
|
+
# then ensure attributes and ordering_key are nil
|
57
|
+
if data.is_a?(::Hash) && (attributes.nil? || attributes.empty?)
|
58
|
+
attributes = data.merge extra_attrs
|
59
|
+
data = nil
|
60
|
+
else
|
61
|
+
attributes = Hash(attributes).merge extra_attrs
|
62
|
+
end
|
63
|
+
# Convert IO-ish objects to strings
|
64
|
+
if data.respond_to?(:read) && data.respond_to?(:rewind)
|
65
|
+
data.rewind
|
66
|
+
data = data.read
|
67
|
+
end
|
68
|
+
# Convert data to encoded byte array to match the protobuf defn
|
69
|
+
data_bytes = String(data).dup.force_encoding(Encoding::ASCII_8BIT).freeze
|
70
|
+
|
71
|
+
# Convert attributes to strings to match the protobuf definition
|
72
|
+
attributes = attributes.to_h { |k, v| [String(k), String(v)] }
|
73
|
+
|
74
|
+
# Ordering Key must always be a string
|
75
|
+
ordering_key = String(ordering_key).freeze
|
76
|
+
|
77
|
+
Google::Cloud::PubSub::V1::PubsubMessage.new(
|
78
|
+
data: data_bytes,
|
79
|
+
attributes: attributes,
|
80
|
+
ordering_key: ordering_key
|
81
|
+
)
|
56
82
|
end
|
57
83
|
end
|
58
84
|
|
@@ -14,7 +14,7 @@
|
|
14
14
|
|
15
15
|
|
16
16
|
require "googleauth"
|
17
|
-
require "google/cloud/pubsub/v1/credentials
|
17
|
+
require "google/cloud/pubsub/v1/publisher/credentials"
|
18
18
|
|
19
19
|
module Google
|
20
20
|
module Cloud
|
@@ -38,7 +38,7 @@ module Google
|
|
38
38
|
#
|
39
39
|
# pubsub.project_id #=> "my-project"
|
40
40
|
#
|
41
|
-
class Credentials < Google::Cloud::PubSub::V1::Credentials
|
41
|
+
class Credentials < Google::Cloud::PubSub::V1::Publisher::Credentials
|
42
42
|
end
|
43
43
|
end
|
44
44
|
|
@@ -0,0 +1,93 @@
|
|
1
|
+
# Copyright 2019 Google LLC
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
|
16
|
+
require "google/cloud/errors"
|
17
|
+
|
18
|
+
module Google
|
19
|
+
module Cloud
|
20
|
+
module PubSub
|
21
|
+
##
|
22
|
+
# Indicates that the {AsyncPublisher} has been stopped and cannot accept
|
23
|
+
# messages to publish.
|
24
|
+
#
|
25
|
+
class AsyncPublisherStopped < Google::Cloud::Error
|
26
|
+
def initialize message = "Can't publish when stopped."
|
27
|
+
super message
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
##
|
32
|
+
# Indicates that the {AsyncPublisher} has not been enabled to publish
|
33
|
+
# messages with an ordering key. Use
|
34
|
+
# {AsyncPublisher#enable_message_ordering!} to enable publishing ordered
|
35
|
+
# messages.
|
36
|
+
#
|
37
|
+
class OrderedMessagesDisabled < Google::Cloud::Error
|
38
|
+
def initialize message = "Ordered messages are disabled."
|
39
|
+
super message
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
##
|
44
|
+
# Indicates that the {Subscriber} for a {Subscription} with message
|
45
|
+
# ordering enabled has observed that a message has been delivered out of
|
46
|
+
# order.
|
47
|
+
#
|
48
|
+
class OrderedMessageDeliveryError < Google::Cloud::Error
|
49
|
+
attr_reader :ordered_message
|
50
|
+
|
51
|
+
def initialize ordered_message
|
52
|
+
@ordered_message = ordered_message
|
53
|
+
|
54
|
+
super "Ordered message delivered out of order."
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
##
|
59
|
+
# Indicates that messages using the {#ordering_key} are not being
|
60
|
+
# published due to error. Future calls to {Topic#publish_async} with the
|
61
|
+
# {#ordering_key} will fail with this error.
|
62
|
+
#
|
63
|
+
# To allow future messages with the {#ordering_key} to be published, the
|
64
|
+
# {#ordering_key} must be passed to {Topic#resume_publish}.
|
65
|
+
#
|
66
|
+
# If this error is retrieved from {PublishResult#error}, inspect `cause`
|
67
|
+
# for the error raised while publishing.
|
68
|
+
#
|
69
|
+
# @!attribute [r] ordering_key
|
70
|
+
# @return [String] The ordering key that is in a failed state.
|
71
|
+
#
|
72
|
+
class OrderingKeyError < Google::Cloud::Error
|
73
|
+
attr_reader :ordering_key
|
74
|
+
|
75
|
+
def initialize ordering_key
|
76
|
+
@ordering_key = ordering_key
|
77
|
+
|
78
|
+
super "Can't publish message using #{ordering_key}."
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
##
|
83
|
+
# Raised when the desired action is `error` and the message would exceed
|
84
|
+
# flow control limits, or when the desired action is `block` and the
|
85
|
+
# message would block forever against the flow control limits.
|
86
|
+
#
|
87
|
+
class FlowControlLimitError < Google::Cloud::Error
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
Pubsub = PubSub unless const_defined? :Pubsub
|
92
|
+
end
|
93
|
+
end
|
@@ -0,0 +1,137 @@
|
|
1
|
+
# Copyright 2021 Google LLC
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
|
16
|
+
require "google/cloud/pubsub/errors"
|
17
|
+
require "concurrent/atomics"
|
18
|
+
|
19
|
+
module Google
|
20
|
+
module Cloud
|
21
|
+
module PubSub
|
22
|
+
##
|
23
|
+
# @private
|
24
|
+
#
|
25
|
+
# Used to control the flow of messages passing through it.
|
26
|
+
#
|
27
|
+
class FlowController
|
28
|
+
attr_reader :message_limit
|
29
|
+
attr_reader :byte_limit
|
30
|
+
attr_reader :limit_exceeded_behavior
|
31
|
+
##
|
32
|
+
# @private Implementation accessors
|
33
|
+
attr_reader :outstanding_messages, :outstanding_bytes, :awaiting
|
34
|
+
|
35
|
+
def initialize message_limit: 1000, byte_limit: 10_000_000, limit_exceeded_behavior: :ignore
|
36
|
+
unless [:ignore, :error, :block].include? limit_exceeded_behavior
|
37
|
+
raise ArgumentError, "limit_exceeded_behavior must be one of :ignore, :error, :block"
|
38
|
+
end
|
39
|
+
if [:error, :block].include?(limit_exceeded_behavior) && message_limit < 1
|
40
|
+
raise ArgumentError,
|
41
|
+
"Flow control message limit (#{message_limit}) exceeded by a single message, would block forever"
|
42
|
+
end
|
43
|
+
@mutex = Mutex.new
|
44
|
+
@message_limit = message_limit
|
45
|
+
@byte_limit = byte_limit
|
46
|
+
@limit_exceeded_behavior = limit_exceeded_behavior
|
47
|
+
@outstanding_messages = 0
|
48
|
+
@outstanding_bytes = 0
|
49
|
+
|
50
|
+
@awaiting = []
|
51
|
+
end
|
52
|
+
|
53
|
+
def acquire message_size
|
54
|
+
return if limit_exceeded_behavior == :ignore
|
55
|
+
@mutex.lock
|
56
|
+
if limit_exceeded_behavior == :error && would_exceed_message_limit?
|
57
|
+
raise FlowControlLimitError, "Flow control message limit (#{message_limit}) would be exceeded"
|
58
|
+
end
|
59
|
+
if limit_exceeded_behavior == :error && would_exceed_byte_limit?(message_size)
|
60
|
+
raise FlowControlLimitError,
|
61
|
+
"Flow control byte limit (#{byte_limit}) would be exceeded, message_size: #{message_size}"
|
62
|
+
end
|
63
|
+
if limit_exceeded_behavior == :block && message_size > byte_limit
|
64
|
+
raise FlowControlLimitError,
|
65
|
+
"Flow control byte limit (#{byte_limit}) exceeded by a single message, would block forever"
|
66
|
+
end
|
67
|
+
|
68
|
+
acquire_or_wait message_size
|
69
|
+
ensure
|
70
|
+
@mutex.unlock if @mutex.owned?
|
71
|
+
end
|
72
|
+
|
73
|
+
def release message_size
|
74
|
+
return if limit_exceeded_behavior == :ignore
|
75
|
+
@mutex.synchronize do
|
76
|
+
raise "Flow control messages count would be negative" if (@outstanding_messages - 1).negative?
|
77
|
+
raise "Flow control bytes count would be negative" if (@outstanding_bytes - message_size).negative?
|
78
|
+
|
79
|
+
@outstanding_messages -= 1
|
80
|
+
@outstanding_bytes -= message_size
|
81
|
+
@awaiting.first.set unless @awaiting.empty?
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
protected
|
86
|
+
|
87
|
+
# rubocop:disable Style/GuardClause
|
88
|
+
|
89
|
+
def acquire_or_wait message_size
|
90
|
+
waiter = nil
|
91
|
+
while is_new_and_others_wait?(waiter) ||
|
92
|
+
would_exceed_byte_limit?(message_size) ||
|
93
|
+
would_exceed_message_limit?
|
94
|
+
|
95
|
+
if waiter.nil?
|
96
|
+
waiter = Concurrent::Event.new
|
97
|
+
# This waiter gets added to the back of the line.
|
98
|
+
@awaiting << waiter
|
99
|
+
else
|
100
|
+
waiter = Concurrent::Event.new
|
101
|
+
# This waiter already in line stays at the head of the line.
|
102
|
+
@awaiting[0] = waiter
|
103
|
+
end
|
104
|
+
@mutex.unlock
|
105
|
+
waiter.wait
|
106
|
+
@mutex.lock
|
107
|
+
end
|
108
|
+
@outstanding_messages += 1
|
109
|
+
@outstanding_bytes += message_size
|
110
|
+
|
111
|
+
@awaiting.shift if waiter # Remove the newly released waiter from the head of the queue.
|
112
|
+
|
113
|
+
# There may be some surplus left; let the next message waiting try to acquire a permit.
|
114
|
+
if !@awaiting.empty? && @outstanding_bytes < byte_limit && @outstanding_messages < message_limit
|
115
|
+
@awaiting.first.set
|
116
|
+
end
|
117
|
+
end
|
118
|
+
|
119
|
+
# rubocop:enable Style/GuardClause
|
120
|
+
|
121
|
+
def is_new_and_others_wait? waiter
|
122
|
+
waiter.nil? && !@awaiting.empty?
|
123
|
+
end
|
124
|
+
|
125
|
+
def would_exceed_message_limit?
|
126
|
+
@outstanding_messages + 1 > message_limit
|
127
|
+
end
|
128
|
+
|
129
|
+
def would_exceed_byte_limit? bytes_requested
|
130
|
+
@outstanding_bytes + bytes_requested > byte_limit
|
131
|
+
end
|
132
|
+
end
|
133
|
+
end
|
134
|
+
|
135
|
+
Pubsub = PubSub unless const_defined? :Pubsub
|
136
|
+
end
|
137
|
+
end
|
@@ -50,7 +50,7 @@ module Google
|
|
50
50
|
# subscriber.start
|
51
51
|
#
|
52
52
|
# # Shut down the subscriber when ready to stop receiving messages.
|
53
|
-
# subscriber.stop
|
53
|
+
# subscriber.stop!
|
54
54
|
#
|
55
55
|
class Message
|
56
56
|
##
|
@@ -62,7 +62,7 @@ module Google
|
|
62
62
|
# This can be used to publish several messages in bulk.
|
63
63
|
def initialize data = nil, attributes = {}
|
64
64
|
# Convert attributes to strings to match the protobuf definition
|
65
|
-
attributes =
|
65
|
+
attributes = attributes.to_h { |k, v| [String(k), String(v)] }
|
66
66
|
|
67
67
|
@grpc = Google::Cloud::PubSub::V1::PubsubMessage.new(
|
68
68
|
data: String(data).dup.force_encoding(Encoding::ASCII_8BIT),
|
@@ -81,8 +81,7 @@ module Google
|
|
81
81
|
# Optional attributes for the message.
|
82
82
|
def attributes
|
83
83
|
return @grpc.attributes.to_h if @grpc.attributes.respond_to? :to_h
|
84
|
-
|
85
|
-
Hash[@grpc.attributes.to_a]
|
84
|
+
@grpc.attributes.to_a.to_h
|
86
85
|
end
|
87
86
|
|
88
87
|
##
|
@@ -100,6 +99,48 @@ module Google
|
|
100
99
|
end
|
101
100
|
alias publish_time published_at
|
102
101
|
|
102
|
+
##
|
103
|
+
# Identifies related messages for which publish order should be
|
104
|
+
# respected.
|
105
|
+
#
|
106
|
+
# Google Cloud Pub/Sub ordering keys provide the ability to ensure
|
107
|
+
# related messages are sent to subscribers in the order in which they
|
108
|
+
# were published. Messages can be tagged with an ordering key, a string
|
109
|
+
# that identifies related messages for which publish order should be
|
110
|
+
# respected. The service guarantees that, for a given ordering key and
|
111
|
+
# publisher, messages are sent to subscribers in the order in which they
|
112
|
+
# were published. Ordering does not require sacrificing high throughput
|
113
|
+
# or scalability, as the service automatically distributes messages for
|
114
|
+
# different ordering keys across subscribers.
|
115
|
+
#
|
116
|
+
# See {Topic#publish_async} and {Subscription#listen}.
|
117
|
+
#
|
118
|
+
# @return [String]
|
119
|
+
#
|
120
|
+
def ordering_key
|
121
|
+
@grpc.ordering_key
|
122
|
+
end
|
123
|
+
|
124
|
+
# @private
|
125
|
+
def hash
|
126
|
+
@grpc.hash
|
127
|
+
end
|
128
|
+
|
129
|
+
# @private
|
130
|
+
def eql? other
|
131
|
+
return false unless other.is_a? self.class
|
132
|
+
@grpc.hash == other.hash
|
133
|
+
end
|
134
|
+
# @private
|
135
|
+
alias == eql?
|
136
|
+
|
137
|
+
# @private
|
138
|
+
def <=> other
|
139
|
+
return nil unless other.is_a? self.class
|
140
|
+
other_grpc = other.instance_variable_get :@grpc
|
141
|
+
@grpc <=> other_grpc
|
142
|
+
end
|
143
|
+
|
103
144
|
##
|
104
145
|
# @private New Message from a Google::Cloud::PubSub::V1::PubsubMessage
|
105
146
|
# object.
|
@@ -68,7 +68,8 @@ module Google
|
|
68
68
|
# end
|
69
69
|
#
|
70
70
|
class Policy
|
71
|
-
attr_reader :etag
|
71
|
+
attr_reader :etag
|
72
|
+
attr_reader :roles
|
72
73
|
|
73
74
|
##
|
74
75
|
# @private Creates a Policy object.
|
@@ -167,7 +168,7 @@ module Google
|
|
167
168
|
role: role_name,
|
168
169
|
members: roles[role_name]
|
169
170
|
)
|
170
|
-
end
|
171
|
+
end.compact
|
171
172
|
)
|
172
173
|
end
|
173
174
|
|