google-cloud-pubsub 0.26.0 → 0.27.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/google/cloud/pubsub.rb +67 -31
- data/lib/google/cloud/pubsub/async_publisher.rb +328 -0
- data/lib/google/cloud/pubsub/batch_publisher.rb +98 -0
- data/lib/google/cloud/pubsub/convert.rb +63 -0
- data/lib/google/cloud/pubsub/message.rb +13 -4
- data/lib/google/cloud/pubsub/project.rb +64 -184
- data/lib/google/cloud/pubsub/publish_result.rb +96 -0
- data/lib/google/cloud/pubsub/received_message.rb +36 -2
- data/lib/google/cloud/pubsub/service.rb +25 -28
- data/lib/google/cloud/pubsub/subscriber.rb +185 -0
- data/lib/google/cloud/pubsub/subscriber/async_pusher.rb +220 -0
- data/lib/google/cloud/pubsub/subscriber/enumerator_queue.rb +52 -0
- data/lib/google/cloud/pubsub/subscriber/stream.rb +376 -0
- data/lib/google/cloud/pubsub/subscription.rb +102 -72
- data/lib/google/cloud/pubsub/topic.rb +125 -32
- data/lib/google/cloud/pubsub/v1/publisher_client_config.json +1 -1
- data/lib/google/cloud/pubsub/v1/subscriber_client_config.json +10 -1
- data/lib/google/cloud/pubsub/version.rb +1 -1
- metadata +38 -3
- data/lib/google/cloud/pubsub/topic/publisher.rb +0 -86
@@ -0,0 +1,96 @@
|
|
1
|
+
# Copyright 2017 Google Inc. All rights reserved.
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
|
16
|
+
module Google
|
17
|
+
module Cloud
|
18
|
+
module Pubsub
|
19
|
+
##
|
20
|
+
# # PublishResult
|
21
|
+
#
|
22
|
+
class PublishResult
|
23
|
+
##
|
24
|
+
# @private Create an PublishResult object.
|
25
|
+
def initialize message, error = nil
|
26
|
+
@message = message
|
27
|
+
@error = error
|
28
|
+
end
|
29
|
+
|
30
|
+
##
|
31
|
+
# The message.
|
32
|
+
def message
|
33
|
+
@message
|
34
|
+
end
|
35
|
+
alias_method :msg, :message
|
36
|
+
|
37
|
+
##
|
38
|
+
# The message's data.
|
39
|
+
def data
|
40
|
+
message.data
|
41
|
+
end
|
42
|
+
|
43
|
+
##
|
44
|
+
# The message's attributes.
|
45
|
+
def attributes
|
46
|
+
message.attributes
|
47
|
+
end
|
48
|
+
|
49
|
+
##
|
50
|
+
# The ID of the message, assigned by the server at publication
|
51
|
+
# time. Guaranteed to be unique within the topic.
|
52
|
+
def message_id
|
53
|
+
message.message_id
|
54
|
+
end
|
55
|
+
alias_method :msg_id, :message_id
|
56
|
+
|
57
|
+
##
|
58
|
+
# The time at which the message was published.
|
59
|
+
def published_at
|
60
|
+
message.published_at
|
61
|
+
end
|
62
|
+
alias_method :publish_time, :published_at
|
63
|
+
|
64
|
+
##
|
65
|
+
# The error that was raised when published, if any.
|
66
|
+
def error
|
67
|
+
@error
|
68
|
+
end
|
69
|
+
|
70
|
+
##
|
71
|
+
# Whether the publish request was successful.
|
72
|
+
def succeeded?
|
73
|
+
error.nil?
|
74
|
+
end
|
75
|
+
|
76
|
+
# Whether the publish request failed.
|
77
|
+
def failed?
|
78
|
+
!succeeded?
|
79
|
+
end
|
80
|
+
|
81
|
+
##
|
82
|
+
# @private Create an PublishResult object from a message protobuf.
|
83
|
+
def self.from_grpc msg_grpc
|
84
|
+
new Message.from_grpc(msg_grpc)
|
85
|
+
end
|
86
|
+
|
87
|
+
##
|
88
|
+
# @private Create an PublishResult object from a message protobuf and an
|
89
|
+
# error.
|
90
|
+
def self.from_error msg_grpc, error
|
91
|
+
new Message.from_grpc(msg_grpc), error
|
92
|
+
end
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|
@@ -66,13 +66,14 @@ module Google
|
|
66
66
|
alias_method :msg, :message
|
67
67
|
|
68
68
|
##
|
69
|
-
# The received message
|
69
|
+
# The received message payload. This data is a list of bytes encoded as
|
70
|
+
# ASCII-8BIT.
|
70
71
|
def data
|
71
72
|
message.data
|
72
73
|
end
|
73
74
|
|
74
75
|
##
|
75
|
-
#
|
76
|
+
# Optional attributes for the received message.
|
76
77
|
def attributes
|
77
78
|
message.attributes
|
78
79
|
end
|
@@ -85,6 +86,13 @@ module Google
|
|
85
86
|
end
|
86
87
|
alias_method :msg_id, :message_id
|
87
88
|
|
89
|
+
##
|
90
|
+
# The time at which the message was published.
|
91
|
+
def published_at
|
92
|
+
message.published_at
|
93
|
+
end
|
94
|
+
alias_method :publish_time, :published_at
|
95
|
+
|
88
96
|
##
|
89
97
|
# Acknowledges receipt of the message.
|
90
98
|
#
|
@@ -135,6 +143,32 @@ module Google
|
|
135
143
|
ensure_subscription!
|
136
144
|
subscription.delay new_deadline, ack_id
|
137
145
|
end
|
146
|
+
alias_method :modify_ack_deadline!, :delay!
|
147
|
+
|
148
|
+
##
|
149
|
+
# Resets the acknowledge deadline for the message without acknowledging
|
150
|
+
# it.
|
151
|
+
#
|
152
|
+
# This will make the message available for redelivery.
|
153
|
+
#
|
154
|
+
# @example
|
155
|
+
# require "google/cloud/pubsub"
|
156
|
+
#
|
157
|
+
# pubsub = Google::Cloud::Pubsub.new
|
158
|
+
#
|
159
|
+
# sub = pubsub.subscription "my-topic-sub"
|
160
|
+
# received_message = sub.pull.first
|
161
|
+
# if received_message
|
162
|
+
# puts received_message.message.data
|
163
|
+
# # Release message back to the API.
|
164
|
+
# received_message.reject!
|
165
|
+
# end
|
166
|
+
#
|
167
|
+
def reject!
|
168
|
+
delay! 0
|
169
|
+
end
|
170
|
+
alias_method :nack!, :reject!
|
171
|
+
alias_method :ignore!, :reject!
|
138
172
|
|
139
173
|
##
|
140
174
|
# @private New ReceivedMessage from a
|
@@ -15,6 +15,7 @@
|
|
15
15
|
|
16
16
|
require "google/cloud/errors"
|
17
17
|
require "google/cloud/pubsub/credentials"
|
18
|
+
require "google/cloud/pubsub/convert"
|
18
19
|
require "google/cloud/pubsub/version"
|
19
20
|
require "google/cloud/pubsub/v1"
|
20
21
|
require "google/gax/errors"
|
@@ -145,11 +146,6 @@ module Google
|
|
145
146
|
# The messages parameter is an array of arrays.
|
146
147
|
# The first element is the data, second is attributes hash.
|
147
148
|
def publish topic, messages
|
148
|
-
messages = messages.map do |data, attributes|
|
149
|
-
Google::Pubsub::V1::PubsubMessage.new(
|
150
|
-
data: data, attributes: attributes)
|
151
|
-
end
|
152
|
-
|
153
149
|
execute do
|
154
150
|
publisher.publish topic_path(topic), messages,
|
155
151
|
options: default_options
|
@@ -214,7 +210,7 @@ module Google
|
|
214
210
|
end
|
215
211
|
deadline = options[:deadline]
|
216
212
|
retain_acked = options[:retain_acked]
|
217
|
-
mrd = number_to_duration options[:retention]
|
213
|
+
mrd = Convert.number_to_duration options[:retention]
|
218
214
|
|
219
215
|
execute do
|
220
216
|
subscriber.create_subscription name,
|
@@ -227,6 +223,14 @@ module Google
|
|
227
223
|
end
|
228
224
|
end
|
229
225
|
|
226
|
+
def update_subscription subscription_obj, *fields
|
227
|
+
mask = Google::Protobuf::FieldMask.new paths: fields.map(&:to_s)
|
228
|
+
execute do
|
229
|
+
subscriber.update_subscription \
|
230
|
+
subscription_obj, mask, options: default_options
|
231
|
+
end
|
232
|
+
end
|
233
|
+
|
230
234
|
##
|
231
235
|
# Deletes an existing subscription.
|
232
236
|
# All pending messages in the subscription are immediately dropped.
|
@@ -252,6 +256,12 @@ module Google
|
|
252
256
|
end
|
253
257
|
end
|
254
258
|
|
259
|
+
def streaming_pull request_enum
|
260
|
+
execute do
|
261
|
+
subscriber.streaming_pull request_enum, options: default_options
|
262
|
+
end
|
263
|
+
end
|
264
|
+
|
255
265
|
##
|
256
266
|
# Acknowledges receipt of a message.
|
257
267
|
def acknowledge subscription, *ack_ids
|
@@ -332,7 +342,8 @@ module Google
|
|
332
342
|
def seek subscription, time_or_snapshot
|
333
343
|
subscription = subscription_path(subscription)
|
334
344
|
execute do
|
335
|
-
if
|
345
|
+
if a_time? time_or_snapshot
|
346
|
+
time = Convert.time_to_timestamp time_or_snapshot
|
336
347
|
subscriber.seek subscription, time: time, options: default_options
|
337
348
|
else
|
338
349
|
if time_or_snapshot.is_a? Snapshot
|
@@ -425,6 +436,13 @@ module Google
|
|
425
436
|
|
426
437
|
protected
|
427
438
|
|
439
|
+
def a_time? obj
|
440
|
+
return false unless obj.respond_to? :to_time
|
441
|
+
# Rails' String#to_time returns nil if the string doesn't parse.
|
442
|
+
return false if obj.to_time.nil?
|
443
|
+
true
|
444
|
+
end
|
445
|
+
|
428
446
|
def default_headers
|
429
447
|
{ "google-cloud-resource-prefix" => "projects/#{@project}" }
|
430
448
|
end
|
@@ -433,27 +451,6 @@ module Google
|
|
433
451
|
Google::Gax::CallOptions.new kwargs: default_headers
|
434
452
|
end
|
435
453
|
|
436
|
-
##
|
437
|
-
# @private Get a Google::Protobuf::Timestamp object from a Time object.
|
438
|
-
def time_to_timestamp time
|
439
|
-
return nil if time.nil?
|
440
|
-
# Make sure to_time is supported.
|
441
|
-
return nil unless time.respond_to? :to_time
|
442
|
-
time = time.to_time
|
443
|
-
# Make sure we have a Time object.
|
444
|
-
# Rails' String#to_time returns nil if the string doesn't parse.
|
445
|
-
return nil unless time
|
446
|
-
Google::Protobuf::Timestamp.new seconds: time.to_i, nanos: time.nsec
|
447
|
-
end
|
448
|
-
|
449
|
-
def number_to_duration number
|
450
|
-
return nil if number.nil?
|
451
|
-
|
452
|
-
Google::Protobuf::Duration.new \
|
453
|
-
seconds: number.to_i,
|
454
|
-
nanos: (number.remainder(1) * 1000000000).round
|
455
|
-
end
|
456
|
-
|
457
454
|
def execute
|
458
455
|
yield
|
459
456
|
rescue Google::Gax::GaxError => e
|
@@ -0,0 +1,185 @@
|
|
1
|
+
# Copyright 2017 Google Inc. All rights reserved.
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
|
16
|
+
require "google/cloud/pubsub/service"
|
17
|
+
require "google/cloud/pubsub/subscriber/stream"
|
18
|
+
require "monitor"
|
19
|
+
|
20
|
+
module Google
|
21
|
+
module Cloud
|
22
|
+
module Pubsub
|
23
|
+
##
|
24
|
+
# Subscriber object used to stream and process messages from a
|
25
|
+
# Subscription. See {Google::Cloud::Pubsub::Subscription#listen}
|
26
|
+
#
|
27
|
+
# @example
|
28
|
+
# require "google/cloud/pubsub"
|
29
|
+
#
|
30
|
+
# pubsub = Google::Cloud::Pubsub.new
|
31
|
+
#
|
32
|
+
# sub = pubsub.subscription "my-topic-sub"
|
33
|
+
#
|
34
|
+
# subscriber = sub.listen do |msg|
|
35
|
+
# # process msg
|
36
|
+
# msg.ack!
|
37
|
+
# end
|
38
|
+
#
|
39
|
+
# subscriber.start
|
40
|
+
#
|
41
|
+
# # Shut down the subscriber when ready to stop receiving messages.
|
42
|
+
# subscriber.stop.wait!
|
43
|
+
#
|
44
|
+
# @attr_reader [String] subscription_name The name of the subscription the
|
45
|
+
# messages are pulled from.
|
46
|
+
# @attr_reader [Proc] callback The procedure that will handle the messages
|
47
|
+
# received from the subscription.
|
48
|
+
# @attr_reader [Numeric] deadline The default number of seconds the stream
|
49
|
+
# will hold received messages before modifying the message's ack
|
50
|
+
# deadline. The minimum is 10, the maximum is 600. Default is 60.
|
51
|
+
# @attr_reader [Integer] streams The number of concurrent streams to open
|
52
|
+
# to pull messages from the subscription. Default is 4.
|
53
|
+
# @attr_reader [Integer] inventory The number of received messages to be
|
54
|
+
# collected by subscriber. Default is 1,000.
|
55
|
+
# @attr_reader [Integer] callback_threads The number of threads used to
|
56
|
+
# handle the received messages. Default is 8.
|
57
|
+
# @attr_reader [Integer] push_threads The number of threads to handle
|
58
|
+
# acknowledgement ({ReceivedMessage#ack!}) and delay messages
|
59
|
+
# ({ReceivedMessage#nack!}, {ReceivedMessage#delay!}). Default is 4.
|
60
|
+
#
|
61
|
+
class Subscriber
|
62
|
+
include MonitorMixin
|
63
|
+
|
64
|
+
attr_reader :subscription_name, :callback, :deadline, :streams,
|
65
|
+
:inventory, :callback_threads, :push_threads
|
66
|
+
|
67
|
+
##
|
68
|
+
# @private Implementation attributes.
|
69
|
+
attr_reader :stream_inventory, :stream_pool, :thread_pool, :service
|
70
|
+
|
71
|
+
##
|
72
|
+
# @private Create an empty {Subscriber} object.
|
73
|
+
def initialize subscription_name, callback, deadline: nil, streams: nil,
|
74
|
+
inventory: nil, threads: {}, service: nil
|
75
|
+
@callback = callback
|
76
|
+
@subscription_name = subscription_name
|
77
|
+
@deadline = deadline || 60
|
78
|
+
@streams = streams || 4
|
79
|
+
@inventory = inventory || 1000
|
80
|
+
@callback_threads = (threads[:callback] || 8).to_i
|
81
|
+
@push_threads = (threads[:push] || 4).to_i
|
82
|
+
|
83
|
+
@stream_inventory = @inventory.fdiv(@streams).ceil
|
84
|
+
@service = service
|
85
|
+
|
86
|
+
@started = nil
|
87
|
+
@stopped = nil
|
88
|
+
|
89
|
+
stream_pool = @streams.times.map do
|
90
|
+
Thread.new { Stream.new self }
|
91
|
+
end
|
92
|
+
@stream_pool = stream_pool.map(&:value)
|
93
|
+
|
94
|
+
super() # to init MonitorMixin
|
95
|
+
end
|
96
|
+
|
97
|
+
##
|
98
|
+
# Starts the subscriber pulling from the subscription and processing the
|
99
|
+
# received messages.
|
100
|
+
#
|
101
|
+
# @return [Subscriber] returns self so calls can be chained.
|
102
|
+
def start
|
103
|
+
start_pool = synchronize do
|
104
|
+
@started = true
|
105
|
+
@stopped = false
|
106
|
+
|
107
|
+
@stream_pool.map do |stream|
|
108
|
+
Thread.new { stream.start }
|
109
|
+
end
|
110
|
+
end
|
111
|
+
start_pool.join
|
112
|
+
|
113
|
+
self
|
114
|
+
end
|
115
|
+
|
116
|
+
##
|
117
|
+
# Begins the process of stopping the subscriber. Unhandled received
|
118
|
+
# messages will be processed, but no new messages will be pulled from
|
119
|
+
# the subscription. Use {#wait!} to block until the subscriber is fully
|
120
|
+
# stopped and all received messages have been processed.
|
121
|
+
#
|
122
|
+
# @return [Subscriber] returns self so calls can be chained.
|
123
|
+
def stop
|
124
|
+
stop_pool = synchronize do
|
125
|
+
@started = false
|
126
|
+
@stopped = true
|
127
|
+
|
128
|
+
@stream_pool.map do |stream|
|
129
|
+
Thread.new { stream.stop }
|
130
|
+
end
|
131
|
+
end
|
132
|
+
stop_pool.join
|
133
|
+
|
134
|
+
self
|
135
|
+
end
|
136
|
+
|
137
|
+
##
|
138
|
+
# Blocks until the subscriber is fully stopped and all received messages
|
139
|
+
# have been handled. Does not stop the subscriber. To stop the
|
140
|
+
# subscriber, first call {#stop} and then call {#wait!} to block until
|
141
|
+
# the subscriber is stopped.
|
142
|
+
#
|
143
|
+
# @return [Subscriber] returns self so calls can be chained.
|
144
|
+
def wait!
|
145
|
+
wait_pool = synchronize do
|
146
|
+
@stream_pool.map do |stream|
|
147
|
+
Thread.new { stream.wait! }
|
148
|
+
end
|
149
|
+
end
|
150
|
+
wait_pool.join
|
151
|
+
|
152
|
+
self
|
153
|
+
end
|
154
|
+
|
155
|
+
##
|
156
|
+
# Whether the subscriber has been started.
|
157
|
+
#
|
158
|
+
# @return [boolean] `true` when started, `false` otherwise.
|
159
|
+
def started?
|
160
|
+
synchronize { @started }
|
161
|
+
end
|
162
|
+
|
163
|
+
##
|
164
|
+
# Whether the subscriber has been stopped.
|
165
|
+
#
|
166
|
+
# @return [boolean] `true` when stopped, `false` otherwise.
|
167
|
+
def stopped?
|
168
|
+
synchronize { @stopped }
|
169
|
+
end
|
170
|
+
|
171
|
+
##
|
172
|
+
# @private
|
173
|
+
def to_s
|
174
|
+
format "(subscription: %s, streams: %i)", subscription_name, streams
|
175
|
+
end
|
176
|
+
|
177
|
+
##
|
178
|
+
# @private
|
179
|
+
def inspect
|
180
|
+
"#<#{self.class.name} #{self}>"
|
181
|
+
end
|
182
|
+
end
|
183
|
+
end
|
184
|
+
end
|
185
|
+
end
|