google-cloud-pubsub 0.26.0 → 2.6.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (56) hide show
  1. checksums.yaml +5 -5
  2. data/.yardopts +12 -2
  3. data/AUTHENTICATION.md +178 -0
  4. data/CHANGELOG.md +659 -0
  5. data/CODE_OF_CONDUCT.md +40 -0
  6. data/CONTRIBUTING.md +187 -0
  7. data/EMULATOR.md +37 -0
  8. data/LICENSE +2 -2
  9. data/LOGGING.md +32 -0
  10. data/OVERVIEW.md +528 -0
  11. data/TROUBLESHOOTING.md +31 -0
  12. data/lib/google/cloud/pubsub/async_publisher/batch.rb +310 -0
  13. data/lib/google/cloud/pubsub/async_publisher.rb +402 -0
  14. data/lib/google/cloud/pubsub/batch_publisher.rb +100 -0
  15. data/lib/google/cloud/pubsub/convert.rb +91 -0
  16. data/lib/google/cloud/pubsub/credentials.rb +26 -10
  17. data/lib/google/cloud/pubsub/errors.rb +85 -0
  18. data/lib/google/cloud/pubsub/message.rb +80 -17
  19. data/lib/google/cloud/pubsub/policy.rb +17 -14
  20. data/lib/google/cloud/pubsub/project.rb +364 -250
  21. data/lib/google/cloud/pubsub/publish_result.rb +103 -0
  22. data/lib/google/cloud/pubsub/received_message.rb +162 -24
  23. data/lib/google/cloud/pubsub/retry_policy.rb +88 -0
  24. data/lib/google/cloud/pubsub/schema/list.rb +180 -0
  25. data/lib/google/cloud/pubsub/schema.rb +310 -0
  26. data/lib/google/cloud/pubsub/service.rb +281 -265
  27. data/lib/google/cloud/pubsub/snapshot/list.rb +21 -21
  28. data/lib/google/cloud/pubsub/snapshot.rb +55 -15
  29. data/lib/google/cloud/pubsub/subscriber/enumerator_queue.rb +54 -0
  30. data/lib/google/cloud/pubsub/subscriber/inventory.rb +173 -0
  31. data/lib/google/cloud/pubsub/subscriber/sequencer.rb +115 -0
  32. data/lib/google/cloud/pubsub/subscriber/stream.rb +400 -0
  33. data/lib/google/cloud/pubsub/subscriber/timed_unary_buffer.rb +230 -0
  34. data/lib/google/cloud/pubsub/subscriber.rb +417 -0
  35. data/lib/google/cloud/pubsub/subscription/list.rb +28 -28
  36. data/lib/google/cloud/pubsub/subscription/push_config.rb +268 -0
  37. data/lib/google/cloud/pubsub/subscription.rb +900 -172
  38. data/lib/google/cloud/pubsub/topic/list.rb +21 -21
  39. data/lib/google/cloud/pubsub/topic.rb +674 -95
  40. data/lib/google/cloud/pubsub/version.rb +6 -4
  41. data/lib/google/cloud/pubsub.rb +104 -439
  42. data/lib/google-cloud-pubsub.rb +60 -29
  43. metadata +88 -50
  44. data/README.md +0 -69
  45. data/lib/google/cloud/pubsub/topic/publisher.rb +0 -86
  46. data/lib/google/cloud/pubsub/v1/doc/google/protobuf/duration.rb +0 -77
  47. data/lib/google/cloud/pubsub/v1/doc/google/protobuf/field_mask.rb +0 -223
  48. data/lib/google/cloud/pubsub/v1/doc/google/protobuf/timestamp.rb +0 -81
  49. data/lib/google/cloud/pubsub/v1/doc/google/pubsub/v1/pubsub.rb +0 -503
  50. data/lib/google/cloud/pubsub/v1/publisher_client.rb +0 -605
  51. data/lib/google/cloud/pubsub/v1/publisher_client_config.json +0 -96
  52. data/lib/google/cloud/pubsub/v1/subscriber_client.rb +0 -1104
  53. data/lib/google/cloud/pubsub/v1/subscriber_client_config.json +0 -127
  54. data/lib/google/cloud/pubsub/v1.rb +0 -17
  55. data/lib/google/pubsub/v1/pubsub_pb.rb +0 -187
  56. data/lib/google/pubsub/v1/pubsub_services_pb.rb +0 -159
@@ -1,10 +1,10 @@
1
- # Copyright 2017 Google Inc. All rights reserved.
1
+ # Copyright 2017 Google LLC
2
2
  #
3
3
  # Licensed under the Apache License, Version 2.0 (the "License");
4
4
  # you may not use this file except in compliance with the License.
5
5
  # You may obtain a copy of the License at
6
6
  #
7
- # http://www.apache.org/licenses/LICENSE-2.0
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
8
  #
9
9
  # Unless required by applicable law or agreed to in writing, software
10
10
  # distributed under the License is distributed on an "AS IS" BASIS,
@@ -17,7 +17,7 @@ require "delegate"
17
17
 
18
18
  module Google
19
19
  module Cloud
20
- module Pubsub
20
+ module PubSub
21
21
  class Snapshot
22
22
  ##
23
23
  # Snapshot::List is a special case Array with additional values.
@@ -25,7 +25,7 @@ module Google
25
25
  ##
26
26
  # If not empty, indicates that there are more snapshots
27
27
  # that match the request and this value should be passed to
28
- # the next {Google::Cloud::Pubsub::Project#snapshots} to continue.
28
+ # the next {Google::Cloud::PubSub::Project#snapshots} to continue.
29
29
  attr_accessor :token
30
30
 
31
31
  ##
@@ -45,7 +45,7 @@ module Google
45
45
  # @example
46
46
  # require "google/cloud/pubsub"
47
47
  #
48
- # pubsub = Google::Cloud::Pubsub.new
48
+ # pubsub = Google::Cloud::PubSub.new
49
49
  #
50
50
  # snapshots = pubsub.snapshots
51
51
  # if snapshots.next?
@@ -64,7 +64,7 @@ module Google
64
64
  # @example
65
65
  # require "google/cloud/pubsub"
66
66
  #
67
- # pubsub = Google::Cloud::Pubsub.new
67
+ # pubsub = Google::Cloud::PubSub.new
68
68
  #
69
69
  # snapshots = pubsub.snapshots
70
70
  # if snapshots.next?
@@ -98,7 +98,7 @@ module Google
98
98
  # @example Iterating each snapshot by passing a block:
99
99
  # require "google/cloud/pubsub"
100
100
  #
101
- # pubsub = Google::Cloud::Pubsub.new
101
+ # pubsub = Google::Cloud::PubSub.new
102
102
  #
103
103
  # snapshots = pubsub.snapshots
104
104
  # snapshots.all do |snapshot|
@@ -108,7 +108,7 @@ module Google
108
108
  # @example Using the enumerator by not passing a block:
109
109
  # require "google/cloud/pubsub"
110
110
  #
111
- # pubsub = Google::Cloud::Pubsub.new
111
+ # pubsub = Google::Cloud::PubSub.new
112
112
  #
113
113
  # snapshots = pubsub.snapshots
114
114
  # all_names = snapshots.all.map do |snapshot|
@@ -118,24 +118,22 @@ module Google
118
118
  # @example Limit the number of API calls made:
119
119
  # require "google/cloud/pubsub"
120
120
  #
121
- # pubsub = Google::Cloud::Pubsub.new
121
+ # pubsub = Google::Cloud::PubSub.new
122
122
  #
123
123
  # snapshots = pubsub.snapshots
124
124
  # snapshots.all(request_limit: 10) do |snapshot|
125
125
  # puts snapshot.name
126
126
  # end
127
127
  #
128
- def all request_limit: nil
128
+ def all request_limit: nil, &block
129
129
  request_limit = request_limit.to_i if request_limit
130
- unless block_given?
131
- return enum_for(:all, request_limit: request_limit)
132
- end
130
+ return enum_for :all, request_limit: request_limit unless block_given?
133
131
  results = self
134
132
  loop do
135
- results.each { |r| yield r }
133
+ results.each(&block)
136
134
  if request_limit
137
135
  request_limit -= 1
138
- break if request_limit < 0
136
+ break if request_limit.negative?
139
137
  end
140
138
  break unless results.next?
141
139
  results = results.next
@@ -144,16 +142,16 @@ module Google
144
142
 
145
143
  ##
146
144
  # @private New Snapshots::List from a
147
- # Google::Pubsub::V1::ListSnapshotsRequest object.
145
+ # Google::Cloud::PubSub::V1::ListSnapshotsRequest object.
148
146
  def self.from_grpc grpc_list, service, max = nil
149
147
  subs = new(Array(grpc_list.snapshots).map do |grpc|
150
148
  Snapshot.from_grpc grpc, service
151
149
  end)
152
150
  token = grpc_list.next_page_token
153
- token = nil if token == ""
154
- subs.instance_variable_set "@token", token
155
- subs.instance_variable_set "@service", service
156
- subs.instance_variable_set "@max", max
151
+ token = nil if token == "".freeze
152
+ subs.instance_variable_set :@token, token
153
+ subs.instance_variable_set :@service, service
154
+ subs.instance_variable_set :@max, max
157
155
  subs
158
156
  end
159
157
 
@@ -163,7 +161,7 @@ module Google
163
161
  # @private Raise an error unless an active connection to the service
164
162
  # is available.
165
163
  def ensure_service!
166
- fail "Must have active connection to service" unless @service
164
+ raise "Must have active connection to service" unless @service
167
165
  end
168
166
 
169
167
  def next_snapshots
@@ -174,5 +172,7 @@ module Google
174
172
  end
175
173
  end
176
174
  end
175
+
176
+ Pubsub = PubSub unless const_defined? :Pubsub
177
177
  end
178
178
  end
@@ -1,10 +1,10 @@
1
- # Copyright 2017 Google Inc. All rights reserved.
1
+ # Copyright 2017 Google LLC
2
2
  #
3
3
  # Licensed under the Apache License, Version 2.0 (the "License");
4
4
  # you may not use this file except in compliance with the License.
5
5
  # You may obtain a copy of the License at
6
6
  #
7
- # http://www.apache.org/licenses/LICENSE-2.0
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
8
  #
9
9
  # Unless required by applicable law or agreed to in writing, software
10
10
  # distributed under the License is distributed on an "AS IS" BASIS,
@@ -15,10 +15,11 @@
15
15
 
16
16
  require "google/cloud/errors"
17
17
  require "google/cloud/pubsub/snapshot/list"
18
+ require "google/cloud/pubsub/v1"
18
19
 
19
20
  module Google
20
21
  module Cloud
21
- module Pubsub
22
+ module PubSub
22
23
  ##
23
24
  # # Snapshot
24
25
  #
@@ -35,7 +36,7 @@ module Google
35
36
  # @example
36
37
  # require "google/cloud/pubsub"
37
38
  #
38
- # pubsub = Google::Cloud::Pubsub.new
39
+ # pubsub = Google::Cloud::PubSub.new
39
40
  # sub = pubsub.subscription "my-sub"
40
41
  #
41
42
  # snapshot = sub.create_snapshot "my-snapshot"
@@ -47,19 +48,21 @@ module Google
47
48
  attr_accessor :service
48
49
 
49
50
  ##
50
- # @private The gRPC Google::Pubsub::V1::Snapshot object.
51
+ # @private The gRPC Google::Cloud::PubSub::V1::Snapshot object.
51
52
  attr_accessor :grpc
52
53
 
53
54
  ##
54
55
  # @private Create an empty {Snapshot} object.
55
56
  def initialize
56
57
  @service = nil
57
- @grpc = Google::Pubsub::V1::Snapshot.new
58
+ @grpc = Google::Cloud::PubSub::V1::Snapshot.new
58
59
  end
59
60
 
60
61
  ##
61
- # The name of the snapshot. Format is
62
- # `projects/{project}/snapshots/{snap}`.
62
+ # The name of the snapshot.
63
+ #
64
+ # @return [String] A fully-qualified snapshot name in the form
65
+ # `projects/{project_id}/snapshots/{snapshot_id}`.
63
66
  def name
64
67
  @grpc.name
65
68
  end
@@ -72,14 +75,14 @@ module Google
72
75
  # @example
73
76
  # require "google/cloud/pubsub"
74
77
  #
75
- # pubsub = Google::Cloud::Pubsub.new
78
+ # pubsub = Google::Cloud::PubSub.new
76
79
  # sub = pubsub.subscription "my-sub"
77
80
  #
78
81
  # snapshot = sub.create_snapshot "my-snapshot"
79
82
  # snapshot.topic.name #=> "projects/my-project/topics/my-topic"
80
83
  #
81
84
  def topic
82
- Topic.new_lazy @grpc.topic, service
85
+ Topic.from_name @grpc.topic, service
83
86
  end
84
87
 
85
88
  ##
@@ -99,7 +102,7 @@ module Google
99
102
  # @example
100
103
  # require "google/cloud/pubsub"
101
104
  #
102
- # pubsub = Google::Cloud::Pubsub.new
105
+ # pubsub = Google::Cloud::PubSub.new
103
106
  # sub = pubsub.subscription "my-sub"
104
107
  #
105
108
  # snapshot = sub.create_snapshot "my-snapshot"
@@ -107,7 +110,42 @@ module Google
107
110
  # snapshot.expiration_time
108
111
  #
109
112
  def expiration_time
110
- self.class.timestamp_from_grpc @grpc.expiration_time
113
+ self.class.timestamp_from_grpc @grpc.expire_time
114
+ end
115
+
116
+ ##
117
+ # A hash of user-provided labels associated with this snapshot.
118
+ # Labels can be used to organize and group snapshots.See [Creating and
119
+ # Managing Labels](https://cloud.google.com/pubsub/docs/labels).
120
+ #
121
+ # The returned hash is frozen and changes are not allowed. Use
122
+ # {#labels=} to update the labels for this snapshot.
123
+ #
124
+ # @return [Hash] The frozen labels hash.
125
+ #
126
+ def labels
127
+ @grpc.labels.to_h.freeze
128
+ end
129
+
130
+ ##
131
+ # Sets the hash of user-provided labels associated with this
132
+ # snapshot. Labels can be used to organize and group snapshots.
133
+ # Label keys and values can be no longer than 63 characters, can only
134
+ # contain lowercase letters, numeric characters, underscores and dashes.
135
+ # International characters are allowed. Label values are optional. Label
136
+ # keys must start with a letter and each label in the list must have a
137
+ # different key. See [Creating and Managing
138
+ # Labels](https://cloud.google.com/pubsub/docs/labels).
139
+ #
140
+ # @param [Hash] new_labels The new labels hash.
141
+ #
142
+ def labels= new_labels
143
+ raise ArgumentError, "Value must be a Hash" if new_labels.nil?
144
+ labels_map = Google::Protobuf::Map.new :string, :string
145
+ Hash(new_labels).each { |k, v| labels_map[String(k)] = String(v) }
146
+ update_grpc = @grpc.dup
147
+ update_grpc.labels = labels_map
148
+ @grpc = service.update_snapshot update_grpc, :labels
111
149
  end
112
150
 
113
151
  ##
@@ -122,7 +160,7 @@ module Google
122
160
  # @example
123
161
  # require "google/cloud/pubsub"
124
162
  #
125
- # pubsub = Google::Cloud::Pubsub.new
163
+ # pubsub = Google::Cloud::PubSub.new
126
164
  #
127
165
  # pubsub.snapshots.each do |snapshot|
128
166
  # snapshot.delete
@@ -135,7 +173,7 @@ module Google
135
173
  end
136
174
 
137
175
  ##
138
- # @private New Snapshot from a Google::Pubsub::V1::Snapshot
176
+ # @private New Snapshot from a Google::Cloud::PubSub::V1::Snapshot
139
177
  # object.
140
178
  def self.from_grpc grpc, service
141
179
  new.tap do |f|
@@ -157,9 +195,11 @@ module Google
157
195
  # @private Raise an error unless an active connection to the service is
158
196
  # available.
159
197
  def ensure_service!
160
- fail "Must have active connection to service" unless service
198
+ raise "Must have active connection to service" unless service
161
199
  end
162
200
  end
163
201
  end
202
+
203
+ Pubsub = PubSub unless const_defined? :Pubsub
164
204
  end
165
205
  end
@@ -0,0 +1,54 @@
1
+ # Copyright 2017 Google LLC
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+
16
+ module Google
17
+ module Cloud
18
+ module PubSub
19
+ class Subscriber
20
+ # @private
21
+ class EnumeratorQueue
22
+ def initialize sentinel = nil
23
+ @queue = Queue.new
24
+ @sentinel = sentinel
25
+ end
26
+
27
+ def push obj
28
+ @queue.push obj
29
+ end
30
+
31
+ def quit_and_dump_queue
32
+ objs = []
33
+ objs << @queue.pop until @queue.empty?
34
+ # Signal that the enumerator is ready to end
35
+ @queue.push @sentinel
36
+ objs
37
+ end
38
+
39
+ def each
40
+ return enum_for :each unless block_given?
41
+
42
+ loop do
43
+ obj = @queue.pop
44
+ break if obj.equal? @sentinel
45
+ yield obj
46
+ end
47
+ end
48
+ end
49
+ end
50
+ end
51
+
52
+ Pubsub = PubSub unless const_defined? :Pubsub
53
+ end
54
+ end
@@ -0,0 +1,173 @@
1
+ # Copyright 2017 Google LLC
2
+ #
3
+ # Licensed under the Apache License, Version 2.0 (the "License");
4
+ # you may not use this file except in compliance with the License.
5
+ # You may obtain a copy of the License at
6
+ #
7
+ # https://www.apache.org/licenses/LICENSE-2.0
8
+ #
9
+ # Unless required by applicable law or agreed to in writing, software
10
+ # distributed under the License is distributed on an "AS IS" BASIS,
11
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+ # See the License for the specific language governing permissions and
13
+ # limitations under the License.
14
+
15
+
16
+ require "monitor"
17
+
18
+ module Google
19
+ module Cloud
20
+ module PubSub
21
+ class Subscriber
22
+ ##
23
+ # @private
24
+ class Inventory
25
+ InventoryItem = Struct.new :bytesize, :pulled_at do
26
+ def self.from rec_msg
27
+ new rec_msg.to_proto.bytesize, Time.now
28
+ end
29
+ end
30
+
31
+ include MonitorMixin
32
+
33
+ attr_reader :stream
34
+ attr_reader :limit
35
+ attr_reader :bytesize
36
+ attr_reader :extension
37
+ attr_reader :max_duration_per_lease_extension
38
+ attr_reader :use_legacy_flow_control
39
+
40
+ def initialize stream, limit:, bytesize:, extension:, max_duration_per_lease_extension:,
41
+ use_legacy_flow_control:
42
+ super()
43
+ @stream = stream
44
+ @limit = limit
45
+ @bytesize = bytesize
46
+ @extension = extension
47
+ @max_duration_per_lease_extension = max_duration_per_lease_extension
48
+ @use_legacy_flow_control = use_legacy_flow_control
49
+ @inventory = {}
50
+ @wait_cond = new_cond
51
+ end
52
+
53
+ def ack_ids
54
+ @inventory.keys
55
+ end
56
+
57
+ def add *rec_msgs
58
+ rec_msgs.flatten!
59
+ rec_msgs.compact!
60
+ return if rec_msgs.empty?
61
+
62
+ synchronize do
63
+ rec_msgs.each do |rec_msg|
64
+ @inventory[rec_msg.ack_id] = InventoryItem.from rec_msg
65
+ end
66
+ @wait_cond.broadcast
67
+ end
68
+ end
69
+
70
+ def remove *ack_ids
71
+ ack_ids.flatten!
72
+ ack_ids.compact!
73
+ return if ack_ids.empty?
74
+
75
+ synchronize do
76
+ @inventory.delete_if { |ack_id, _| ack_ids.include? ack_id }
77
+ @wait_cond.broadcast
78
+ end
79
+ end
80
+
81
+ def remove_expired!
82
+ synchronize do
83
+ extension_time = Time.new - extension
84
+ @inventory.delete_if { |_ack_id, item| item.pulled_at < extension_time }
85
+ @wait_cond.broadcast
86
+ end
87
+ end
88
+
89
+ def count
90
+ synchronize do
91
+ @inventory.count
92
+ end
93
+ end
94
+
95
+ def total_bytesize
96
+ synchronize do
97
+ @inventory.values.sum(&:bytesize)
98
+ end
99
+ end
100
+
101
+ def empty?
102
+ synchronize do
103
+ @inventory.empty?
104
+ end
105
+ end
106
+
107
+ def start
108
+ @background_thread ||= Thread.new { background_run }
109
+
110
+ self
111
+ end
112
+
113
+ def stop
114
+ synchronize do
115
+ @stopped = true
116
+ @wait_cond.broadcast
117
+ end
118
+
119
+ self
120
+ end
121
+
122
+ def stopped?
123
+ synchronize { @stopped }
124
+ end
125
+
126
+ def full?
127
+ synchronize do
128
+ @inventory.count >= limit || @inventory.values.sum(&:bytesize) >= bytesize
129
+ end
130
+ end
131
+
132
+ protected
133
+
134
+ def background_run
135
+ delay_target = nil
136
+
137
+ until stopped?
138
+ if empty?
139
+ delay_target = nil
140
+
141
+ synchronize { @wait_cond.wait } # wait until broadcast
142
+ next
143
+ end
144
+
145
+ delay_target ||= calc_target
146
+ delay_gap = delay_target - Time.now
147
+
148
+ unless delay_gap.positive?
149
+ delay_target = calc_target
150
+ stream.renew_lease!
151
+ next
152
+ end
153
+
154
+ synchronize { @wait_cond.wait delay_gap }
155
+ end
156
+ end
157
+
158
+ def calc_target
159
+ Time.now + calc_delay
160
+ end
161
+
162
+ def calc_delay
163
+ delay = (stream.subscriber.deadline - 3) * rand(0.8..0.9)
164
+ delay = [delay, max_duration_per_lease_extension].min if max_duration_per_lease_extension.positive?
165
+ delay
166
+ end
167
+ end
168
+ end
169
+ end
170
+
171
+ Pubsub = PubSub unless const_defined? :Pubsub
172
+ end
173
+ end