google-gax 0.4.0 → 0.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/google/gax/api_callable.rb +7 -5
- data/lib/google/gax/bundling.rb +445 -0
- data/lib/google/gax/settings.rb +1 -1
- data/lib/google/gax/version.rb +1 -1
- data/spec/google/gax/api_callable_spec.rb +76 -29
- data/spec/google/gax/bundling_spec.rb +627 -0
- data/spec/google/gax/settings_spec.rb +2 -3
- metadata +7 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 56e4ca24c4b15383126e5401cffc794a47f0192a
|
4
|
+
data.tar.gz: f7849c578e8d97aee04dc216698437e971c7d831
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 10eb2d5554f5b82e9b4c890228a11402c3e2a43bc35108f3e56d6f67658207db5ae7c40468731c6fcb05f30b4ade556d7e04d15b7dd8783490f23ced39d629fe
|
7
|
+
data.tar.gz: e2c762e9ffc3bfd562f9d0a7e6595f81154399cb75788a013d74378371d1bfa45d1950b8b570e10665bb51a70c9cba0b7524a21236d0c4e9c960b94542bbbd9d
|
@@ -30,6 +30,7 @@
|
|
30
30
|
require 'time'
|
31
31
|
|
32
32
|
require 'google/gax/errors'
|
33
|
+
require 'google/gax/bundling'
|
33
34
|
|
34
35
|
# rubocop:disable Metrics/ModuleLength
|
35
36
|
|
@@ -285,9 +286,11 @@ module Google
|
|
285
286
|
def bundleable(desc)
|
286
287
|
proc do |api_call, request, settings|
|
287
288
|
return api_call(request) unless settings.bundler
|
288
|
-
the_id =
|
289
|
-
|
290
|
-
|
289
|
+
the_id = Google::Gax.compute_bundle_id(
|
290
|
+
request,
|
291
|
+
desc.request_discriminator_fields
|
292
|
+
)
|
293
|
+
settings.bundler.schedule(api_call, the_id, desc, request)
|
291
294
|
end
|
292
295
|
end
|
293
296
|
|
@@ -373,8 +376,7 @@ module Google
|
|
373
376
|
# @return [Proc] the original proc updated to the timeout arg
|
374
377
|
def add_timeout_arg(a_func, timeout, kwargs)
|
375
378
|
proc do |request|
|
376
|
-
|
377
|
-
a_func.call(request, **kwargs)
|
379
|
+
a_func.call(request, deadline: Time.now + timeout, metadata: kwargs)
|
378
380
|
end
|
379
381
|
end
|
380
382
|
|
@@ -0,0 +1,445 @@
|
|
1
|
+
# Copyright 2016, Google Inc.
|
2
|
+
# All rights reserved.
|
3
|
+
#
|
4
|
+
# Redistribution and use in source and binary forms, with or without
|
5
|
+
# modification, are permitted provided that the following conditions are
|
6
|
+
# met:
|
7
|
+
#
|
8
|
+
# * Redistributions of source code must retain the above copyright
|
9
|
+
# notice, this list of conditions and the following disclaimer.
|
10
|
+
# * Redistributions in binary form must reproduce the above
|
11
|
+
# copyright notice, this list of conditions and the following disclaimer
|
12
|
+
# in the documentation and/or other materials provided with the
|
13
|
+
# distribution.
|
14
|
+
# * Neither the name of Google Inc. nor the names of its
|
15
|
+
# contributors may be used to endorse or promote products derived from
|
16
|
+
# this software without specific prior written permission.
|
17
|
+
#
|
18
|
+
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
19
|
+
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
20
|
+
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
21
|
+
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
22
|
+
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
23
|
+
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
24
|
+
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
25
|
+
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
26
|
+
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
27
|
+
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
28
|
+
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
29
|
+
|
30
|
+
module Google
|
31
|
+
# Gax defines Google API extensions
|
32
|
+
module Gax
|
33
|
+
DEMUX_WARNING = [
|
34
|
+
'Warning: cannot demultiplex the bundled response, got ',
|
35
|
+
'%d subresponses; want %d, each bundled request will ',
|
36
|
+
'receive all responses'
|
37
|
+
].join
|
38
|
+
|
39
|
+
# Helper function for #compute_bundle_id.
|
40
|
+
# Used to retrieve a nested field signified by name where dots in name
|
41
|
+
# indicate nested objects.
|
42
|
+
#
|
43
|
+
# @param obj [Object] an object.
|
44
|
+
# @param name [String] a name for a field in the object.
|
45
|
+
# @return [String, nil] value of named attribute. Can be nil.
|
46
|
+
def str_dotted_access(obj, name)
|
47
|
+
name.split('.').each do |part|
|
48
|
+
obj = obj[part]
|
49
|
+
break if obj.nil?
|
50
|
+
end
|
51
|
+
obj.nil? ? nil : obj.to_s
|
52
|
+
end
|
53
|
+
|
54
|
+
# Computes a bundle id from the discriminator fields of `obj`.
|
55
|
+
#
|
56
|
+
# +discriminator_fields+ may include '.' as a separator, which is used to
|
57
|
+
# indicate object traversal. This is meant to allow fields in the
|
58
|
+
# computed bundle_id.
|
59
|
+
# the return is an array computed by going through the discriminator fields
|
60
|
+
# in order and obtaining the str(value) object field (or nested object
|
61
|
+
# field) if any discriminator field cannot be found, ValueError is raised.
|
62
|
+
#
|
63
|
+
# @param obj [Object] an object.
|
64
|
+
# @param discriminator_fields [Array<String>] a list of discriminator
|
65
|
+
# fields in the order to be to be used in the id.
|
66
|
+
# @return [Array<Object>] array of objects computed as described above.
|
67
|
+
def compute_bundle_id(obj, discriminator_fields)
|
68
|
+
result = []
|
69
|
+
discriminator_fields.each do |field|
|
70
|
+
result.push(str_dotted_access(obj, field))
|
71
|
+
end
|
72
|
+
result
|
73
|
+
end
|
74
|
+
|
75
|
+
# rubocop:disable Metrics/ClassLength
|
76
|
+
|
77
|
+
# Coordinates the execution of a single bundle.
|
78
|
+
#
|
79
|
+
# @!attribute [r] bundle_id
|
80
|
+
# @return [String] the id of this bundle.
|
81
|
+
# @!attribute [r] bundled_field
|
82
|
+
# @return [String] the field used to create the bundled request.
|
83
|
+
# @!attribute [r] subresponse_field
|
84
|
+
# @return [String] tptional field used to demultiplex responses.
|
85
|
+
class Task
|
86
|
+
attr_reader :bundle_id, :bundled_field,
|
87
|
+
:subresponse_field
|
88
|
+
|
89
|
+
# @param api_call [Proc] used to make an api call when the task is run.
|
90
|
+
# @param bundle_id [String] the id of this bundle.
|
91
|
+
# @param bundled_field [String] the field used to create the
|
92
|
+
# bundled request.
|
93
|
+
# @param bundling_request [Object] the request to pass as the arg to
|
94
|
+
# the api_call.
|
95
|
+
# @param subresponse_field [String] optional field used to demultiplex
|
96
|
+
# responses.
|
97
|
+
def initialize(api_call,
|
98
|
+
bundle_id,
|
99
|
+
bundled_field,
|
100
|
+
bundling_request,
|
101
|
+
subresponse_field: nil)
|
102
|
+
@api_call = api_call
|
103
|
+
@bundle_id = bundle_id
|
104
|
+
@bundled_field = bundled_field
|
105
|
+
@bundling_request = bundling_request
|
106
|
+
@subresponse_field = subresponse_field
|
107
|
+
@inputs = []
|
108
|
+
@events = []
|
109
|
+
end
|
110
|
+
|
111
|
+
# The number of bundled elements in the repeated field.
|
112
|
+
# @return [Numeric]
|
113
|
+
def element_count
|
114
|
+
@inputs.reduce(0) { |a, e| a + e.count }
|
115
|
+
end
|
116
|
+
|
117
|
+
# The size of the request in bytes of the bundled field elements.
|
118
|
+
# @return [Numeric]
|
119
|
+
def request_bytesize
|
120
|
+
@inputs.reduce(0) do |sum, elts|
|
121
|
+
sum + elts.reduce(0) do |inner_sum, elt|
|
122
|
+
inner_sum + elt.to_s.bytesize
|
123
|
+
end
|
124
|
+
end
|
125
|
+
end
|
126
|
+
|
127
|
+
# Call the task's api_call.
|
128
|
+
#
|
129
|
+
# The task's func will be called with the bundling requests function.
|
130
|
+
def run
|
131
|
+
return if @inputs.count == 0
|
132
|
+
request = @bundling_request
|
133
|
+
request[@bundled_field].clear
|
134
|
+
request[@bundled_field].concat(@inputs.flatten)
|
135
|
+
if !@subresponse_field.nil?
|
136
|
+
run_with_subresponses(request)
|
137
|
+
else
|
138
|
+
run_with_no_subresponse(request)
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
142
|
+
# Helper for #run to run the api call with no subresponses.
|
143
|
+
#
|
144
|
+
# @param request [Object] the request to pass as the arg to
|
145
|
+
# the api_call.
|
146
|
+
def run_with_no_subresponse(request)
|
147
|
+
response = @api_call.call(request)
|
148
|
+
@events.each do |event|
|
149
|
+
event.result = response
|
150
|
+
end
|
151
|
+
rescue GaxError => err
|
152
|
+
@events.each do |event|
|
153
|
+
event.result = err
|
154
|
+
end
|
155
|
+
ensure
|
156
|
+
@inputs.clear
|
157
|
+
@events.clear
|
158
|
+
end
|
159
|
+
|
160
|
+
# Helper for #run to run the api call with subresponses.
|
161
|
+
#
|
162
|
+
# @param request [Object] the request to pass as the arg to
|
163
|
+
# the api_call.
|
164
|
+
# @param subresponse_field subresponse_field.
|
165
|
+
def run_with_subresponses(request)
|
166
|
+
response = @api_call.call(request)
|
167
|
+
in_sizes_sum = 0
|
168
|
+
@inputs.each { |elts| in_sizes_sum += elts.count }
|
169
|
+
all_subresponses = response[@subresponse_field.to_s]
|
170
|
+
if all_subresponses.count != in_sizes_sum
|
171
|
+
# TODO: Implement a logging class to handle this.
|
172
|
+
# warn DEMUX_WARNING
|
173
|
+
@events.each do |event|
|
174
|
+
event.result = response
|
175
|
+
end
|
176
|
+
else
|
177
|
+
start = 0
|
178
|
+
@inputs.zip(@events).each do |i, event|
|
179
|
+
response_copy = response.dup
|
180
|
+
subresponses = all_subresponses[start, i.count]
|
181
|
+
response_copy[@subresponse_field].clear
|
182
|
+
response_copy[@subresponse_field].concat(subresponses)
|
183
|
+
start += i.count
|
184
|
+
event.result = response_copy
|
185
|
+
end
|
186
|
+
end
|
187
|
+
rescue GaxError => err
|
188
|
+
@events.each do |event|
|
189
|
+
event.result = err
|
190
|
+
end
|
191
|
+
ensure
|
192
|
+
@inputs.clear
|
193
|
+
@events.clear
|
194
|
+
end
|
195
|
+
|
196
|
+
# This adds elements to the tasks.
|
197
|
+
#
|
198
|
+
# @param elts [Array<Object>] an array of elements that can be appended
|
199
|
+
# to the tasks bundle_field.
|
200
|
+
# @return [Event] an Event that can be used to wait on the response.
|
201
|
+
def extend(elts)
|
202
|
+
elts = [*elts]
|
203
|
+
@inputs.push(elts)
|
204
|
+
event = event_for(elts)
|
205
|
+
@events.push(event)
|
206
|
+
event
|
207
|
+
end
|
208
|
+
|
209
|
+
# Creates an Event that is set when the bundle with elts is sent.
|
210
|
+
#
|
211
|
+
# @param elts [Array<Object>] an array of elements that can be appended
|
212
|
+
# to the tasks bundle_field.
|
213
|
+
# @return [Event] an Event that can be used to wait on the response.
|
214
|
+
def event_for(elts)
|
215
|
+
event = Event.new
|
216
|
+
event.canceller = canceller_for(elts, event)
|
217
|
+
event
|
218
|
+
end
|
219
|
+
|
220
|
+
# Creates a cancellation proc that removes elts.
|
221
|
+
#
|
222
|
+
# The returned proc returns true if all elements were successfully removed
|
223
|
+
# from @inputs and @events.
|
224
|
+
#
|
225
|
+
# @param elts [Array<Object>] an array of elements that can be appended
|
226
|
+
# to the tasks bundle_field.
|
227
|
+
# @param [Event] an Event that can be used to wait on the response.
|
228
|
+
# @return [Proc] the canceller that when called removes the elts
|
229
|
+
# and events.
|
230
|
+
def canceller_for(elts, event)
|
231
|
+
proc do
|
232
|
+
event_index = @events.find_index(event) || -1
|
233
|
+
in_index = @inputs.find_index(elts) || -1
|
234
|
+
@events.delete_at(event_index) unless event_index == -1
|
235
|
+
@inputs.delete_at(in_index) unless in_index == -1
|
236
|
+
if event_index == -1 || in_index == -1
|
237
|
+
false
|
238
|
+
else
|
239
|
+
true
|
240
|
+
end
|
241
|
+
end
|
242
|
+
end
|
243
|
+
|
244
|
+
private :run_with_no_subresponse,
|
245
|
+
:run_with_subresponses,
|
246
|
+
:event_for,
|
247
|
+
:canceller_for
|
248
|
+
end
|
249
|
+
|
250
|
+
# Organizes bundling for an api service that requires it.
|
251
|
+
class Executor
|
252
|
+
# @param options [BundleOptions]configures strategy this instance
|
253
|
+
# uses when executing bundled functions.
|
254
|
+
# @param timer [Timer] the timer is used to handle the functionality of
|
255
|
+
# timing threads.
|
256
|
+
def initialize(options, timer: Timer.new)
|
257
|
+
@options = options
|
258
|
+
@tasks = {}
|
259
|
+
@timer = timer
|
260
|
+
|
261
|
+
# Use a Monitor in order to have the mutex behave like a reentrant lock.
|
262
|
+
@tasks_lock = Monitor.new
|
263
|
+
end
|
264
|
+
|
265
|
+
# Schedules bundle_desc of bundling_request as part of
|
266
|
+
# bundle id.
|
267
|
+
#
|
268
|
+
# @param api_call [Proc] used to make an api call when the task is run.
|
269
|
+
# @param bundle_id [String] the id of this bundle.
|
270
|
+
# @param bundle_desc [BundleDescriptor] describes the structure of the
|
271
|
+
# bundled call.
|
272
|
+
# @param bundling_request [Object] the request to pass as the arg to
|
273
|
+
# the api_call.
|
274
|
+
# @return [Event] an Event that can be used to wait on the response.
|
275
|
+
def schedule(api_call, bundle_id, bundle_desc,
|
276
|
+
bundling_request)
|
277
|
+
bundle = bundle_for(api_call, bundle_id, bundle_desc,
|
278
|
+
bundling_request)
|
279
|
+
elts = bundling_request[bundle_desc.bundled_field.to_s]
|
280
|
+
event = bundle.extend(elts)
|
281
|
+
|
282
|
+
count_threshold = @options.element_count_threshold
|
283
|
+
if count_threshold > 0 && bundle.element_count >= count_threshold
|
284
|
+
run_now(bundle.bundle_id)
|
285
|
+
end
|
286
|
+
|
287
|
+
size_threshold = @options.request_byte_threshold
|
288
|
+
if size_threshold > 0 && bundle.request_bytesize >= size_threshold
|
289
|
+
run_now(bundle.bundle_id)
|
290
|
+
end
|
291
|
+
|
292
|
+
# TODO: Implement byte and element count limits.
|
293
|
+
|
294
|
+
event
|
295
|
+
end
|
296
|
+
|
297
|
+
# Helper function for #schedule.
|
298
|
+
#
|
299
|
+
# Given a return the corresponding bundle for a certain bundle id. Create
|
300
|
+
# a new bundle if the bundle does not exist yet.
|
301
|
+
#
|
302
|
+
# @param api_call [Proc] used to make an api call when the task is run.
|
303
|
+
# @param bundle_id [String] the id of this bundle.
|
304
|
+
# @param bundle_desc [BundleDescriptor] describes the structure of the
|
305
|
+
# bundled call.
|
306
|
+
# @param bundling_request [Object] the request to pass as the arg to
|
307
|
+
# the api_call.
|
308
|
+
# @return [Task] the bundle containing the +api_call+.
|
309
|
+
def bundle_for(api_call, bundle_id, bundle_desc, bundling_request)
|
310
|
+
@tasks_lock.synchronize do
|
311
|
+
return @tasks[bundle_id] if @tasks.key?(bundle_id)
|
312
|
+
bundle = Task.new(api_call, bundle_id, bundle_desc.bundled_field,
|
313
|
+
bundling_request,
|
314
|
+
subresponse_field: bundle_desc.subresponse_field)
|
315
|
+
delay_threshold_millis = @options.delay_threshold_millis
|
316
|
+
if delay_threshold_millis > 0
|
317
|
+
run_later(bundle.bundle_id, delay_threshold_millis)
|
318
|
+
end
|
319
|
+
@tasks[bundle_id] = bundle
|
320
|
+
return bundle
|
321
|
+
end
|
322
|
+
end
|
323
|
+
|
324
|
+
# Helper function for #schedule.
|
325
|
+
#
|
326
|
+
# Creates a new thread that will execute the encapsulated api calls after
|
327
|
+
# the +delay_threshold_millis+ has elapsed. The thread that is
|
328
|
+
# spawned is added to the @threads hash to ensure that the thread will
|
329
|
+
# api call is made before the main thread exits.
|
330
|
+
#
|
331
|
+
# @param bundle_id [String] the id corresponding to the bundle that
|
332
|
+
# is run.
|
333
|
+
# @param delay_threshold_millis [Numeric] the number of micro-seconds to
|
334
|
+
# wait before running the bundle.
|
335
|
+
def run_later(bundle_id, delay_threshold_millis)
|
336
|
+
Thread.new do
|
337
|
+
@timer.run_after(delay_threshold_millis / MILLIS_PER_SECOND) do
|
338
|
+
run_now(bundle_id)
|
339
|
+
end
|
340
|
+
end
|
341
|
+
end
|
342
|
+
|
343
|
+
# Helper function for #schedule.
|
344
|
+
#
|
345
|
+
# Immediately runs the bundle corresponding to the bundle id.
|
346
|
+
# @param bundle_id [String] the id corresponding to the bundle that
|
347
|
+
# is run.
|
348
|
+
def run_now(bundle_id)
|
349
|
+
@tasks_lock.synchronize do
|
350
|
+
if @tasks.key?(bundle_id)
|
351
|
+
task = @tasks.delete(bundle_id)
|
352
|
+
task.run
|
353
|
+
end
|
354
|
+
end
|
355
|
+
end
|
356
|
+
|
357
|
+
# This function should be called before the main thread exits in order to
|
358
|
+
# ensure that all api calls are made.
|
359
|
+
def close
|
360
|
+
@tasks_lock.synchronize do
|
361
|
+
@tasks.each do |bundle_id, _|
|
362
|
+
run_now(bundle_id)
|
363
|
+
end
|
364
|
+
end
|
365
|
+
end
|
366
|
+
|
367
|
+
private :bundle_for, :run_later, :run_now
|
368
|
+
end
|
369
|
+
|
370
|
+
# Container for a thread adding the ability to cancel, check if set, and
|
371
|
+
# get the result of the thread.
|
372
|
+
class Event
|
373
|
+
attr_accessor :canceller
|
374
|
+
attr_reader :result
|
375
|
+
|
376
|
+
def initialize
|
377
|
+
@canceller = nil
|
378
|
+
@result = nil
|
379
|
+
@is_set = false
|
380
|
+
@mutex = Mutex.new
|
381
|
+
@resource = ConditionVariable.new
|
382
|
+
end
|
383
|
+
|
384
|
+
# Setter for the result that is synchronized and broadcasts when set.
|
385
|
+
#
|
386
|
+
# @param obj [Object] an object.
|
387
|
+
# @return [Object] return the passed in param to maintain closure.
|
388
|
+
def result=(obj)
|
389
|
+
@mutex.synchronize do
|
390
|
+
@result = obj
|
391
|
+
@is_set = true
|
392
|
+
@resource.broadcast
|
393
|
+
@result
|
394
|
+
end
|
395
|
+
end
|
396
|
+
|
397
|
+
# Checks to see if the event has been set. A set Event signals that
|
398
|
+
# there is data in @result.
|
399
|
+
# @return [Boolean] Whether the event has been set.
|
400
|
+
def set?
|
401
|
+
@is_set
|
402
|
+
end
|
403
|
+
|
404
|
+
# Invokes the cancellation function provided.
|
405
|
+
# The returned cancellation function returns true if all elements
|
406
|
+
# was removed successfully from the inputs, and false if it was not.
|
407
|
+
def cancel
|
408
|
+
@mutex.synchronize do
|
409
|
+
cancelled = canceller.nil? ? false : canceller.call
|
410
|
+
# Broadcast if the event was successfully cancelled. If not,
|
411
|
+
# the result should end up getting set by the sent api request.
|
412
|
+
# When the result is set, the resource is going to broadcast.
|
413
|
+
@resource.broadcast if cancelled
|
414
|
+
cancelled
|
415
|
+
end
|
416
|
+
end
|
417
|
+
|
418
|
+
# This is used to wait for a bundle request is complete and the event
|
419
|
+
# result is set.
|
420
|
+
#
|
421
|
+
# @param timeout_millis [Numeric] The number of milliseconds to wait
|
422
|
+
# before ceasing to wait. If nil, this function will wait
|
423
|
+
# indefinitely.
|
424
|
+
def wait(timeout_millis: nil)
|
425
|
+
@mutex.synchronize do
|
426
|
+
return @is_set if @is_set
|
427
|
+
t = timeout_millis.nil? ? nil : timeout_millis / MILLIS_PER_SECOND
|
428
|
+
@resource.wait(@mutex, t)
|
429
|
+
@is_set
|
430
|
+
end
|
431
|
+
end
|
432
|
+
end
|
433
|
+
|
434
|
+
# This class will be used to run the #run_later tasks for the bundle.
|
435
|
+
class Timer
|
436
|
+
def run_after(delay_threshold)
|
437
|
+
sleep delay_threshold
|
438
|
+
yield
|
439
|
+
end
|
440
|
+
end
|
441
|
+
|
442
|
+
module_function :compute_bundle_id, :str_dotted_access
|
443
|
+
private_constant :DEMUX_WARNING
|
444
|
+
end
|
445
|
+
end
|