google-gax 0.4.0 → 0.4.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: f00dba40d04710327a4348f2b0919f423b81ab3d
4
- data.tar.gz: 16ebb311b89300f865849238f840acedd3aec149
3
+ metadata.gz: 56e4ca24c4b15383126e5401cffc794a47f0192a
4
+ data.tar.gz: f7849c578e8d97aee04dc216698437e971c7d831
5
5
  SHA512:
6
- metadata.gz: 128e33ba96e202d5abf7b941d922737fc56115cd3c4d82b9fe3b331beb7639704f9ac1e86980f64757c2169b0fdee8e9668b985d8a81cb2f9c4aee46c3eb1255
7
- data.tar.gz: 234a0c126f754c7db3690f53aeda09a689143cc52e141d787b3cd17259c7bc0db371cca13544a85253a450ed11178a45e8b05c0c06c3c342f84ceabacab465f9
6
+ metadata.gz: 10eb2d5554f5b82e9b4c890228a11402c3e2a43bc35108f3e56d6f67658207db5ae7c40468731c6fcb05f30b4ade556d7e04d15b7dd8783490f23ced39d629fe
7
+ data.tar.gz: e2c762e9ffc3bfd562f9d0a7e6595f81154399cb75788a013d74378371d1bfa45d1950b8b570e10665bb51a70c9cba0b7524a21236d0c4e9c960b94542bbbd9d
@@ -30,6 +30,7 @@
30
30
  require 'time'
31
31
 
32
32
  require 'google/gax/errors'
33
+ require 'google/gax/bundling'
33
34
 
34
35
  # rubocop:disable Metrics/ModuleLength
35
36
 
@@ -285,9 +286,11 @@ module Google
285
286
  def bundleable(desc)
286
287
  proc do |api_call, request, settings|
287
288
  return api_call(request) unless settings.bundler
288
- the_id = bundling.compute_bundle_id(request,
289
- desc.request_discriminator_fields)
290
- return bundler.schedule(api_call, the_id, desc, request)
289
+ the_id = Google::Gax.compute_bundle_id(
290
+ request,
291
+ desc.request_discriminator_fields
292
+ )
293
+ settings.bundler.schedule(api_call, the_id, desc, request)
291
294
  end
292
295
  end
293
296
 
@@ -373,8 +376,7 @@ module Google
373
376
  # @return [Proc] the original proc updated to the timeout arg
374
377
  def add_timeout_arg(a_func, timeout, kwargs)
375
378
  proc do |request|
376
- kwargs[:timeout] = timeout
377
- a_func.call(request, **kwargs)
379
+ a_func.call(request, deadline: Time.now + timeout, metadata: kwargs)
378
380
  end
379
381
  end
380
382
 
@@ -0,0 +1,445 @@
1
+ # Copyright 2016, Google Inc.
2
+ # All rights reserved.
3
+ #
4
+ # Redistribution and use in source and binary forms, with or without
5
+ # modification, are permitted provided that the following conditions are
6
+ # met:
7
+ #
8
+ # * Redistributions of source code must retain the above copyright
9
+ # notice, this list of conditions and the following disclaimer.
10
+ # * Redistributions in binary form must reproduce the above
11
+ # copyright notice, this list of conditions and the following disclaimer
12
+ # in the documentation and/or other materials provided with the
13
+ # distribution.
14
+ # * Neither the name of Google Inc. nor the names of its
15
+ # contributors may be used to endorse or promote products derived from
16
+ # this software without specific prior written permission.
17
+ #
18
+ # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19
+ # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20
+ # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21
+ # A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22
+ # OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23
+ # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24
+ # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25
+ # DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26
+ # THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27
+ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28
+ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
+
30
+ module Google
31
+ # Gax defines Google API extensions
32
+ module Gax
33
+ DEMUX_WARNING = [
34
+ 'Warning: cannot demultiplex the bundled response, got ',
35
+ '%d subresponses; want %d, each bundled request will ',
36
+ 'receive all responses'
37
+ ].join
38
+
39
+ # Helper function for #compute_bundle_id.
40
+ # Used to retrieve a nested field signified by name where dots in name
41
+ # indicate nested objects.
42
+ #
43
+ # @param obj [Object] an object.
44
+ # @param name [String] a name for a field in the object.
45
+ # @return [String, nil] value of named attribute. Can be nil.
46
+ def str_dotted_access(obj, name)
47
+ name.split('.').each do |part|
48
+ obj = obj[part]
49
+ break if obj.nil?
50
+ end
51
+ obj.nil? ? nil : obj.to_s
52
+ end
53
+
54
+ # Computes a bundle id from the discriminator fields of `obj`.
55
+ #
56
+ # +discriminator_fields+ may include '.' as a separator, which is used to
57
+ # indicate object traversal. This is meant to allow fields in the
58
+ # computed bundle_id.
59
+ # the return is an array computed by going through the discriminator fields
60
+ # in order and obtaining the str(value) object field (or nested object
61
+ # field) if any discriminator field cannot be found, ValueError is raised.
62
+ #
63
+ # @param obj [Object] an object.
64
+ # @param discriminator_fields [Array<String>] a list of discriminator
65
+ # fields in the order to be to be used in the id.
66
+ # @return [Array<Object>] array of objects computed as described above.
67
+ def compute_bundle_id(obj, discriminator_fields)
68
+ result = []
69
+ discriminator_fields.each do |field|
70
+ result.push(str_dotted_access(obj, field))
71
+ end
72
+ result
73
+ end
74
+
75
+ # rubocop:disable Metrics/ClassLength
76
+
77
+ # Coordinates the execution of a single bundle.
78
+ #
79
+ # @!attribute [r] bundle_id
80
+ # @return [String] the id of this bundle.
81
+ # @!attribute [r] bundled_field
82
+ # @return [String] the field used to create the bundled request.
83
+ # @!attribute [r] subresponse_field
84
+ # @return [String] tptional field used to demultiplex responses.
85
+ class Task
86
+ attr_reader :bundle_id, :bundled_field,
87
+ :subresponse_field
88
+
89
+ # @param api_call [Proc] used to make an api call when the task is run.
90
+ # @param bundle_id [String] the id of this bundle.
91
+ # @param bundled_field [String] the field used to create the
92
+ # bundled request.
93
+ # @param bundling_request [Object] the request to pass as the arg to
94
+ # the api_call.
95
+ # @param subresponse_field [String] optional field used to demultiplex
96
+ # responses.
97
+ def initialize(api_call,
98
+ bundle_id,
99
+ bundled_field,
100
+ bundling_request,
101
+ subresponse_field: nil)
102
+ @api_call = api_call
103
+ @bundle_id = bundle_id
104
+ @bundled_field = bundled_field
105
+ @bundling_request = bundling_request
106
+ @subresponse_field = subresponse_field
107
+ @inputs = []
108
+ @events = []
109
+ end
110
+
111
+ # The number of bundled elements in the repeated field.
112
+ # @return [Numeric]
113
+ def element_count
114
+ @inputs.reduce(0) { |a, e| a + e.count }
115
+ end
116
+
117
+ # The size of the request in bytes of the bundled field elements.
118
+ # @return [Numeric]
119
+ def request_bytesize
120
+ @inputs.reduce(0) do |sum, elts|
121
+ sum + elts.reduce(0) do |inner_sum, elt|
122
+ inner_sum + elt.to_s.bytesize
123
+ end
124
+ end
125
+ end
126
+
127
+ # Call the task's api_call.
128
+ #
129
+ # The task's func will be called with the bundling requests function.
130
+ def run
131
+ return if @inputs.count == 0
132
+ request = @bundling_request
133
+ request[@bundled_field].clear
134
+ request[@bundled_field].concat(@inputs.flatten)
135
+ if !@subresponse_field.nil?
136
+ run_with_subresponses(request)
137
+ else
138
+ run_with_no_subresponse(request)
139
+ end
140
+ end
141
+
142
+ # Helper for #run to run the api call with no subresponses.
143
+ #
144
+ # @param request [Object] the request to pass as the arg to
145
+ # the api_call.
146
+ def run_with_no_subresponse(request)
147
+ response = @api_call.call(request)
148
+ @events.each do |event|
149
+ event.result = response
150
+ end
151
+ rescue GaxError => err
152
+ @events.each do |event|
153
+ event.result = err
154
+ end
155
+ ensure
156
+ @inputs.clear
157
+ @events.clear
158
+ end
159
+
160
+ # Helper for #run to run the api call with subresponses.
161
+ #
162
+ # @param request [Object] the request to pass as the arg to
163
+ # the api_call.
164
+ # @param subresponse_field subresponse_field.
165
+ def run_with_subresponses(request)
166
+ response = @api_call.call(request)
167
+ in_sizes_sum = 0
168
+ @inputs.each { |elts| in_sizes_sum += elts.count }
169
+ all_subresponses = response[@subresponse_field.to_s]
170
+ if all_subresponses.count != in_sizes_sum
171
+ # TODO: Implement a logging class to handle this.
172
+ # warn DEMUX_WARNING
173
+ @events.each do |event|
174
+ event.result = response
175
+ end
176
+ else
177
+ start = 0
178
+ @inputs.zip(@events).each do |i, event|
179
+ response_copy = response.dup
180
+ subresponses = all_subresponses[start, i.count]
181
+ response_copy[@subresponse_field].clear
182
+ response_copy[@subresponse_field].concat(subresponses)
183
+ start += i.count
184
+ event.result = response_copy
185
+ end
186
+ end
187
+ rescue GaxError => err
188
+ @events.each do |event|
189
+ event.result = err
190
+ end
191
+ ensure
192
+ @inputs.clear
193
+ @events.clear
194
+ end
195
+
196
+ # This adds elements to the tasks.
197
+ #
198
+ # @param elts [Array<Object>] an array of elements that can be appended
199
+ # to the tasks bundle_field.
200
+ # @return [Event] an Event that can be used to wait on the response.
201
+ def extend(elts)
202
+ elts = [*elts]
203
+ @inputs.push(elts)
204
+ event = event_for(elts)
205
+ @events.push(event)
206
+ event
207
+ end
208
+
209
+ # Creates an Event that is set when the bundle with elts is sent.
210
+ #
211
+ # @param elts [Array<Object>] an array of elements that can be appended
212
+ # to the tasks bundle_field.
213
+ # @return [Event] an Event that can be used to wait on the response.
214
+ def event_for(elts)
215
+ event = Event.new
216
+ event.canceller = canceller_for(elts, event)
217
+ event
218
+ end
219
+
220
+ # Creates a cancellation proc that removes elts.
221
+ #
222
+ # The returned proc returns true if all elements were successfully removed
223
+ # from @inputs and @events.
224
+ #
225
+ # @param elts [Array<Object>] an array of elements that can be appended
226
+ # to the tasks bundle_field.
227
+ # @param [Event] an Event that can be used to wait on the response.
228
+ # @return [Proc] the canceller that when called removes the elts
229
+ # and events.
230
+ def canceller_for(elts, event)
231
+ proc do
232
+ event_index = @events.find_index(event) || -1
233
+ in_index = @inputs.find_index(elts) || -1
234
+ @events.delete_at(event_index) unless event_index == -1
235
+ @inputs.delete_at(in_index) unless in_index == -1
236
+ if event_index == -1 || in_index == -1
237
+ false
238
+ else
239
+ true
240
+ end
241
+ end
242
+ end
243
+
244
+ private :run_with_no_subresponse,
245
+ :run_with_subresponses,
246
+ :event_for,
247
+ :canceller_for
248
+ end
249
+
250
+ # Organizes bundling for an api service that requires it.
251
+ class Executor
252
+ # @param options [BundleOptions]configures strategy this instance
253
+ # uses when executing bundled functions.
254
+ # @param timer [Timer] the timer is used to handle the functionality of
255
+ # timing threads.
256
+ def initialize(options, timer: Timer.new)
257
+ @options = options
258
+ @tasks = {}
259
+ @timer = timer
260
+
261
+ # Use a Monitor in order to have the mutex behave like a reentrant lock.
262
+ @tasks_lock = Monitor.new
263
+ end
264
+
265
+ # Schedules bundle_desc of bundling_request as part of
266
+ # bundle id.
267
+ #
268
+ # @param api_call [Proc] used to make an api call when the task is run.
269
+ # @param bundle_id [String] the id of this bundle.
270
+ # @param bundle_desc [BundleDescriptor] describes the structure of the
271
+ # bundled call.
272
+ # @param bundling_request [Object] the request to pass as the arg to
273
+ # the api_call.
274
+ # @return [Event] an Event that can be used to wait on the response.
275
+ def schedule(api_call, bundle_id, bundle_desc,
276
+ bundling_request)
277
+ bundle = bundle_for(api_call, bundle_id, bundle_desc,
278
+ bundling_request)
279
+ elts = bundling_request[bundle_desc.bundled_field.to_s]
280
+ event = bundle.extend(elts)
281
+
282
+ count_threshold = @options.element_count_threshold
283
+ if count_threshold > 0 && bundle.element_count >= count_threshold
284
+ run_now(bundle.bundle_id)
285
+ end
286
+
287
+ size_threshold = @options.request_byte_threshold
288
+ if size_threshold > 0 && bundle.request_bytesize >= size_threshold
289
+ run_now(bundle.bundle_id)
290
+ end
291
+
292
+ # TODO: Implement byte and element count limits.
293
+
294
+ event
295
+ end
296
+
297
+ # Helper function for #schedule.
298
+ #
299
+ # Given a return the corresponding bundle for a certain bundle id. Create
300
+ # a new bundle if the bundle does not exist yet.
301
+ #
302
+ # @param api_call [Proc] used to make an api call when the task is run.
303
+ # @param bundle_id [String] the id of this bundle.
304
+ # @param bundle_desc [BundleDescriptor] describes the structure of the
305
+ # bundled call.
306
+ # @param bundling_request [Object] the request to pass as the arg to
307
+ # the api_call.
308
+ # @return [Task] the bundle containing the +api_call+.
309
+ def bundle_for(api_call, bundle_id, bundle_desc, bundling_request)
310
+ @tasks_lock.synchronize do
311
+ return @tasks[bundle_id] if @tasks.key?(bundle_id)
312
+ bundle = Task.new(api_call, bundle_id, bundle_desc.bundled_field,
313
+ bundling_request,
314
+ subresponse_field: bundle_desc.subresponse_field)
315
+ delay_threshold_millis = @options.delay_threshold_millis
316
+ if delay_threshold_millis > 0
317
+ run_later(bundle.bundle_id, delay_threshold_millis)
318
+ end
319
+ @tasks[bundle_id] = bundle
320
+ return bundle
321
+ end
322
+ end
323
+
324
+ # Helper function for #schedule.
325
+ #
326
+ # Creates a new thread that will execute the encapsulated api calls after
327
+ # the +delay_threshold_millis+ has elapsed. The thread that is
328
+ # spawned is added to the @threads hash to ensure that the thread will
329
+ # api call is made before the main thread exits.
330
+ #
331
+ # @param bundle_id [String] the id corresponding to the bundle that
332
+ # is run.
333
+ # @param delay_threshold_millis [Numeric] the number of micro-seconds to
334
+ # wait before running the bundle.
335
+ def run_later(bundle_id, delay_threshold_millis)
336
+ Thread.new do
337
+ @timer.run_after(delay_threshold_millis / MILLIS_PER_SECOND) do
338
+ run_now(bundle_id)
339
+ end
340
+ end
341
+ end
342
+
343
+ # Helper function for #schedule.
344
+ #
345
+ # Immediately runs the bundle corresponding to the bundle id.
346
+ # @param bundle_id [String] the id corresponding to the bundle that
347
+ # is run.
348
+ def run_now(bundle_id)
349
+ @tasks_lock.synchronize do
350
+ if @tasks.key?(bundle_id)
351
+ task = @tasks.delete(bundle_id)
352
+ task.run
353
+ end
354
+ end
355
+ end
356
+
357
+ # This function should be called before the main thread exits in order to
358
+ # ensure that all api calls are made.
359
+ def close
360
+ @tasks_lock.synchronize do
361
+ @tasks.each do |bundle_id, _|
362
+ run_now(bundle_id)
363
+ end
364
+ end
365
+ end
366
+
367
+ private :bundle_for, :run_later, :run_now
368
+ end
369
+
370
+ # Container for a thread adding the ability to cancel, check if set, and
371
+ # get the result of the thread.
372
+ class Event
373
+ attr_accessor :canceller
374
+ attr_reader :result
375
+
376
+ def initialize
377
+ @canceller = nil
378
+ @result = nil
379
+ @is_set = false
380
+ @mutex = Mutex.new
381
+ @resource = ConditionVariable.new
382
+ end
383
+
384
+ # Setter for the result that is synchronized and broadcasts when set.
385
+ #
386
+ # @param obj [Object] an object.
387
+ # @return [Object] return the passed in param to maintain closure.
388
+ def result=(obj)
389
+ @mutex.synchronize do
390
+ @result = obj
391
+ @is_set = true
392
+ @resource.broadcast
393
+ @result
394
+ end
395
+ end
396
+
397
+ # Checks to see if the event has been set. A set Event signals that
398
+ # there is data in @result.
399
+ # @return [Boolean] Whether the event has been set.
400
+ def set?
401
+ @is_set
402
+ end
403
+
404
+ # Invokes the cancellation function provided.
405
+ # The returned cancellation function returns true if all elements
406
+ # was removed successfully from the inputs, and false if it was not.
407
+ def cancel
408
+ @mutex.synchronize do
409
+ cancelled = canceller.nil? ? false : canceller.call
410
+ # Broadcast if the event was successfully cancelled. If not,
411
+ # the result should end up getting set by the sent api request.
412
+ # When the result is set, the resource is going to broadcast.
413
+ @resource.broadcast if cancelled
414
+ cancelled
415
+ end
416
+ end
417
+
418
+ # This is used to wait for a bundle request is complete and the event
419
+ # result is set.
420
+ #
421
+ # @param timeout_millis [Numeric] The number of milliseconds to wait
422
+ # before ceasing to wait. If nil, this function will wait
423
+ # indefinitely.
424
+ def wait(timeout_millis: nil)
425
+ @mutex.synchronize do
426
+ return @is_set if @is_set
427
+ t = timeout_millis.nil? ? nil : timeout_millis / MILLIS_PER_SECOND
428
+ @resource.wait(@mutex, t)
429
+ @is_set
430
+ end
431
+ end
432
+ end
433
+
434
+ # This class will be used to run the #run_later tasks for the bundle.
435
+ class Timer
436
+ def run_after(delay_threshold)
437
+ sleep delay_threshold
438
+ yield
439
+ end
440
+ end
441
+
442
+ module_function :compute_bundle_id, :str_dotted_access
443
+ private_constant :DEMUX_WARNING
444
+ end
445
+ end