aws-flow 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. data/Gemfile +8 -0
  2. data/LICENSE.TXT +15 -0
  3. data/NOTICE.TXT +14 -0
  4. data/Rakefile +39 -0
  5. data/aws-flow-core/Gemfile +9 -0
  6. data/aws-flow-core/LICENSE.TXT +15 -0
  7. data/aws-flow-core/NOTICE.TXT +14 -0
  8. data/aws-flow-core/Rakefile +27 -0
  9. data/aws-flow-core/aws-flow-core.gemspec +12 -0
  10. data/aws-flow-core/lib/aws/flow.rb +26 -0
  11. data/aws-flow-core/lib/aws/flow/async_backtrace.rb +134 -0
  12. data/aws-flow-core/lib/aws/flow/async_scope.rb +195 -0
  13. data/aws-flow-core/lib/aws/flow/begin_rescue_ensure.rb +386 -0
  14. data/aws-flow-core/lib/aws/flow/fiber.rb +77 -0
  15. data/aws-flow-core/lib/aws/flow/flow_utils.rb +50 -0
  16. data/aws-flow-core/lib/aws/flow/future.rb +109 -0
  17. data/aws-flow-core/lib/aws/flow/implementation.rb +151 -0
  18. data/aws-flow-core/lib/aws/flow/simple_dfa.rb +85 -0
  19. data/aws-flow-core/lib/aws/flow/tasks.rb +405 -0
  20. data/aws-flow-core/test/aws/async_backtrace_spec.rb +41 -0
  21. data/aws-flow-core/test/aws/async_scope_spec.rb +118 -0
  22. data/aws-flow-core/test/aws/begin_rescue_ensure_spec.rb +665 -0
  23. data/aws-flow-core/test/aws/external_task_spec.rb +197 -0
  24. data/aws-flow-core/test/aws/factories.rb +52 -0
  25. data/aws-flow-core/test/aws/fiber_condition_variable_spec.rb +163 -0
  26. data/aws-flow-core/test/aws/fiber_spec.rb +78 -0
  27. data/aws-flow-core/test/aws/flow_spec.rb +255 -0
  28. data/aws-flow-core/test/aws/future_spec.rb +210 -0
  29. data/aws-flow-core/test/aws/rubyflow.rb +22 -0
  30. data/aws-flow-core/test/aws/simple_dfa_spec.rb +63 -0
  31. data/aws-flow-core/test/aws/spec_helper.rb +36 -0
  32. data/aws-flow.gemspec +13 -0
  33. data/lib/aws/decider.rb +67 -0
  34. data/lib/aws/decider/activity.rb +408 -0
  35. data/lib/aws/decider/activity_definition.rb +111 -0
  36. data/lib/aws/decider/async_decider.rb +673 -0
  37. data/lib/aws/decider/async_retrying_executor.rb +153 -0
  38. data/lib/aws/decider/data_converter.rb +40 -0
  39. data/lib/aws/decider/decider.rb +511 -0
  40. data/lib/aws/decider/decision_context.rb +60 -0
  41. data/lib/aws/decider/exceptions.rb +178 -0
  42. data/lib/aws/decider/executor.rb +149 -0
  43. data/lib/aws/decider/flow_defaults.rb +70 -0
  44. data/lib/aws/decider/generic_client.rb +178 -0
  45. data/lib/aws/decider/history_helper.rb +173 -0
  46. data/lib/aws/decider/implementation.rb +82 -0
  47. data/lib/aws/decider/options.rb +607 -0
  48. data/lib/aws/decider/state_machines.rb +373 -0
  49. data/lib/aws/decider/task_handler.rb +76 -0
  50. data/lib/aws/decider/task_poller.rb +207 -0
  51. data/lib/aws/decider/utilities.rb +187 -0
  52. data/lib/aws/decider/worker.rb +324 -0
  53. data/lib/aws/decider/workflow_client.rb +374 -0
  54. data/lib/aws/decider/workflow_clock.rb +104 -0
  55. data/lib/aws/decider/workflow_definition.rb +101 -0
  56. data/lib/aws/decider/workflow_definition_factory.rb +53 -0
  57. data/lib/aws/decider/workflow_enabled.rb +26 -0
  58. data/test/aws/decider_spec.rb +1299 -0
  59. data/test/aws/factories.rb +45 -0
  60. data/test/aws/integration_spec.rb +3108 -0
  61. data/test/aws/spec_helper.rb +23 -0
  62. metadata +138 -0
@@ -0,0 +1,386 @@
1
+ ##
2
+ # Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License").
5
+ # You may not use this file except in compliance with the License.
6
+ # A copy of the License is located at
7
+ #
8
+ # http://aws.amazon.com/apache2.0
9
+ #
10
+ # or in the "license" file accompanying this file. This file is distributed
11
+ # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12
+ # express or implied. See the License for the specific language governing
13
+ # permissions and limitations under the License.
14
+ ##
15
+
16
+ require 'aws/flow/simple_dfa'
17
+ require 'set'
18
+
19
+ module AWS
20
+ module Flow
21
+ module Core
22
+
23
+ # This class allows asynchronous error handling within the AWS Flow Framework for Ruby. Calling
24
+ # {#begin}/{#rescue}/{#ensure} is similar to Ruby's native `begin`/`rescue`/`end` semantics.
25
+ class BeginRescueEnsure < FlowFiber
26
+
27
+ extend SimpleDFA
28
+ attr_accessor :parent, :begin_task, :ensure_task, :rescue_tasks,
29
+ :rescue_exceptions, :failure, :cancelled, :heirs, :nonDaemonHeirsCount, :executor, :result
30
+ attr_reader :backtrace, :__context__
31
+
32
+ # Create a new BeginRescueEnsure object, with the provided options.
33
+ #
34
+ # @param options
35
+ # Options to set for the class.
36
+ #
37
+ # @option options [Object] :parent
38
+ # The parent object.
39
+ #
40
+ def initialize(options = {})
41
+ # We have two different arrays, rather than a hash,
42
+ # because we want to ensure that we process the rescues in the order
43
+ # they are written, and because prior to Ruby 1.9, hashes will not
44
+ # return their elements in the order they were inserted.
45
+ @rescue_exceptions = []
46
+ @rescue_tasks = []
47
+ @parent = options[:parent] || Fiber.current.__context__
48
+ @current = @parent
49
+ @executor = @parent.executor
50
+ @__context__ = self
51
+ @nonDaemonHeirsCount = 0
52
+ @current_state ||= self.class.get_start_state
53
+ @heirs = Set.new
54
+ @backtrace = make_backtrace(@parent.backtrace)
55
+ @result = Future.new
56
+ super() { consume(:run) }
57
+ end
58
+
59
+
60
+ # @!visibility private
61
+ def is_daemon?
62
+ false
63
+ end
64
+
65
+
66
+ # @!visibility private
67
+ def <<(async_task)
68
+ # Not going to include the promise to wait for, as it would appear that
69
+ # Fibers can wait on futures from their point of origin as part of their
70
+ # implementation, as opposed to adding the callback here.
71
+ check_closed
72
+ if ! @heirs.member? async_task
73
+ @heirs << async_task
74
+ if ! async_task.is_daemon?
75
+ @nonDaemonHeirsCount += 1
76
+ end
77
+ end
78
+ @executor << async_task
79
+ self
80
+ end
81
+
82
+ # @!visibility private
83
+ def get_closest_containing_scope
84
+ # BRE's are special in that they act as a containing scope, so that things
85
+ # created in BRE's treat it as the parent, so that it can track the heirs
86
+ # correctly and close only when nonDaemonHeirsCount is 0
87
+ self
88
+ end
89
+
90
+ # @!visibility private
91
+ def check_closed
92
+ raise IllegalStateException, @failure if @current_state == :closed
93
+ end
94
+
95
+ # Fails the task, cancels all of its heirs, and then updates the state.
96
+ #
97
+ # @param this_task
98
+ # The task to fail.
99
+ #
100
+ # @param error
101
+ # The error associated with the failure.
102
+ #
103
+ def fail(this_task, error)
104
+ check_closed
105
+ if ( ! (error.class <= CancellationException) || @failure == nil && !@daemondCausedCancellation)
106
+ backtrace = AsyncBacktrace.create_from_exception(@backtrace, error)
107
+ error.set_backtrace(backtrace.backtrace) if backtrace
108
+ @failure = error
109
+ end
110
+ task_out = @heirs.delete?(this_task)
111
+ raise "There was a task attempted to be removed from a BRE, when the BRE did not have that task as an heir" unless task_out
112
+ @nonDaemonHeirsCount -= 1 if ! this_task.is_daemon?
113
+ cancelHeirs
114
+ update_state
115
+ end
116
+
117
+ # Removes the task and updates the state
118
+ #
119
+ # @param this_task
120
+ # The task to remove.
121
+ #
122
+ def remove(this_task)
123
+ check_closed
124
+
125
+ task_out = @heirs.delete?(this_task)
126
+ raise "There was a task attempted to be removed from a BRE, when the BRE did not have that task as an heir" unless task_out
127
+ @nonDaemonHeirsCount -= 1 if ! this_task.is_daemon?
128
+ update_state
129
+ end
130
+
131
+ # @!visibility private
132
+ def cancelHeirs
133
+ toCancel = @heirs.dup
134
+ toCancel.each { |heir| heir.cancel(@failure) }
135
+ end
136
+
137
+ # @!visibility private
138
+ def merge_stacktraces(failure, this_backtrace, error)
139
+ backtrace = AsyncBacktrace.create_from_exception(this_backtrace, error)
140
+ failure.set_backtrace(backtrace.backtrace) if backtrace
141
+ end
142
+
143
+ # @!visibility private
144
+ def cancel(error)
145
+ if @current_state == :created
146
+ @current_state = :closed
147
+ @parent.remove(self)
148
+ return
149
+ end
150
+ if @failure == nil
151
+ @cancelled = true
152
+ details = (error.respond_to? :details) ? error.details : nil
153
+ reason = (error.respond_to? :reason) ? error.reason : nil
154
+ @failure = CancellationException.new(reason, details)
155
+ @failure.set_backtrace(@backtrace.backtrace) if @backtrace
156
+ if @current_state == :begin
157
+ cancelHeirs
158
+ end
159
+ end
160
+ end
161
+
162
+ # Actually runs the BRE, by going through the DFA with the symbol :run.
163
+ # @!visibility private
164
+ def run
165
+ this_failure = @failure
166
+ begin
167
+ consume(:run)
168
+ rescue Exception => error
169
+ if this_failure != error
170
+ backtrace = AsyncBacktrace.create_from_exception(@backtrace, error)
171
+ error.set_backtrace(backtrace.backtrace) if backtrace
172
+ end
173
+ @failure = error
174
+ cancelHeirs
175
+ ensure
176
+ update_state
177
+ raise @failure if (!!@failure && @current_state == :closed)
178
+ end
179
+ end
180
+
181
+ # @!visibility private
182
+ def alive?
183
+ @current_state != :closed
184
+ end
185
+
186
+ # Updates the state based on the most recent transitions in the DFA
187
+ # @!visibility private
188
+ def update_state
189
+ #TODO ? Add the ! @executed part
190
+ #return if @current_state == :closed || ! @executed
191
+ return if @current_state == :closed
192
+ if @nonDaemonHeirsCount == 0
193
+ if @heirs.empty?
194
+ consume(:update_state)
195
+ else
196
+ @daemondCausedCancellation = true if @failure == nil
197
+ cancelHeirs
198
+ end
199
+ end
200
+ end
201
+
202
+ # @!visibility private
203
+ def get_heirs
204
+ # TODO fix this so it returns string instead of printing to stdout
205
+ str = "I am a BeginRescueEnsure with #{heirs.length} heirs
206
+ my begin block looks like #{@begin_task}" +
207
+ @heirs.map(&:get_heirs).to_s
208
+
209
+ # (@heirs.each(&:get_heirs) + [self]).flatten
210
+ end
211
+
212
+ # @!visibility private
213
+ init(:created)
214
+ {
215
+ [:created, :run] => lambda { |bre| bre.current_state = :begin; bre.run },
216
+ [:begin, :run] => lambda { |bre| bre << bre.begin_task },
217
+ [:begin, :update_state] => lambda do |bre|
218
+ if bre.failure == nil
219
+ bre.current_state = :ensure
220
+ else
221
+ bre.current_state = :rescue;
222
+ end
223
+ bre.run
224
+ end,
225
+ [:rescue, :run] => lambda do |bre|
226
+ # Emulates the behavior of the actual Ruby rescue, see
227
+ # http://Ruby-doc.org/docs/ProgrammingRuby/html/tut_exceptions.html
228
+ # for more details
229
+ bre.rescue_exceptions.each_index do |index|
230
+ this_failure = bre.failure
231
+ failure_class = bre.failure.is_a?(Exception) ? bre.failure.class : bre.failure
232
+ if failure_class <= bre.rescue_exceptions[index]
233
+ bre.result.unset
234
+ bre.failure = nil
235
+ task = bre.rescue_tasks[index]
236
+ bre << Task.new(bre) { bre.result.set(task.call(this_failure)) }
237
+ # bre.rescue_tasks[index].call(this_failure)
238
+ break
239
+ end
240
+ end
241
+ end,
242
+ [:rescue, :update_state] => lambda { |bre| bre.current_state = :ensure; bre.run},
243
+ [:ensure, :run] => lambda do |bre|
244
+ bre << bre.ensure_task if bre.ensure_task
245
+ end,
246
+ [:ensure, :update_state] => lambda do |bre|
247
+ bre.current_state = :closed
248
+ if bre.failure == nil
249
+ bre.parent.remove(bre)
250
+ else
251
+ bre.parent.fail(bre, bre.failure)
252
+ end
253
+ end,
254
+ }.each_pair do |key, func|
255
+ add_transition(key.first, key.last) { |t| func.call(t) }
256
+ end
257
+ # That is, any transition from closed leads back to itself
258
+ define_general(:closed) { |t| t.current_state = :closed }
259
+
260
+ # Binds the block to the a lambda to be called when we get to the begin part of the DFA
261
+ #
262
+ # @param block
263
+ # The code block to be called when asynchronous *begin* starts.
264
+ #
265
+ def begin(block)
266
+ raise "Duplicated begin" if @begin_task
267
+ # @begin_task = lambda { block.call }
268
+ @begin_task = Task.new(self) { @result.set(block.call) }
269
+ end
270
+
271
+ # Binds the block to the a lambda to be called when we get to the rescue part of the DFA
272
+ #
273
+ # @param error_type
274
+ # The error type.
275
+ #
276
+ # @param block
277
+ # The code block to be called when asynchronous *rescue* starts.
278
+ #
279
+ def rescue(error_type, block)
280
+ this_task = lambda { |failure| block.call(failure) }
281
+ if @rescue_exceptions.include? error_type
282
+ raise "You have already registered #{error_type}!"
283
+ end
284
+ @rescue_exceptions << error_type
285
+ @rescue_tasks << this_task
286
+ end
287
+
288
+ # Binds the block to the a lambda to be called when we get to the ensure part of the DFA
289
+ #
290
+ # @param block
291
+ # The code block to be called when asynchronous *ensure* starts.
292
+ #
293
+ def ensure(block)
294
+ raise "Duplicated ensure" if @ensure_task
295
+ @ensure_task = Task.new(self) { block.call }
296
+ end
297
+
298
+ def schedule
299
+ @parent << self
300
+ end
301
+ end
302
+
303
+ # Class to ensure that all the inner guts of BRE aren't exposed. This function is passed in when error_handler is
304
+ # called, like this:
305
+ #
306
+ # error_handler do |t|
307
+ # t.begin { "This is the begin" }
308
+ # t.rescue(Exception) { "This is the rescue" }
309
+ # t.ensure { trace << t.begin_task }
310
+ # end
311
+ #
312
+ # The *t* that is passed in is actually a {BeginRescueEnsureWrapper}, which will only pass begin/rescue/ensure
313
+ # onto the {BeginRescueEnsure} class itself.
314
+ #
315
+ class BeginRescueEnsureWrapper < FlowFiber
316
+ # Also has a few methods to ensure Fiber-ness, such as get_heirs and cancel.
317
+ attr_reader :__context__
318
+
319
+ # Creates a new BeginRescueEnsureWrapper instance.
320
+ #
321
+ # @param block
322
+ # A code block to be called.
323
+ #
324
+ # @param begin_rescue_ensure
325
+ # The {BeginRescueEnsure} instance to wrap.
326
+ #
327
+ def initialize(block, begin_rescue_ensure)
328
+ @beginRescueEnsure = begin_rescue_ensure
329
+ @__context__ = @beginRescueEnsure
330
+ super() do
331
+ begin
332
+ block.call(self)
333
+ ensure
334
+ @__context__.parent.remove(self)
335
+ end
336
+
337
+ end
338
+ end
339
+
340
+ # @!visibility private
341
+ def get_heirs
342
+ p "I am a BREWrapper"
343
+ return
344
+ end
345
+
346
+ def cancel(error_type)
347
+ @beginRescueEnsure.parent.cancel(self)
348
+ end
349
+
350
+ # @!visibility private
351
+ #
352
+ # @return [false]
353
+ # Always returns `false`.
354
+ #
355
+ def is_daemon?
356
+ false
357
+ end
358
+
359
+ # Gets the parent of the {BeginRescueEnsure} instance held by this class.
360
+ def get_closest_containing_scope
361
+ @beginRescueEnsure.parent
362
+ end
363
+
364
+ # (see BeginRescueEnsure#begin)
365
+ def begin(&block) @beginRescueEnsure.begin(block) end
366
+
367
+ # (see BeginRescueEnsure#ensure)
368
+ def ensure(&block) @beginRescueEnsure.ensure(block) end
369
+
370
+ # (see BeginRescueEnsure#rescue)
371
+ def rescue(error_type, &block)
372
+ @beginRescueEnsure.rescue(error_type, block)
373
+ end
374
+
375
+ private
376
+ attr_accessor :beginRescueEnsure
377
+ end
378
+
379
+ class DaemonBeginRescueEnsure < BeginRescueEnsure
380
+ def is_daemon?
381
+ true
382
+ end
383
+ end
384
+ end
385
+ end
386
+ end
@@ -0,0 +1,77 @@
1
+ ##
2
+ # Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License").
5
+ # You may not use this file except in compliance with the License.
6
+ # A copy of the License is located at
7
+ #
8
+ # http://aws.amazon.com/apache2.0
9
+ #
10
+ # or in the "license" file accompanying this file. This file is distributed
11
+ # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12
+ # express or implied. See the License for the specific language governing
13
+ # permissions and limitations under the License.
14
+ ##
15
+
16
+ # This file contains our implementation of fibers for 1.8
17
+ module AWS
18
+ module Flow
19
+ module Core
20
+ require 'fiber'
21
+ class FlowFiber < Fiber
22
+ def initialize(*args)
23
+ ObjectSpace.define_finalizer(self, self.class.finalize(self.object_id))
24
+ super(args)
25
+ end
26
+ class << self
27
+ attr_accessor :local_variables
28
+ end
29
+ @local_variables = Hash.new {|hash, key| hash[key] = {}}
30
+ def self.finalize(obj_id)
31
+ proc { FlowFiber.local_variables.delete(obj_id) }
32
+ end
33
+ def self.[](index)
34
+ self.local_variables[index]
35
+ end
36
+ def self.[]=(key, value)
37
+ self.local_variables[key] = value
38
+ end
39
+
40
+ # Will unset all the values for ancestors of this fiber, assuming that
41
+ # they have the same value for key. That is, they will unset upwards until
42
+ # the first time the value stored at key is changed
43
+ def self.unset(current_fiber, key)
44
+ current_value = FlowFiber[current_fiber.object_id][key]
45
+ parent = FlowFiber[current_fiber.object_id][:parent]
46
+ ancestor_fibers = []
47
+ while parent != nil
48
+ ancestor_fibers << parent
49
+ parent = FlowFiber[parent.object_id][:parent]
50
+ end
51
+ ancestor_fibers.each do |fiber|
52
+ FlowFiber[fiber.object_id].delete(key) if FlowFiber[fiber.object_id][key] == current_value
53
+ end
54
+ FlowFiber[current_fiber.object_id].delete(key)
55
+ end
56
+
57
+ def initialize
58
+ # Child fibers should inherit their parents FiberLocals
59
+ FlowFiber[Fiber.current.object_id].each_pair do |key, val|
60
+ FlowFiber[self.object_id][key] = val
61
+ end
62
+ FlowFiber[self.object_id][:parent] = Fiber.current
63
+ super
64
+ end
65
+
66
+ def [](key)
67
+ FlowFiber[self.object_id][key]
68
+ end
69
+ def []=(key, value)
70
+ FlowFiber[self.object_id][key] = value
71
+ end
72
+
73
+ end
74
+
75
+ end
76
+ end
77
+ end