aws-flow 1.0.8 → 1.0.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. checksums.yaml +7 -0
  2. data/Gemfile +1 -1
  3. data/Rakefile +18 -31
  4. data/aws-flow.gemspec +1 -1
  5. data/lib/aws/decider.rb +1 -2
  6. data/lib/aws/decider/activity.rb +99 -53
  7. data/lib/aws/decider/activity_definition.rb +43 -7
  8. data/lib/aws/decider/async_decider.rb +56 -57
  9. data/lib/aws/decider/async_retrying_executor.rb +4 -5
  10. data/lib/aws/decider/data_converter.rb +2 -2
  11. data/lib/aws/decider/decider.rb +46 -41
  12. data/lib/aws/decider/decision_context.rb +2 -2
  13. data/lib/aws/decider/exceptions.rb +6 -6
  14. data/lib/aws/decider/executor.rb +15 -11
  15. data/lib/aws/decider/flow_defaults.rb +54 -22
  16. data/lib/aws/decider/generic_client.rb +7 -7
  17. data/lib/aws/decider/history_helper.rb +0 -0
  18. data/lib/aws/decider/implementation.rb +5 -5
  19. data/lib/aws/decider/options.rb +285 -155
  20. data/lib/aws/decider/state_machines.rb +10 -10
  21. data/lib/aws/decider/task_handler.rb +5 -5
  22. data/lib/aws/decider/task_poller.rb +152 -15
  23. data/lib/aws/decider/utilities.rb +14 -14
  24. data/lib/aws/decider/version.rb +1 -1
  25. data/lib/aws/decider/worker.rb +21 -20
  26. data/lib/aws/decider/workflow_client.rb +78 -31
  27. data/lib/aws/decider/workflow_clock.rb +1 -1
  28. data/lib/aws/decider/workflow_definition.rb +5 -5
  29. data/lib/aws/decider/workflow_definition_factory.rb +1 -1
  30. data/lib/aws/decider/workflow_enabled.rb +1 -1
  31. data/lib/aws/flow/async_backtrace.rb +19 -18
  32. data/lib/aws/flow/async_scope.rb +32 -16
  33. data/lib/aws/flow/begin_rescue_ensure.rb +61 -56
  34. data/lib/aws/flow/fiber.rb +14 -6
  35. data/lib/aws/flow/flow_utils.rb +9 -6
  36. data/lib/aws/flow/future.rb +43 -18
  37. data/lib/aws/flow/implementation.rb +34 -18
  38. data/lib/aws/flow/simple_dfa.rb +12 -4
  39. data/lib/aws/flow/tasks.rb +120 -86
  40. data/{test/aws → spec/aws/integration}/integration_spec.rb +0 -0
  41. data/{test/aws → spec/aws/unit}/async_backtrace_spec.rb +1 -0
  42. data/{test/aws → spec/aws/unit}/async_scope_spec.rb +0 -0
  43. data/{test/aws → spec/aws/unit}/begin_rescue_ensure_spec.rb +90 -2
  44. data/{test/aws → spec/aws/unit}/decider_spec.rb +41 -53
  45. data/{test/aws → spec/aws/unit}/external_task_spec.rb +0 -0
  46. data/{test/aws → spec/aws/unit}/factories.rb +0 -0
  47. data/{test/aws → spec/aws/unit}/fiber_condition_variable_spec.rb +0 -0
  48. data/{test/aws → spec/aws/unit}/fiber_spec.rb +0 -0
  49. data/{test/aws → spec/aws/unit}/flow_spec.rb +0 -0
  50. data/{test/aws → spec/aws/unit}/future_spec.rb +0 -0
  51. data/{test/aws → spec/aws/unit}/preinclude_tests.rb +0 -0
  52. data/{test/aws → spec/aws/unit}/rubyflow.rb +0 -0
  53. data/{test/aws → spec/aws/unit}/simple_dfa_spec.rb +0 -0
  54. data/{test/aws → spec}/spec_helper.rb +0 -0
  55. metadata +30 -30
@@ -16,7 +16,7 @@
16
16
  module AWS
17
17
  module Flow
18
18
 
19
- # @!visibility private
19
+ # @api private
20
20
  module DecisionStateMachineDFA
21
21
  attr_accessor :transitions, :symbols, :states, :start_state
22
22
 
@@ -40,7 +40,7 @@ module AWS
40
40
  end
41
41
 
42
42
  def get_transitions
43
- # Turns out, you are your own ancestor
43
+ # Turns out, you are your own ancestor.
44
44
  ancestors.slice(1..-1).map {|x| x.transitions if x.respond_to? :transitions}.compact.
45
45
  inject({}) {|x, y| x.merge(y)}.merge(@transitions)
46
46
  end
@@ -77,7 +77,7 @@ module AWS
77
77
  end
78
78
  end
79
79
 
80
- # @!visibility private
80
+ # @api private
81
81
  class CompleteWorkflowStateMachine
82
82
  extend DecisionStateMachineDFA
83
83
  attr_reader :id
@@ -86,10 +86,10 @@ module AWS
86
86
  return if symbol == :handle_decision_task_started_event
87
87
  raise "UnsupportedOperation"
88
88
  end
89
- # Creates a new CompleteWorkflowStateMachine
89
+ # Creates a new `CompleteWorkflowStateMachine`.
90
90
  #
91
91
  # @param id
92
- # The decider id.
92
+ # The decider ID.
93
93
  #
94
94
  # @param attributes
95
95
  #
@@ -110,7 +110,7 @@ module AWS
110
110
  end
111
111
 
112
112
 
113
- # @!visibility private
113
+ # @api private
114
114
  class DecisionStateMachineBase
115
115
  extend DecisionStateMachineDFA
116
116
  attr_reader :id
@@ -152,7 +152,7 @@ module AWS
152
152
  end
153
153
 
154
154
 
155
- # @!visibility private
155
+ # @api private
156
156
  class ActivityDecisionStateMachine < DecisionStateMachineBase
157
157
 
158
158
  attr_reader :attributes
@@ -207,7 +207,7 @@ module AWS
207
207
  end
208
208
 
209
209
 
210
- # @!visibility private
210
+ # @api private
211
211
  class TimerDecisionStateMachine < DecisionStateMachineBase
212
212
  attr_accessor :cancelled
213
213
  def initialize(decision_id, attributes)
@@ -257,7 +257,7 @@ module AWS
257
257
  end
258
258
 
259
259
 
260
- # @!visibility private
260
+ # @api private
261
261
  class SignalDecisionStateMachine < DecisionStateMachineBase
262
262
  def initialize(decision_id, attributes)
263
263
  @attributes = attributes
@@ -305,7 +305,7 @@ module AWS
305
305
  end
306
306
 
307
307
 
308
- # @!visibility private
308
+ # @api private
309
309
  class ChildWorkflowDecisionStateMachine < DecisionStateMachineBase
310
310
  attr_accessor :run_id, :attributes
311
311
  def initialize(decision_id, attributes)
@@ -17,11 +17,11 @@ module AWS
17
17
  module Flow
18
18
 
19
19
 
20
- # A decision task handler to work with a {WorkflowTaskPoller}. Create a DecisionTaskHandler and pass it to
21
- # WorkflowTaskPoller on {WorkflowTaskPoller#initialize construction}.
20
+ # A decision task handler to work with a {WorkflowTaskPoller}. Create a `DecisionTaskHandler` and pass it to
21
+ # {WorkflowTaskPoller} on {WorkflowTaskPoller#initialize construction}.
22
22
  class DecisionTaskHandler
23
23
 
24
- # Creates a new DecisionTaskHandler
24
+ # Creates a new `DecisionTaskHandler`.
25
25
  #
26
26
  # @param workflow_definition_map
27
27
  #
@@ -35,7 +35,7 @@ module AWS
35
35
  end
36
36
 
37
37
 
38
- # Handles a decision task
38
+ # Handles a decision task.
39
39
  #
40
40
  # @param decision_task_iterator
41
41
  #
@@ -58,7 +58,7 @@ module AWS
58
58
  #
59
59
  # @param history_helper
60
60
  #
61
- # @return [AsyncDecider] the created AsyncDecider.
61
+ # @return [AsyncDecider] The created {AsyncDecider}.
62
62
  #
63
63
  def create_async_decider(history_helper)
64
64
  task = history_helper.get_decision_task
@@ -21,7 +21,7 @@ module AWS
21
21
  class WorkflowTaskPoller
22
22
 
23
23
 
24
- # Creates a new WorkflowTaskPoller
24
+ # Creates a new `WorkflowTaskPoller`.
25
25
  #
26
26
  # @param service
27
27
  # The Amazon SWF service object on which this task poller will operate.
@@ -72,15 +72,46 @@ module AWS
72
72
  end
73
73
  @service.respond_decision_task_completed(task_completed_request)
74
74
  rescue AWS::SimpleWorkflow::Errors::UnknownResourceFault => e
75
- @logger.debug "Error in the poller, #{e}"
76
- @logger.debug "The error class in #{e.class}"
75
+ @logger.error "Error in the poller, #{e}"
76
+ @logger.error "The error class in #{e.class}"
77
77
  rescue Exception => e
78
- @logger.debug "Error in the poller, #{e}"
78
+ @logger.error "Error in the poller, #{e}"
79
79
  end
80
80
  end
81
81
  end
82
82
 
83
+ # A poller for activity tasks.
84
+ #
83
85
  class ActivityTaskPoller
86
+
87
+ # Initializes a new `ActivityTaskPoller`.
88
+ #
89
+ # @param service
90
+ # *Required*. The AWS::SimpleWorkflow instance to use.
91
+ #
92
+ # @param domain
93
+ # *Required*. The domain used by the workflow.
94
+ #
95
+ # @param task_list
96
+ # *Required*. The task list used to poll for activity tasks.
97
+ #
98
+ # @param activity_definition_map
99
+ # *Required*. The {ActivityDefinition} instance that implements the
100
+ # activity to run. This map is in the form:
101
+ #
102
+ # { :activity_type => 'activity_definition_name' }
103
+ #
104
+ # The named activity definition will be run when the {#execute} method
105
+ # is called.
106
+ #
107
+ # @param options
108
+ # *Optional*. Options to set for the activity poller. You can set the
109
+ # following options:
110
+ #
111
+ # * `logger` - The logger to use.
112
+ # * `max_workers` - The maximum number of workers that can be running at
113
+ # once. The default is 20.
114
+ #
84
115
  def initialize(service, domain, task_list, activity_definition_map, executor, options=nil)
85
116
  @service = service
86
117
  @domain = domain
@@ -91,6 +122,13 @@ module AWS
91
122
  @executor = executor
92
123
  end
93
124
 
125
+ # Executes the specified activity task.
126
+ #
127
+ # @param task
128
+ # *Required*. The
129
+ # [AWS::SimpleWorkflow::ActivityTask](http://docs.aws.amazon.com/AWSRubySDK/latest/AWS/SimpleWorkflow/ActivityTask.html)
130
+ # object to run.
131
+ #
94
132
  def execute(task)
95
133
  activity_type = task.activity_type
96
134
  begin
@@ -106,13 +144,34 @@ module AWS
106
144
  @service.respond_activity_task_completed(:task_token => task.task_token, :result => output)
107
145
  end
108
146
  rescue ActivityFailureException => e
109
- @logger.debug "The activity failed, with original output of #{original_result} and dataconverted result of #{output}. aws-flow will now attempt to fail it."
147
+ @logger.error "The activity failed, with original output of #{original_result} and dataconverted result of #{output}. aws-flow will now attempt to fail it."
110
148
  respond_activity_task_failed_with_retry(task.task_token, e.message, e.details)
111
149
 
112
150
  end
113
151
  #TODO all the completion stuffs
114
152
  end
115
153
 
154
+ # Responds to the decider that the activity task has failed, and attempts
155
+ # to retry the task.
156
+ #
157
+ # @note Retry behavior for this method is currently *not implemented*. For
158
+ # now, it simply wraps {#respond_activity_task_failed}.
159
+ #
160
+ # @param task_token
161
+ # *Required*. The task token from the {ActivityDefinition} object to
162
+ # retry. The task token is generated by the service and should be
163
+ # treated as an opaque value.
164
+ #
165
+ # @param reason
166
+ # *Required*. Description of the error that may assist in diagnostics.
167
+ # Although this value is *required*, you can set it to an empty string
168
+ # if you don't need this information.
169
+ #
170
+ # @param details
171
+ # *Required*. Detailed information about the failure. Although this
172
+ # value is *required*, you can set it to an empty string if you don't
173
+ # need this information.
174
+ #
116
175
  def respond_activity_task_failed_with_retry(task_token, reason, details)
117
176
  #TODO Set up this variable
118
177
  if @failure_retrier.nil?
@@ -121,6 +180,20 @@ module AWS
121
180
  end
122
181
  end
123
182
 
183
+ # Responds to the decider that the activity task should be canceled, and
184
+ # attempts to retry the task.
185
+ #
186
+ # @note Retry behavior for this method is currently *not implemented*. For
187
+ # now, it simply wraps {#respond_activity_task_canceled}.
188
+ #
189
+ # @param task_token
190
+ # *Required*. The task token from the {ActivityDefinition} object to
191
+ # retry.
192
+ #
193
+ # @param message
194
+ # *Required*. A message that provides detail about why the activity task
195
+ # is cancelled.
196
+ #
124
197
  def respond_activity_task_canceled_with_retry(task_token, message)
125
198
  if @failure_retrier.nil?
126
199
  respond_activity_task_canceled(task_token, message)
@@ -128,20 +201,68 @@ module AWS
128
201
  #TODO Set up other stuff to do if we have it
129
202
  end
130
203
 
204
+ # Responds to the decider that the activity task should be canceled. No
205
+ # retry is attempted.
206
+ #
207
+ # @param task_token
208
+ # *Required*. The task token from the {ActivityDefinition} object to
209
+ # retry.
210
+ #
211
+ # @param message
212
+ # *Required*. A message that provides detail about why the activity task
213
+ # is cancelled.
214
+ #
131
215
  def respond_activity_task_canceled(task_token, message)
132
216
  @service.respond_activity_task_canceled({:task_token => task_token, :details => message})
133
217
  end
134
218
 
219
+ # Responds to the decider that the activity task has failed. No retry is
220
+ # attempted.
221
+ #
222
+ # @param task_token
223
+ # *Required*. The task token from the {ActivityDefinition} object to
224
+ # retry. The task token is generated by the service and should be
225
+ # treated as an opaque value.
226
+ #
227
+ # @param reason
228
+ # *Required*. Description of the error that may assist in diagnostics.
229
+ # Although this value is *required*, you can set it to an empty string
230
+ # if you don't need this information.
231
+ #
232
+ # @param details
233
+ # *Required*. Detailed information about the failure. Although this
234
+ # value is *required*, you can set it to an empty string if you don't
235
+ # need this information.
236
+ #
135
237
  def respond_activity_task_failed(task_token, reason, details)
136
238
  @logger.debug "The task token to be reported on is #{task_token}"
137
239
  @service.respond_activity_task_failed(:task_token => task_token, :reason => reason.to_s, :details => details.to_s)
138
240
  end
139
241
 
242
+ # Processes the specified activity task.
243
+ #
244
+ # @param task
245
+ # *Required*. The
246
+ # [AWS::SimpleWorkflow::ActivityTask](http://docs.aws.amazon.com/AWSRubySDK/latest/AWS/SimpleWorkflow/ActivityTask.html)
247
+ # object to process.
248
+ #
140
249
  def process_single_task(task)
141
- previous_config = @service.config.to_h
142
- previous_config.delete(:http_handler)
143
- @service = AWS::SimpleWorkflow.new(previous_config).client
144
- @service = @service.with_http_handler(AWS::Core::Http::NetHttpHandler.new(previous_config))
250
+
251
+ # We are using the 'build' method to create a new ConnectionPool here to
252
+ # make sure that connection pools are not shared among forked processes.
253
+ # The default behavior of the ConnectionPool class is to cache a pool
254
+ # for a set of options created by the 'new' method and always use the
255
+ # same pool for the same set of options. This is undesirable when
256
+ # multiple processes want to use different connection pools with same
257
+ # options as is the case here.
258
+ # Since we can't change the pool of an already existing NetHttpHandler,
259
+ # we also create a new NetHttpHandler in order to use the new pool.
260
+
261
+ options = @service.config.to_h
262
+ options[:connection_pool] = AWS::Core::Http::ConnectionPool.build(options[:http_handler].pool.options)
263
+ options[:http_handler] = AWS::Core::Http::NetHttpHandler.new(options)
264
+ @service = AWS::SimpleWorkflow.new(options).client
265
+
145
266
  begin
146
267
  begin
147
268
  execute(task)
@@ -157,13 +278,24 @@ module AWS
157
278
  end
158
279
  rescue Exception => e
159
280
  semaphore_needs_release = true
160
- @logger.debug "Got into the other error mode"
281
+ @logger.error "Got into the other error mode"
161
282
  raise e
162
283
  ensure
163
284
  @poll_semaphore.release if semaphore_needs_release
164
285
  end
165
286
  end
166
287
 
288
+ # Polls the task list for a new activity task, and executes it if one is
289
+ # found.
290
+ #
291
+ # If `use_forking` is set to `true` and the maximum number of workers (as
292
+ # set in {#initialize}) are already executing, this method will block
293
+ # until the number of running workers is less than the maximum.
294
+ #
295
+ # @param use_forking
296
+ # *Optional*. Whether to use forking to execute the task. On Windows,
297
+ # you should set this to `false`.
298
+ #
167
299
  def poll_and_process_single_task(use_forking = true)
168
300
  @poll_semaphore ||= SuspendableSemaphore.new
169
301
  @poll_semaphore.acquire
@@ -171,16 +303,17 @@ module AWS
171
303
  @logger.debug "before the poll\n\n"
172
304
  # This is to warm the lazily loaded clients in the @service, so we don't
173
305
  # pay for their loading in every forked client
174
- @service.config.to_h
175
306
  begin
176
307
  if use_forking
177
308
  @executor.block_on_max_workers
178
309
  end
179
310
  task = @domain.activity_tasks.poll_for_single_task(@task_list)
180
- @logger.error "got a task, #{task.activity_type.name}"
181
- @logger.error "The task token I got was: #{task.task_token}"
311
+ if task
312
+ @logger.info "got a task, #{task.activity_type.name}"
313
+ @logger.info "The task token I got was: #{task.task_token}"
314
+ end
182
315
  rescue Exception => e
183
- @logger.debug "I have not been able to poll successfully, and am now bailing out, with error #{e}"
316
+ @logger.error "I have not been able to poll successfully, and am now bailing out, with error #{e}"
184
317
  @poll_semaphore.release
185
318
  return false
186
319
  end
@@ -202,15 +335,19 @@ module AWS
202
335
  end
203
336
  end
204
337
 
338
+ # @api private
339
+ # @note This class is currently not implemented.
205
340
  class SuspendableSemaphore
206
341
 
342
+ # @note This method is not implemented.
207
343
  def initialize
208
-
209
344
  end
210
345
 
346
+ # @note This method is not implemented.
211
347
  def acquire
212
348
  end
213
349
 
350
+ # @note This method is not implemented.
214
351
  def release
215
352
  end
216
353
  end
@@ -17,9 +17,9 @@ require 'tmpdir'
17
17
 
18
18
  module AWS
19
19
  module Flow
20
- # Utilities for the AWS Flow Framework for Ruby
20
+ # Utilities for the AWS Flow Framework for Ruby.
21
21
  module Utilities
22
- # @!visibility private
22
+ # @api private
23
23
  class LogFactory
24
24
  def self.make_logger(klass, name)
25
25
  logname = "#{Dir.tmpdir}/#{klass.class.to_s}_#{name}"
@@ -31,7 +31,7 @@ module AWS
31
31
  end
32
32
 
33
33
 
34
- # @!visibility private
34
+ # @api private
35
35
  def self.drill_on_future(future)
36
36
  while (future.respond_to? :is_flow_future?) && future.is_flow_future?
37
37
  future = future.get
@@ -39,7 +39,7 @@ module AWS
39
39
  future
40
40
  end
41
41
 
42
- # @!visibility private
42
+ # @api private
43
43
  def self.merge_all_options(*args)
44
44
  args.compact!
45
45
  youngest = args.last
@@ -49,7 +49,7 @@ module AWS
49
49
  end
50
50
 
51
51
 
52
- # @!visibility private
52
+ # @api private
53
53
  def self.interpret_block_for_options(option_class, block, use_defaults = false)
54
54
 
55
55
  return option_class.new({}, use_defaults) if block.nil?
@@ -87,7 +87,7 @@ module AWS
87
87
  @return_value = Future.new
88
88
  end
89
89
 
90
- # determines whether the object is a flow future. The contract is that
90
+ # Determines whether the object is a flow future. The contract is that
91
91
  # flow futures must have a #get method.
92
92
  def is_flow_future?
93
93
  true
@@ -102,7 +102,7 @@ module AWS
102
102
  end
103
103
  end
104
104
 
105
- # @!visibility private
105
+ # @api private
106
106
  def self.is_external
107
107
  if (defined? Fiber).nil?
108
108
  return true
@@ -112,9 +112,9 @@ module AWS
112
112
  return true
113
113
  end
114
114
 
115
- # @!visibility private
115
+ # @api private
116
116
  module SelfMethods
117
- # @!visibility private
117
+ # @api private
118
118
  def handle_event(event, options)
119
119
  id = options[:id_lambda].call(event) if options[:id_lambda]
120
120
  id = event.attributes
@@ -139,11 +139,11 @@ module AWS
139
139
  end
140
140
  end
141
141
 
142
- # @!visibility private
142
+ # @api private
143
143
  module UpwardLookups
144
144
  attr_accessor :precursors
145
145
 
146
- # @!visibility private
146
+ # @api private
147
147
  def held_properties
148
148
  precursors = self.ancestors.dup
149
149
  precursors.delete(self)
@@ -152,7 +152,7 @@ module AWS
152
152
  result.flatten
153
153
  end
154
154
 
155
- # @!visibility private
155
+ # @api private
156
156
  def property(name, methods_to_prepare = [lambda(&:to_s)])
157
157
  @held_properties ||= []
158
158
  @held_properties << name
@@ -170,12 +170,12 @@ module AWS
170
170
  end
171
171
  end
172
172
 
173
- # @!visibility private
173
+ # @api private
174
174
  def properties(*args)
175
175
  args.each { |arg| property(arg) }
176
176
  end
177
177
 
178
- # @!visibility private
178
+ # @api private
179
179
  module InstanceMethods
180
180
  attr_accessor :precursors
181
181
  def look_upwards(variable)