aws-flow 1.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. data/Gemfile +8 -0
  2. data/LICENSE.TXT +15 -0
  3. data/NOTICE.TXT +14 -0
  4. data/Rakefile +39 -0
  5. data/aws-flow-core/Gemfile +9 -0
  6. data/aws-flow-core/LICENSE.TXT +15 -0
  7. data/aws-flow-core/NOTICE.TXT +14 -0
  8. data/aws-flow-core/Rakefile +27 -0
  9. data/aws-flow-core/aws-flow-core.gemspec +12 -0
  10. data/aws-flow-core/lib/aws/flow.rb +26 -0
  11. data/aws-flow-core/lib/aws/flow/async_backtrace.rb +134 -0
  12. data/aws-flow-core/lib/aws/flow/async_scope.rb +195 -0
  13. data/aws-flow-core/lib/aws/flow/begin_rescue_ensure.rb +386 -0
  14. data/aws-flow-core/lib/aws/flow/fiber.rb +77 -0
  15. data/aws-flow-core/lib/aws/flow/flow_utils.rb +50 -0
  16. data/aws-flow-core/lib/aws/flow/future.rb +109 -0
  17. data/aws-flow-core/lib/aws/flow/implementation.rb +151 -0
  18. data/aws-flow-core/lib/aws/flow/simple_dfa.rb +85 -0
  19. data/aws-flow-core/lib/aws/flow/tasks.rb +405 -0
  20. data/aws-flow-core/test/aws/async_backtrace_spec.rb +41 -0
  21. data/aws-flow-core/test/aws/async_scope_spec.rb +118 -0
  22. data/aws-flow-core/test/aws/begin_rescue_ensure_spec.rb +665 -0
  23. data/aws-flow-core/test/aws/external_task_spec.rb +197 -0
  24. data/aws-flow-core/test/aws/factories.rb +52 -0
  25. data/aws-flow-core/test/aws/fiber_condition_variable_spec.rb +163 -0
  26. data/aws-flow-core/test/aws/fiber_spec.rb +78 -0
  27. data/aws-flow-core/test/aws/flow_spec.rb +255 -0
  28. data/aws-flow-core/test/aws/future_spec.rb +210 -0
  29. data/aws-flow-core/test/aws/rubyflow.rb +22 -0
  30. data/aws-flow-core/test/aws/simple_dfa_spec.rb +63 -0
  31. data/aws-flow-core/test/aws/spec_helper.rb +36 -0
  32. data/aws-flow.gemspec +13 -0
  33. data/lib/aws/decider.rb +67 -0
  34. data/lib/aws/decider/activity.rb +408 -0
  35. data/lib/aws/decider/activity_definition.rb +111 -0
  36. data/lib/aws/decider/async_decider.rb +673 -0
  37. data/lib/aws/decider/async_retrying_executor.rb +153 -0
  38. data/lib/aws/decider/data_converter.rb +40 -0
  39. data/lib/aws/decider/decider.rb +511 -0
  40. data/lib/aws/decider/decision_context.rb +60 -0
  41. data/lib/aws/decider/exceptions.rb +178 -0
  42. data/lib/aws/decider/executor.rb +149 -0
  43. data/lib/aws/decider/flow_defaults.rb +70 -0
  44. data/lib/aws/decider/generic_client.rb +178 -0
  45. data/lib/aws/decider/history_helper.rb +173 -0
  46. data/lib/aws/decider/implementation.rb +82 -0
  47. data/lib/aws/decider/options.rb +607 -0
  48. data/lib/aws/decider/state_machines.rb +373 -0
  49. data/lib/aws/decider/task_handler.rb +76 -0
  50. data/lib/aws/decider/task_poller.rb +207 -0
  51. data/lib/aws/decider/utilities.rb +187 -0
  52. data/lib/aws/decider/worker.rb +324 -0
  53. data/lib/aws/decider/workflow_client.rb +374 -0
  54. data/lib/aws/decider/workflow_clock.rb +104 -0
  55. data/lib/aws/decider/workflow_definition.rb +101 -0
  56. data/lib/aws/decider/workflow_definition_factory.rb +53 -0
  57. data/lib/aws/decider/workflow_enabled.rb +26 -0
  58. data/test/aws/decider_spec.rb +1299 -0
  59. data/test/aws/factories.rb +45 -0
  60. data/test/aws/integration_spec.rb +3108 -0
  61. data/test/aws/spec_helper.rb +23 -0
  62. metadata +138 -0
@@ -0,0 +1,187 @@
1
+ #--
2
+ # Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License").
5
+ # You may not use this file except in compliance with the License.
6
+ # A copy of the License is located at
7
+ #
8
+ # http://aws.amazon.com/apache2.0
9
+ #
10
+ # or in the "license" file accompanying this file. This file is distributed
11
+ # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12
+ # express or implied. See the License for the specific language governing
13
+ # permissions and limitations under the License.
14
+ #++
15
+
16
+ require 'tmpdir'
17
+
18
+ module AWS
19
+ module Flow
20
+ # Utilities for the AWS Flow Framework for Ruby
21
+ module Utilities
22
+ # @!visibility private
23
+ class LogFactory
24
+ def self.make_logger(klass, name)
25
+ logname = "#{Dir.tmpdir}/#{klass.class.to_s}_#{name}"
26
+ logname.gsub!(/::/, '-')
27
+ log = Logger.new(logname)
28
+ log.level = Logger::DEBUG
29
+ log
30
+ end
31
+ end
32
+
33
+
34
+ # @!visibility private
35
+ def self.drill_on_future(future)
36
+ while future.get.is_a? Future
37
+ future = future.get
38
+ end
39
+ future.get
40
+ end
41
+
42
+
43
+ # @!visibility private
44
+ def self.merge_all_options(*args)
45
+ args.compact!
46
+ youngest = args.last
47
+ args.delete(youngest)
48
+ youngest.precursors.concat(args.reverse)
49
+ youngest.get_full_options
50
+ end
51
+
52
+
53
+ # @!visibility private
54
+ def self.interpret_block_for_options(option_class, block, use_defaults = false)
55
+
56
+ return option_class.new({}, use_defaults) if block.nil?
57
+ if block.arity <= 0
58
+ result = block.call
59
+ if result.is_a? Hash
60
+ options = option_class.new(result, use_defaults)
61
+ else
62
+ raise "If using 0 arguments to the option configuration, you must return a hash"
63
+ end
64
+ else
65
+ options = option_class.new({}, use_defaults)
66
+ block.call(options)
67
+ end
68
+
69
+ if options.from_class
70
+ # Insert into the next-to-last position, as these options should be used excepting where they might conflict with the options specified in the block
71
+ klass = get_const(options.from_class) rescue nil
72
+ if options.precursors.empty?
73
+ options.precursors = klass._options
74
+ else
75
+ options.precursors.insert(-2, klass._options).flatten!
76
+ end
77
+ options.prefix_name ||= options.from_class
78
+ end
79
+ options
80
+ end
81
+
82
+
83
+ class AddressableFuture
84
+
85
+ attr_accessor :return_value, :_metadata
86
+ def initialize(initial_metadata = nil)
87
+ @_metadata = initial_metadata
88
+ @return_value = Future.new
89
+ end
90
+
91
+ def metadata
92
+ @_metadata
93
+ end
94
+
95
+ def method_missing(method_name, *args, &block)
96
+ @return_value.send(method_name, *args, &block)
97
+ end
98
+ end
99
+
100
+ # @!visibility private
101
+ def self.is_external
102
+ if (defined? Fiber).nil?
103
+ return true
104
+ elsif FlowFiber.current != nil && FlowFiber.current.class != Fiber && FlowFiber.current[:decision_context] != nil
105
+ return false
106
+ end
107
+ return true
108
+ end
109
+
110
+ # @!visibility private
111
+ module SelfMethods
112
+ # @!visibility private
113
+ def handle_event(event, options)
114
+ id = options[:id_lambda].call(event) if options[:id_lambda]
115
+ id = event.attributes
116
+ options[:id_methods].each {|method| id = id.send(method)}
117
+ id = options[:id_methods].reduce(event.attributes, :send)
118
+ id = @decision_helper.send(options[:decision_helper_id])[id] if options[:decision_helper_id]
119
+ state_machine = @decision_helper[id]
120
+ state_machine.consume(options[:consume_symbol])
121
+ if options[:decision_helper_scheduled]
122
+ if state_machine.done?
123
+ scheduled_array = options[:decision_helper_scheduled]
124
+ open_request = @decision_helper.send(scheduled_array).delete(id)
125
+ else
126
+ scheduled_array = options[:decision_helper_scheduled]
127
+ open_request = @decision_helper.send(scheduled_array)[id]
128
+ end
129
+ if options[:handle_open_request]
130
+ options[:handle_open_request].call(event, open_request)
131
+ end
132
+ end
133
+ return state_machine.done?
134
+ end
135
+ end
136
+
137
+ # @!visibility private
138
+ module UpwardLookups
139
+ attr_accessor :precursors
140
+
141
+ # @!visibility private
142
+ def held_properties
143
+ precursors = self.ancestors.dup
144
+ precursors.delete(self)
145
+ result = precursors.map{|precursor| precursor.held_properties if precursor.methods.map(&:to_sym).include? :held_properties}.flatten.compact
146
+ result << @held_properties
147
+ result.flatten
148
+ end
149
+
150
+ # @!visibility private
151
+ def property(name, methods_to_prepare = [lambda(&:to_s)])
152
+ @held_properties ||= []
153
+ @held_properties << name
154
+ define_method(name) do
155
+ return_value = instance_variable_get("@#{name}")
156
+ # Make sure we correctly return false values
157
+ return_value = (look_upwards(name) || nil) if return_value.nil?
158
+ return nil if return_value.nil?
159
+ return_value = "NONE" if return_value == Float::INFINITY
160
+ methods_to_prepare.each {|method| return_value = method.call(return_value)}
161
+ return_value
162
+ end
163
+ define_method("#{name}=") do |*args|
164
+ instance_variable_set("@#{name}", args.first) unless args.first.nil?
165
+ end
166
+ end
167
+
168
+ # @!visibility private
169
+ def properties(*args)
170
+ args.each { |arg| property(arg) }
171
+ end
172
+
173
+ # @!visibility private
174
+ module InstanceMethods
175
+ attr_accessor :precursors
176
+ def look_upwards(variable)
177
+ all_precursors = @precursors.dup
178
+ all_precursors.concat self.class.default_classes
179
+ results = all_precursors.map { |x| x.send(variable) if x.methods.map(&:to_sym).include? variable }.compact
180
+ results.first
181
+ end
182
+ end
183
+ end
184
+
185
+ end
186
+ end
187
+ end
@@ -0,0 +1,324 @@
1
+ #--
2
+ # Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License").
5
+ # You may not use this file except in compliance with the License.
6
+ # A copy of the License is located at
7
+ #
8
+ # http://aws.amazon.com/apache2.0
9
+ #
10
+ # or in the "license" file accompanying this file. This file is distributed
11
+ # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12
+ # express or implied. See the License for the specific language governing
13
+ # permissions and limitations under the License.
14
+ #++
15
+
16
+ module AWS
17
+ module Flow
18
+
19
+
20
+
21
+ # A generic Activity/Workflow worker class.
22
+ class GenericWorker
23
+ # Creates a new GenericWorker
24
+ # @param service
25
+ # The AWS service class to use.
26
+ #
27
+ # @param domain
28
+ # The SWF domain to use.
29
+ #
30
+ # @param task_list_to_poll
31
+ # The list of tasks to poll for this worker.
32
+ #
33
+ # @param args
34
+ # Arguments for the workflow worker.
35
+ #
36
+ # @param [WorkerOptions] block
37
+ # A set of {WorkerOptions} for the worker.
38
+ #
39
+ def initialize(service, domain, task_list_to_poll, *args, &block)
40
+ @service = service
41
+ @domain = domain
42
+ @task_list = task_list_to_poll
43
+ @options = Utilities::interpret_block_for_options(WorkerOptions, block)
44
+ args.each { |klass_or_instance| add_implementation(klass_or_instance) } if args
45
+ end
46
+
47
+ # @!visibility private
48
+ def camel_case_to_snake_case(camel_case)
49
+ camel_case.
50
+ gsub(/(.)([A-Z])/,'\1_\2').
51
+ downcase
52
+ end
53
+
54
+ end
55
+
56
+ module GenericTypeModule
57
+ def hash
58
+ [@name.to_sym, @version].hash
59
+ end
60
+ def eql?(other)
61
+ @name.to_sym == other.name.to_sym && @version == other.version
62
+ end
63
+ end
64
+
65
+ class GenericType
66
+ include GenericTypeModule
67
+ attr_accessor :name, :version, :options
68
+ def initialize(name, version, options = {})
69
+ @name = name
70
+ @version = version
71
+ @options = options
72
+ end
73
+ end
74
+
75
+ [:ActivityType, :WorkflowType].each do |type|
76
+ klass = AWS::SimpleWorkflow.const_get(type)
77
+ klass.class_eval { include GenericTypeModule }
78
+ end
79
+
80
+
81
+
82
+ # Represents a workflow type.
83
+ class WorkflowType < GenericType; end
84
+
85
+
86
+ # Represents an activity type.
87
+ class ActivityType < GenericType; end
88
+
89
+ # This worker class is intended for use by the workflow implementation. It is configured with
90
+ # a task list and a workflow implementation. The worker class polls for decision tasks in the
91
+ # specified task list. When a decision task is received, it creates an instance of the workflow implementation and
92
+ # calls the @ execute() decorated method to process the task.
93
+ class WorkflowWorker < GenericWorker
94
+
95
+ # The workflow type for this workflow worker.
96
+ attr_accessor :workflow_type
97
+
98
+ # Creates a new WorkflowWorker instance.
99
+ #
100
+ # @param service
101
+ # The service used with this workflow worker.
102
+ #
103
+ # @param [String] domain
104
+ # The SWF domain to operate on.
105
+ #
106
+ # @param [Array] task_list
107
+ # The default task list to put all of the decision requests.
108
+ #
109
+ # @param args
110
+ # The decisions to use.
111
+ #
112
+ def initialize(service, domain, task_list, *args)
113
+ @workflow_definition_map = {}
114
+ @executor = ForkingExecutor.new(:max_workers => 2, :log_level => 5)
115
+ @workflow_type_options = []
116
+ super(service, domain, task_list, *args)
117
+ end
118
+
119
+ def set_workflow_implementation_types(workflow_implementation_types)
120
+ workflow_implementation_types.each {|type| add_workflow_implementation_type(type)}
121
+ end
122
+
123
+ def add_implementation(workflow_class)
124
+ add_workflow_implementation(workflow_class)
125
+ end
126
+
127
+ # Called by {#add_implementation}
128
+ # @!visibility private
129
+ def add_workflow_implementation(workflow_class)
130
+ workflow_class.workflows.delete_if do |workflow_type|
131
+ workflow_type.version.nil? || workflow_type.name.nil?
132
+ end
133
+ workflow_class.workflows.each do |workflow_type|
134
+ options = workflow_type.options
135
+ execution_method = options.execution_method
136
+ version = workflow_type.version
137
+ registration_options = nil
138
+ implementation_options = nil
139
+ get_state_method = workflow_class.get_state_method
140
+ signals = workflow_class.signals
141
+ @workflow_definition_map[workflow_type] = WorkflowDefinitionFactory.new(workflow_class, workflow_type, registration_options, options, execution_method, signals, get_state_method)
142
+ # TODO should probably do something like GenericWorkflowWorker#registerWorkflowTypes
143
+ workflow_hash = options.get_options([:default_task_start_to_close_timeout, :default_execution_start_to_close_timeout, :default_child_policy], {
144
+ :domain => @domain.name,
145
+ :name => workflow_type.name,
146
+ :version => version
147
+ })
148
+ if options.default_task_list
149
+ workflow_hash.merge!({:default_task_list => {:name => options.default_task_list} })
150
+ end
151
+ @workflow_type_options << workflow_hash
152
+ end
153
+ end
154
+
155
+
156
+ # Registers this workflow with Amazon SWF.
157
+ def register
158
+ @workflow_type_options.delete_if {|workflow_type_options| workflow_type_options[:version].nil?}
159
+ @workflow_type_options.each do |workflow_type_options|
160
+ begin
161
+ @service.register_workflow_type(workflow_type_options)
162
+ rescue AWS::SimpleWorkflow::Errors::TypeAlreadyExistsFault => e
163
+ # Purposefully eaten up, the alternative is to check first, and who
164
+ # wants to do two trips when one will do?
165
+ end
166
+ end
167
+ end
168
+
169
+
170
+ # Starts the workflow with a {WorkflowTaskPoller}.
171
+ #
172
+ # @param [true,false] should_register
173
+ # Indicates whether or not the workflow needs to be registered with SWF first. If {#register} was already called
174
+ # for this workflow worker, specify `false`.
175
+ #
176
+ def start(should_register = true)
177
+ # TODO check to make sure that the correct properties are set
178
+ # TODO Register the domain if not already registered
179
+ # TODO register types to poll
180
+ # TODO Set up throttler
181
+ # TODO Set up a timeout on the throttler correctly,
182
+ # TODO Make this a generic poller, go to the right kind correctly
183
+ poller = WorkflowTaskPoller.new(@service, @domain, DecisionTaskHandler.new(@workflow_definition_map, @options), @task_list, @options)
184
+ register if should_register
185
+ loop do
186
+ run_once(false, poller)
187
+ end
188
+ end
189
+
190
+ # Starts the workflow and runs it once, with an optional {WorkflowTaskPoller}.
191
+ #
192
+ # @param should_register (see #start)
193
+ #
194
+ # @param poller
195
+ # An optional {WorkflowTaskPoller} to use.
196
+ #
197
+ def run_once(should_register = false, poller = nil)
198
+ register if should_register
199
+ poller = WorkflowTaskPoller.new(@service, @domain, DecisionTaskHandler.new(@workflow_definition_map, @options), @task_list, @options) if poller.nil?
200
+ poller.poll_and_process_single_task
201
+ end
202
+ end
203
+
204
+
205
+ # For implementing activity workers, you can use the ActivityWorker class to conveniently poll a task list for
206
+ # activity tasks.
207
+ #
208
+ # You configure the activity worker with activity implementation objects. This worker class then polls for activity
209
+ # tasks in the specified task list. When an activity task is received, it looks up the appropriate implementation
210
+ # that you provided, and calls the activity method to process the task. Unlike the {WorkflowWorker}, which creates a
211
+ # new instance for every decision task, the ActivityWorker simply uses the object you provided.
212
+ #
213
+ class ActivityWorker < GenericWorker
214
+
215
+ # Creates a new ActivityWorker instance.
216
+ #
217
+ # @param service
218
+ # The SWF [Client](http://docs.aws.amazon.com/AWSRubySDK/latest/AWS/SimpleWorkflow/Client.html) used to register
219
+ # this activity worker.
220
+ #
221
+ # @param [String] domain
222
+ # The SWF [Domain](http://docs.aws.amazon.com/AWSRubySDK/latest/AWS/SimpleWorkflow/Domain.html) to operate on.
223
+ #
224
+ # @param [Array] task_list
225
+ # The default task list to put all of the activity requests.
226
+ #
227
+ # @param args
228
+ # The activities to use.
229
+ #
230
+ def initialize(service, domain, task_list, *args, &block)
231
+ @activity_definition_map = {}
232
+ @executor = ForkingExecutor.new(:max_workers => 1)
233
+ @activity_type_options = []
234
+ @options = Utilities::interpret_block_for_options(WorkerOptions, block)
235
+ super(service, domain, task_list, *args)
236
+ end
237
+
238
+ # Adds an Activity implementation to this ActivityWorker.
239
+ #
240
+ # @param [Activity] class_or_instance
241
+ # The {Activity} class or instance to add.
242
+ #
243
+ def add_implementation(class_or_instance)
244
+ add_activities_implementation(class_or_instance)
245
+ end
246
+
247
+
248
+ # Registers the activity type
249
+ def register
250
+ @activity_type_options.each do |activity_type_options|
251
+ begin
252
+ @service.register_activity_type(activity_type_options)
253
+ rescue AWS::SimpleWorkflow::Errors::TypeAlreadyExistsFault => e
254
+ previous_registration = @service.describe_activity_type(:domain => @domain.name, :activity_type => {:name => activity_type_options[:name], :version => activity_type_options[:version]})
255
+ default_options = activity_type_options.select {|key, val| key =~ /default/}
256
+ previous_keys = previous_registration["configuration"].keys.map {|x| camel_case_to_snake_case(x).to_sym}
257
+
258
+ previous_registration = Hash[previous_keys.zip(previous_registration["configuration"].values)]
259
+ if previous_registration[:default_task_list]
260
+ previous_registration[:default_task_list][:name] = previous_registration[:default_task_list].delete("name")
261
+ end
262
+ registration_difference = default_options.sort.to_a - previous_registration.sort.to_a
263
+ raise "There is a difference between the types you have registered previously and the types you are currently registering, but you haven't changed the version. These new changes will not be picked up. In particular, these options are different #{Hash[registration_difference]}" unless registration_difference.empty?
264
+ # Purposefully eaten up, the alternative is to check first, and who
265
+ # wants to do two trips when one will do?
266
+ end
267
+ end
268
+ end
269
+
270
+ # Adds an Activity implementation to this ActivityWorker.
271
+ #
272
+ # @param [Activity] class_or_instance
273
+ # The {Activity} class or instance to add.
274
+ #
275
+ def add_activities_implementation(class_or_instance)
276
+ klass = (class_or_instance.class == Class) ? class_or_instance : class_or_instance.class
277
+ instance = (class_or_instance.class == Class) ? class_or_instance.new : class_or_instance
278
+ klass.activities.each do |activity_type|
279
+
280
+ #TODO this should assign to an activityImplementation, so that we can call execute on it later
281
+ @activity_definition_map[activity_type] = ActivityDefinition.new(instance, activity_type.name.split(".").last, nil, activity_type.options, activity_type.options.data_converter)
282
+ options = activity_type.options
283
+ option_hash = {
284
+ :domain => @domain.name,
285
+ :name => activity_type.name.to_s,
286
+ :version => activity_type.version
287
+ }
288
+ option_hash.merge!(options.get_default_options)
289
+ option_hash.merge!(:default_task_list => {:name => options.default_task_list}) if options.default_task_list
290
+ @activity_type_options << option_hash
291
+ end
292
+ end
293
+
294
+
295
+ # Starts the Activity that was added to the ActivityWorker
296
+ #
297
+ # @param [true, false] should_register
298
+ # Set to false if the Activity should not register itself (it is already registered).
299
+ #
300
+ def start(should_register = true)
301
+ register if should_register
302
+ poller = ActivityTaskPoller.new(@service, @domain, @task_list, @activity_definition_map, @options)
303
+ loop do
304
+ run_once(false, poller)
305
+ end
306
+ end
307
+
308
+ # Starts the Activity that was added to the ActivityWorker and, optionally, sets the ActivityTaskPoller.
309
+ #
310
+ # @param [true, false] should_register
311
+ # Set to `false` if the Activity should not register itself (it is already registered).
312
+ #
313
+ # @param [ActivityTaskPoller] poller
314
+ # The {ActivityTaskPoller} to use. If this is not set, a default ActivityTaskPoller will be created.
315
+ #
316
+ def run_once(should_register = true, poller = nil)
317
+ register if should_register
318
+ poller = ActivityTaskPoller.new(@service, @domain, @task_list, @activity_definition_map, @options) if poller.nil?
319
+ poller.poll_and_process_single_task(@options.use_forking)
320
+ end
321
+ end
322
+
323
+ end
324
+ end