aws-flow 1.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (62) hide show
  1. data/Gemfile +8 -0
  2. data/LICENSE.TXT +15 -0
  3. data/NOTICE.TXT +14 -0
  4. data/Rakefile +39 -0
  5. data/aws-flow-core/Gemfile +9 -0
  6. data/aws-flow-core/LICENSE.TXT +15 -0
  7. data/aws-flow-core/NOTICE.TXT +14 -0
  8. data/aws-flow-core/Rakefile +27 -0
  9. data/aws-flow-core/aws-flow-core.gemspec +12 -0
  10. data/aws-flow-core/lib/aws/flow.rb +26 -0
  11. data/aws-flow-core/lib/aws/flow/async_backtrace.rb +134 -0
  12. data/aws-flow-core/lib/aws/flow/async_scope.rb +195 -0
  13. data/aws-flow-core/lib/aws/flow/begin_rescue_ensure.rb +386 -0
  14. data/aws-flow-core/lib/aws/flow/fiber.rb +77 -0
  15. data/aws-flow-core/lib/aws/flow/flow_utils.rb +50 -0
  16. data/aws-flow-core/lib/aws/flow/future.rb +109 -0
  17. data/aws-flow-core/lib/aws/flow/implementation.rb +151 -0
  18. data/aws-flow-core/lib/aws/flow/simple_dfa.rb +85 -0
  19. data/aws-flow-core/lib/aws/flow/tasks.rb +405 -0
  20. data/aws-flow-core/test/aws/async_backtrace_spec.rb +41 -0
  21. data/aws-flow-core/test/aws/async_scope_spec.rb +118 -0
  22. data/aws-flow-core/test/aws/begin_rescue_ensure_spec.rb +665 -0
  23. data/aws-flow-core/test/aws/external_task_spec.rb +197 -0
  24. data/aws-flow-core/test/aws/factories.rb +52 -0
  25. data/aws-flow-core/test/aws/fiber_condition_variable_spec.rb +163 -0
  26. data/aws-flow-core/test/aws/fiber_spec.rb +78 -0
  27. data/aws-flow-core/test/aws/flow_spec.rb +255 -0
  28. data/aws-flow-core/test/aws/future_spec.rb +210 -0
  29. data/aws-flow-core/test/aws/rubyflow.rb +22 -0
  30. data/aws-flow-core/test/aws/simple_dfa_spec.rb +63 -0
  31. data/aws-flow-core/test/aws/spec_helper.rb +36 -0
  32. data/aws-flow.gemspec +13 -0
  33. data/lib/aws/decider.rb +67 -0
  34. data/lib/aws/decider/activity.rb +408 -0
  35. data/lib/aws/decider/activity_definition.rb +111 -0
  36. data/lib/aws/decider/async_decider.rb +673 -0
  37. data/lib/aws/decider/async_retrying_executor.rb +153 -0
  38. data/lib/aws/decider/data_converter.rb +40 -0
  39. data/lib/aws/decider/decider.rb +511 -0
  40. data/lib/aws/decider/decision_context.rb +60 -0
  41. data/lib/aws/decider/exceptions.rb +178 -0
  42. data/lib/aws/decider/executor.rb +149 -0
  43. data/lib/aws/decider/flow_defaults.rb +70 -0
  44. data/lib/aws/decider/generic_client.rb +178 -0
  45. data/lib/aws/decider/history_helper.rb +173 -0
  46. data/lib/aws/decider/implementation.rb +82 -0
  47. data/lib/aws/decider/options.rb +607 -0
  48. data/lib/aws/decider/state_machines.rb +373 -0
  49. data/lib/aws/decider/task_handler.rb +76 -0
  50. data/lib/aws/decider/task_poller.rb +207 -0
  51. data/lib/aws/decider/utilities.rb +187 -0
  52. data/lib/aws/decider/worker.rb +324 -0
  53. data/lib/aws/decider/workflow_client.rb +374 -0
  54. data/lib/aws/decider/workflow_clock.rb +104 -0
  55. data/lib/aws/decider/workflow_definition.rb +101 -0
  56. data/lib/aws/decider/workflow_definition_factory.rb +53 -0
  57. data/lib/aws/decider/workflow_enabled.rb +26 -0
  58. data/test/aws/decider_spec.rb +1299 -0
  59. data/test/aws/factories.rb +45 -0
  60. data/test/aws/integration_spec.rb +3108 -0
  61. data/test/aws/spec_helper.rb +23 -0
  62. metadata +138 -0
data/Gemfile ADDED
@@ -0,0 +1,8 @@
1
+ source "http://www.rubygems.org"
2
+
3
+ gemspec
4
+
5
+ group :test do
6
+ gem "rspec", "1.3.0"
7
+ gem "rake"
8
+ end
@@ -0,0 +1,15 @@
1
+ # @markup markdown
2
+ # @title AWS Flow Framework for Ruby License
3
+
4
+ Copyright 2011-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
5
+
6
+ Licensed under the Apache License, Version 2.0 (the "License"). You
7
+ may not use this file except in compliance with the License. A copy of
8
+ the License is located at:
9
+
10
+ * <http://aws.amazon.com/apache2.0/>
11
+
12
+ or in the "LICENSE" file accompanying this file. This file is
13
+ distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
14
+ ANY KIND, either express or implied. See the License for the specific
15
+ language governing permissions and limitations under the License.
@@ -0,0 +1,14 @@
1
+ AWS Flow for Ruby
2
+ Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3
+
4
+ This product includes software developed by
5
+ Amazon Technologies, Inc (http://www.amazon.com/).
6
+
7
+ **********************
8
+ THIRD PARTY COMPONENTS
9
+ **********************
10
+ This software includes third party software subject to the following copyrights:
11
+ - XML parsing and utility functions from JetS3t - Copyright 2006-2009 James Murty.
12
+ - JSON parsing and utility functions from JSON.org - Copyright 2002 JSON.org.
13
+
14
+ The licenses for these third party components are included in LICENSE.txt
@@ -0,0 +1,39 @@
1
+ ##
2
+ # Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License").
5
+ # You may not use this file except in compliance with the License.
6
+ # A copy of the License is located at
7
+ #
8
+ # http://aws.amazon.com/apache2.0
9
+ #
10
+ # or in the "license" file accompanying this file. This file is distributed
11
+ # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12
+ # express or implied. See the License for the specific language governing
13
+ # permissions and limitations under the License.
14
+ ##
15
+
16
+ require 'spec/rake/spectask'
17
+
18
+ desc "Run unit tests in test/"
19
+ Spec::Rake::SpecTask.new(:unit_tests) do |t|
20
+ t.libs << 'lib'
21
+ t.spec_opts = ['--color', '--format nested']
22
+ t.spec_files = FileList['test/**/*.rb']
23
+ t.spec_files.delete_if {|x| x =~ /.*factories.rb/ || x =~ /.*spec_helper.rb/ || x =~ /.*integration.*/}
24
+ t.spec_files.unshift("test/aws/factories.rb")
25
+ t.spec_files.unshift("test/aws/spec_helper.rb")
26
+ end
27
+ task :test => :unit_tests
28
+
29
+ desc "Run integration tests in test/"
30
+ Spec::Rake::SpecTask.new do |t|
31
+ t.libs << 'lib'
32
+ #t.ruby_opts = ['-rspec/test/unit'] # Add this line in if you're using Test::Unit instead of RSpec
33
+ t.spec_opts = ['--color', '--format nested']
34
+ t.spec_files = FileList['test/**/*.rb']
35
+ t.spec_files.delete_if {|x| x =~ /.*factories.rb/ || x =~ /.*spec_helper.rb/}
36
+ t.spec_files.unshift("test/aws/factories.rb")
37
+ t.spec_files.unshift("test/aws/spec_helper.rb")
38
+ end
39
+ task :integration_tests => :spec
@@ -0,0 +1,9 @@
1
+ source "http://www.rubygems.org"
2
+
3
+ gemspec
4
+
5
+ group :test do
6
+ gem "rspec", "1.3.0"
7
+ gem "factory_girl"
8
+ gem "rake"
9
+ end
@@ -0,0 +1,15 @@
1
+ # @markup markdown
2
+ # @title AWS Flow Framework for Ruby License
3
+
4
+ Copyright 2011-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
5
+
6
+ Licensed under the Apache License, Version 2.0 (the "License"). You
7
+ may not use this file except in compliance with the License. A copy of
8
+ the License is located at:
9
+
10
+ * <http://aws.amazon.com/apache2.0/>
11
+
12
+ or in the "LICENSE" file accompanying this file. This file is
13
+ distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
14
+ ANY KIND, either express or implied. See the License for the specific
15
+ language governing permissions and limitations under the License.
@@ -0,0 +1,14 @@
1
+ AWS Flow for Ruby
2
+ Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3
+
4
+ This product includes software developed by
5
+ Amazon Technologies, Inc (http://www.amazon.com/).
6
+
7
+ **********************
8
+ THIRD PARTY COMPONENTS
9
+ **********************
10
+ This software includes third party software subject to the following copyrights:
11
+ - XML parsing and utility functions from JetS3t - Copyright 2006-2009 James Murty.
12
+ - JSON parsing and utility functions from JSON.org - Copyright 2002 JSON.org.
13
+
14
+ The licenses for these third party components are included in LICENSE.txt
@@ -0,0 +1,27 @@
1
+ ##
2
+ # Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License").
5
+ # You may not use this file except in compliance with the License.
6
+ # A copy of the License is located at
7
+ #
8
+ # http://aws.amazon.com/apache2.0
9
+ #
10
+ # or in the "license" file accompanying this file. This file is distributed
11
+ # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12
+ # express or implied. See the License for the specific language governing
13
+ # permissions and limitations under the License.
14
+ ##
15
+
16
+ require 'spec/rake/spectask'
17
+
18
+ Spec::Rake::SpecTask.new(:spec) do |t|
19
+ t.libs << 'lib'
20
+ t.spec_opts = ['--color', '--format nested']
21
+ t.spec_files = FileList['test/**/*.rb']
22
+ t.spec_files.delete_if {|x| x =~ /.*factories.rb/ || x =~ /.*spec_helper.rb/}
23
+ t.spec_files.unshift("test/aws/factories.rb")
24
+ t.spec_files.unshift("test/aws/spec_helper.rb")
25
+ end
26
+
27
+ task :test => :spec
@@ -0,0 +1,12 @@
1
+ Gem::Specification.new do |s|
2
+ s.name = 'aws-flow-core'
3
+ s.version = '1.0.0'
4
+ s.date = Time.now
5
+ s.summary = "AWS Flow Core"
6
+ s.description = "Library to provide all the base asynchronous constructs that aws-flow uses"
7
+ s.authors = "Michael Steger"
8
+ s.email = ""
9
+ s.files = `git ls-files`.split("\n")
10
+ s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
11
+ s.require_paths << "lib/aws/"
12
+ end
@@ -0,0 +1,26 @@
1
+ ##
2
+ # Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License").
5
+ # You may not use this file except in compliance with the License.
6
+ # A copy of the License is located at
7
+ #
8
+ # http://aws.amazon.com/apache2.0
9
+ #
10
+ # or in the "license" file accompanying this file. This file is distributed
11
+ # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12
+ # express or implied. See the License for the specific language governing
13
+ # permissions and limitations under the License.
14
+ ##
15
+
16
+ def require_all(path)
17
+ glob = File.join(File.dirname(__FILE__), path, "*.rb")
18
+ Dir[glob].each { |f| require f}
19
+ Dir[glob].map { |f| File.basename(f) }
20
+ end
21
+
22
+
23
+ # Everything depends on fiber, so we have to require that before anything else
24
+ require 'aws/flow/fiber'
25
+
26
+ $RUBY_FLOW_FILES = require_all 'flow/'
@@ -0,0 +1,134 @@
1
+ ##
2
+ # Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License").
5
+ # You may not use this file except in compliance with the License.
6
+ # A copy of the License is located at
7
+ #
8
+ # http://aws.amazon.com/apache2.0
9
+ #
10
+ # or in the "license" file accompanying this file. This file is distributed
11
+ # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12
+ # express or implied. See the License for the specific language governing
13
+ # permissions and limitations under the License.
14
+ ##
15
+
16
+ # This file contains AsyncBacktrace, which takes care of decorating and properly filtering backtraces
17
+
18
+ module AWS
19
+ module Flow
20
+ module Core
21
+ # @!visibility private
22
+ class AsyncBacktrace
23
+
24
+ # @!visibility private
25
+ def initialize(parent, backtrace)
26
+ @backtrace = AsyncBacktrace.filter(backtrace)
27
+ @parent = parent
28
+ end
29
+
30
+ # @!visibility private
31
+ def backtrace
32
+ if @parent
33
+ AsyncBacktrace.merge(@backtrace, @parent.backtrace)
34
+ else
35
+ @backtrace
36
+ end
37
+ end
38
+
39
+ # @!visibility private
40
+ class << self
41
+
42
+ # @!visibility private
43
+ def create(parent, frames_to_skip)
44
+
45
+ unless @disable_async_backtrace
46
+ b = AsyncBacktrace.caller(frames_to_skip)
47
+ AsyncBacktrace.new(parent, b)
48
+ end
49
+ end
50
+
51
+ # @!visibility private
52
+ def create_from_exception(parent, exception)
53
+ unless @disable_async_backtrace
54
+ AsyncBacktrace.new(parent, exception.backtrace);
55
+ end
56
+ end
57
+
58
+ # Remove all framework related frames after application frames. Keep framework frames before application
59
+ # frames.
60
+ #
61
+ # @todo
62
+ # The correct implementation should not have framework frames before application frames as it is expected to
63
+ # call Kernel.caller with the correct number. But in cases when due to changes this number is not correct
64
+ # the frames are kept to not create confusion.
65
+ #
66
+ def filter(backtrace)
67
+ if @disable_filtering
68
+ backtrace
69
+ else
70
+ do_filter(backtrace)
71
+ end
72
+ end
73
+
74
+ # @!visibility private
75
+ def merge(*backtraces)
76
+ result = []
77
+ backtraces.each do | b |
78
+ if b
79
+ result << "------ continuation ------" if result.size > 0
80
+ result += b
81
+ end
82
+ end
83
+ result
84
+ end
85
+
86
+ # @!visibility private
87
+ def disable_filtering
88
+ @disable_filtering = true
89
+ end
90
+
91
+ # @!visibility private
92
+ def enable_filtering
93
+ @disable_filtering = false
94
+ end
95
+
96
+ # @!visibility private
97
+ def disable
98
+ @disable_async_backtrace = true
99
+ end
100
+
101
+ # @!visibility private
102
+ def enable
103
+ @disable_async_backtrace = false
104
+ end
105
+
106
+ # @!visibility private
107
+ def caller(skip)
108
+ random_var = Kernel.caller 0
109
+ this_stuff = 1.upto(6).map { |x| Kernel.caller(x) }
110
+ other_var = Kernel.caller skip
111
+ Kernel.caller(@disable_filtering ? 0 : skip)
112
+ end
113
+
114
+ private
115
+
116
+ # @!visibility private
117
+ def do_filter(backtrace)
118
+ return nil unless backtrace
119
+ # keep asynchrony frames at the top of the backtrace only
120
+ # then cut all frames starting from asynchrony frame
121
+ skip_asynchrony_frames = false
122
+ @backtrace = backtrace.take_while do |frame|
123
+ if ! $RUBY_FLOW_FILES.select {|file| Regexp.new(file) =~ frame}.empty?
124
+ !skip_asynchrony_frames
125
+ else
126
+ skip_asynchrony_frames = true
127
+ end
128
+ end
129
+ end
130
+ end
131
+ end
132
+ end
133
+ end
134
+ end
@@ -0,0 +1,195 @@
1
+ ##
2
+ # Copyright 2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
3
+ #
4
+ # Licensed under the Apache License, Version 2.0 (the "License").
5
+ # You may not use this file except in compliance with the License.
6
+ # A copy of the License is located at
7
+ #
8
+ # http://aws.amazon.com/apache2.0
9
+ #
10
+ # or in the "license" file accompanying this file. This file is distributed
11
+ # on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12
+ # express or implied. See the License for the specific language governing
13
+ # permissions and limitations under the License.
14
+ ##
15
+
16
+ # This module contains the Root of the heirarchy for calls into flow, the AsyncScope
17
+
18
+ module AWS
19
+ module Flow
20
+ module Core
21
+
22
+ def gate_by_version(version, method, &block)
23
+ if RUBY_VERSION.send(method, version)
24
+ block.call
25
+ end
26
+ end
27
+
28
+ # @!visibility private
29
+ class AsyncScope
30
+ attr_accessor :stackTrace, :root, :failure, :root_context
31
+
32
+ def is_complete?
33
+ @root_context.complete
34
+ end
35
+
36
+ def get_closest_containing_scope
37
+ @root_error_handler
38
+ end
39
+
40
+ def cancel(error); @root_error_handler.cancel(error); end
41
+
42
+ def initialize(&block)
43
+ @root_context = RootAsyncScope.new
44
+
45
+
46
+
47
+ # 1 for the function that skips frames
48
+ # 1 for the create function
49
+ # 1 for the the initialize of the backtrace
50
+
51
+ # "./lib/aws/rubyflow/asyncBacktrace.rb:75:in `caller'"
52
+ # "./lib/aws/rubyflow/asyncBacktrace.rb:21:in `create'"
53
+ # "./lib/aws/rubyflow/asyncScope.rb:18:in `initialize'"
54
+
55
+ @root_context.backtrace = AsyncBacktrace.create(nil, 3)
56
+ @root_error_handler = BeginRescueEnsure.new(:parent => @root_context)
57
+ begin
58
+ @root_error_handler.begin lambda { block.call if ! block.nil? }
59
+ @root_error_handler.rescue(Exception, lambda { |e| raise e })
60
+ end
61
+ @root_context << @root_error_handler
62
+ end
63
+
64
+ # Collects all the heirs of a task for use in async_stack_dump
65
+ def get_heirs
66
+ @root_error_handler.get_heirs
67
+ end
68
+
69
+ # Execute all queued tasks. If execution of those tasks results in addition of new tasks to the queue, execute
70
+ # them as well.
71
+ #
72
+ # Unless there are external dependencies or bugs in the tasks to be executed, a single call to this method
73
+ # performs the complete asynchronous execution.
74
+ #
75
+ # @note In the presence of external dependencies, it is expected that {AsyncScope#eventLoop} is called every
76
+ # time after a change in the state in a dependency can unblock asynchronous execution.
77
+ #
78
+ def eventLoop
79
+ #TODO Figure out when to raise Done raise "Done" if ! @root_task.alive?
80
+ raise IllegalStateException, "Already complete" if is_complete?
81
+ @root_context.eventLoop
82
+ # TODO Does this need to be taken care of? It's supposed to protect
83
+ # against people having errors that are classes, so like, passing
84
+ # Exception into cancel. We might want to just catch that at the
85
+ # entry point
86
+ if @root_context.failure
87
+ if @root_context.failure.respond_to? :message
88
+ failure_message = @root_context.failure.message + "\n" +
89
+ @root_context.failure.backtrace.join("\n")
90
+ raise @root_context.failure, failure_message
91
+ else
92
+ raise @root_context.failure
93
+ end
94
+ end
95
+
96
+ return is_complete?
97
+ end
98
+
99
+ def <<(task)
100
+ @root_context << task
101
+ task.parent = @root_context
102
+ end
103
+ end
104
+
105
+ # @!visibility private
106
+ class RootAsyncScope < FlowFiber
107
+
108
+ attr_accessor :backtrace, :failure, :executor, :complete
109
+
110
+ def initialize(options = {}, &block)
111
+ @parent = options[:parent_context]
112
+ @daemon = options[:daemon]
113
+ @context = @parent
114
+ @executor = AsyncEventLoop.new
115
+ @task_queue = []
116
+ @complete = false
117
+ @task_queue << Task.new(context, &block) if block
118
+ end
119
+
120
+ # The only thing that should be removed from the RootAsyncScope is the
121
+ # root BeginRescueEnsure, so upon removal we are complete
122
+ def remove(task)
123
+ @complete = true
124
+ end
125
+
126
+ # As with remove, the only thing that is under RootAsyncScope should be
127
+ # the root BeginRescueEnsure, so upon failure we will be complete. Also
128
+ # sets failure variable for later raising.
129
+ def fail(task, error)
130
+ @failure = error
131
+ @complete = true
132
+ end
133
+
134
+ def <<(this_task)
135
+ @executor << this_task
136
+ end
137
+
138
+ # Reture self, a RootAsyncScope is the closest containing scope
139
+ def get_closest_containing_scope
140
+ self
141
+ end
142
+
143
+ # Call out to the AsyncEventLoop
144
+ def eventLoop
145
+ @executor.executeQueuedTasks
146
+ end
147
+
148
+
149
+ private
150
+ DELEGATED_METHODS = [:push, :<<, :enq, :empty?, :length, :size, :delete, :shift]
151
+
152
+ def method_missing(method_name, *args)
153
+ if DELEGATED_METHODS.include? method_name
154
+ @executor.send(method_name, *args)
155
+ else
156
+ super
157
+ end
158
+ end
159
+ end
160
+
161
+ # @!visibility private
162
+ class AsyncEventLoop
163
+
164
+ def initialize
165
+ @tasks = []
166
+ end
167
+
168
+ def remove(task)
169
+ @tasks.delete(task)
170
+ end
171
+ # TODO Make sure that it's okay to fail from the AsyncEventLoop, and that
172
+ # this is the correct behavior
173
+ def fail(task, error)
174
+ raise error
175
+ end
176
+ def <<(task)
177
+ @tasks << task
178
+
179
+ end
180
+
181
+
182
+ # TODO should this be synchronized somehow?
183
+
184
+ # Actually executes the eventLoop
185
+ def executeQueuedTasks
186
+ until @tasks.empty?
187
+ task = @tasks.shift
188
+ task.resume if task.alive?
189
+ end
190
+ end
191
+ end
192
+
193
+ end
194
+ end
195
+ end