little_monster 0.1.3 → 0.1.5
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +15 -15
- data/lib/little_monster/core/job.rb +17 -2
- data/lib/little_monster/core/job_factory.rb +13 -9
- data/lib/little_monster/core/job_orchrestator.rb +15 -13
- data/lib/little_monster/generators/templates/config/application.rb +1 -1
- data/lib/little_monster/generators/templates/config/{enviroments → environments}/development.rb +0 -0
- data/lib/little_monster/generators/templates/config/{enviroments → environments}/production.rb +0 -0
- data/lib/little_monster/generators/templates/config/{enviroments → environments}/test.rb +0 -0
- data/lib/little_monster/generators/templates/jobs_spec_temp.erb +1 -1
- data/lib/little_monster/generators/templates/jobs_temp.erb +2 -2
- data/lib/little_monster/generators/templates/spec_helper_temp.erb +4 -5
- data/lib/little_monster/generators/templates/tasks_spec_temp.erb +1 -1
- data/lib/little_monster/version.rb +1 -1
- metadata +5 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 283b4365cbdbc25750ad3f5aba02c12ab76ea0e9
|
4
|
+
data.tar.gz: 1f844883d60194fc20cfb0c0a6bd86bdcf2f344f
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: bf94d31a48de77055bcd28297e05c4f9aa28523b04333fa0a7f2e7073204b1fcde83c15c177032ec895218137693e383b43c41e505f8936c2b0ff1ac90780c78
|
7
|
+
data.tar.gz: 4993daa4219fd9c31c9869f4228810a3e86c6e6280a92f73916a5972979b8ec087e0a2049ca5a323449ee9747a714cff934994f517542fceccac3196bd654c0e
|
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
little_monster (0.1.
|
4
|
+
little_monster (0.1.5)
|
5
5
|
activesupport
|
6
6
|
multi_json
|
7
7
|
thor
|
@@ -14,19 +14,19 @@ PATH
|
|
14
14
|
GEM
|
15
15
|
remote: https://rubygems.org/
|
16
16
|
specs:
|
17
|
-
activesupport (5.0.0)
|
17
|
+
activesupport (5.0.0.1)
|
18
18
|
concurrent-ruby (~> 1.0, >= 1.0.2)
|
19
19
|
i18n (~> 0.7)
|
20
20
|
minitest (~> 5.1)
|
21
21
|
tzinfo (~> 1.1)
|
22
22
|
addressable (2.4.0)
|
23
23
|
ast (2.3.0)
|
24
|
-
aws-sdk (2.
|
25
|
-
aws-sdk-resources (= 2.
|
26
|
-
aws-sdk-core (2.
|
24
|
+
aws-sdk (2.6.2)
|
25
|
+
aws-sdk-resources (= 2.6.2)
|
26
|
+
aws-sdk-core (2.6.2)
|
27
27
|
jmespath (~> 1.0)
|
28
|
-
aws-sdk-resources (2.
|
29
|
-
aws-sdk-core (= 2.
|
28
|
+
aws-sdk-resources (2.6.2)
|
29
|
+
aws-sdk-core (= 2.6.2)
|
30
30
|
byebug (9.0.5)
|
31
31
|
codeclimate-test-reporter (0.6.0)
|
32
32
|
simplecov (>= 0.7.1, < 1.0.0)
|
@@ -38,7 +38,7 @@ GEM
|
|
38
38
|
safe_yaml (~> 1.0.0)
|
39
39
|
diff-lcs (1.2.5)
|
40
40
|
docile (1.1.5)
|
41
|
-
ethon (0.9.
|
41
|
+
ethon (0.9.1)
|
42
42
|
ffi (>= 1.3.0)
|
43
43
|
ffi (1.9.14)
|
44
44
|
hashdiff (0.3.0)
|
@@ -48,8 +48,8 @@ GEM
|
|
48
48
|
method_source (0.8.2)
|
49
49
|
minitest (5.9.0)
|
50
50
|
multi_json (1.12.1)
|
51
|
-
oj (2.17.
|
52
|
-
parser (2.3.1.
|
51
|
+
oj (2.17.4)
|
52
|
+
parser (2.3.1.4)
|
53
53
|
ast (~> 2.2)
|
54
54
|
powerpack (0.1.1)
|
55
55
|
pry (0.10.4)
|
@@ -57,13 +57,13 @@ GEM
|
|
57
57
|
method_source (~> 0.8.1)
|
58
58
|
slop (~> 3.4)
|
59
59
|
rainbow (2.1.0)
|
60
|
-
rake (11.
|
60
|
+
rake (11.3.0)
|
61
61
|
require_all (1.3.3)
|
62
62
|
rspec (3.5.0)
|
63
63
|
rspec-core (~> 3.5.0)
|
64
64
|
rspec-expectations (~> 3.5.0)
|
65
65
|
rspec-mocks (~> 3.5.0)
|
66
|
-
rspec-core (3.5.
|
66
|
+
rspec-core (3.5.3)
|
67
67
|
rspec-support (~> 3.5.0)
|
68
68
|
rspec-expectations (3.5.0)
|
69
69
|
diff-lcs (>= 1.2.0, < 2.0)
|
@@ -72,7 +72,7 @@ GEM
|
|
72
72
|
diff-lcs (>= 1.2.0, < 2.0)
|
73
73
|
rspec-support (~> 3.5.0)
|
74
74
|
rspec-support (3.5.0)
|
75
|
-
rubocop (0.
|
75
|
+
rubocop (0.43.0)
|
76
76
|
parser (>= 2.3.1.1, < 3.0)
|
77
77
|
powerpack (~> 0.1)
|
78
78
|
rainbow (>= 1.99.1, < 3.0)
|
@@ -89,7 +89,7 @@ GEM
|
|
89
89
|
thor (0.19.1)
|
90
90
|
thread_safe (0.3.5)
|
91
91
|
tilt (2.0.5)
|
92
|
-
toiler (0.4.
|
92
|
+
toiler (0.4.2)
|
93
93
|
aws-sdk (~> 2.2, >= 2.2.10)
|
94
94
|
concurrent-ruby (~> 1.0, >= 1.0.0)
|
95
95
|
concurrent-ruby-edge (~> 0.2.0, >= 0.2.0)
|
@@ -97,7 +97,7 @@ GEM
|
|
97
97
|
ethon (>= 0.9.0)
|
98
98
|
tzinfo (1.2.2)
|
99
99
|
thread_safe (~> 0.1)
|
100
|
-
unicode-display_width (1.1.
|
100
|
+
unicode-display_width (1.1.1)
|
101
101
|
vcr (3.0.3)
|
102
102
|
webmock (2.1.0)
|
103
103
|
addressable (>= 2.3.6)
|
@@ -42,6 +42,7 @@ module LittleMonster::Core
|
|
42
42
|
attr_accessor :retries
|
43
43
|
attr_accessor :current_action
|
44
44
|
attr_accessor :data
|
45
|
+
attr_accessor :error
|
45
46
|
|
46
47
|
attr_reader :orchrestator
|
47
48
|
|
@@ -60,6 +61,7 @@ module LittleMonster::Core
|
|
60
61
|
end
|
61
62
|
|
62
63
|
@status = options.fetch(:status, :pending)
|
64
|
+
@error= options.fetch(:error, {})
|
63
65
|
|
64
66
|
@orchrestator = Job::Orchrestator.new(self)
|
65
67
|
|
@@ -91,9 +93,11 @@ module LittleMonster::Core
|
|
91
93
|
|
92
94
|
def notify_task(status, options = {})
|
93
95
|
params = { body: { tasks: [{ name: @current_action, status: status }] } }
|
96
|
+
|
94
97
|
params[:body][:data] = options[:data] if options[:data]
|
98
|
+
params[:body][:tasks].first[:exception] = serialize_error(options[:exception]) if options[:exception]
|
95
99
|
|
96
|
-
params[:body][:tasks].first.merge!(options.except(:data))
|
100
|
+
params[:body][:tasks].first.merge!(options.except(:data, :exception))
|
97
101
|
|
98
102
|
notify_job params, retries: LittleMonster.task_requests_retries,
|
99
103
|
retry_wait: LittleMonster.task_requests_retry_wait
|
@@ -102,7 +106,10 @@ module LittleMonster::Core
|
|
102
106
|
def notify_callback(status, options = {})
|
103
107
|
return true unless should_request?
|
104
108
|
params = { body: { name: @current_action, status: status } }
|
105
|
-
|
109
|
+
|
110
|
+
params[:body][:exception] = serialize_error(options[:exception]) if options[:exception]
|
111
|
+
|
112
|
+
params[:body].merge!(options.except(:exception))
|
106
113
|
|
107
114
|
resp = LittleMonster::API.put "/jobs/#{id}/callbacks/#{@current_action}", params,
|
108
115
|
retries: LittleMonster.task_requests_retries,
|
@@ -180,6 +187,14 @@ module LittleMonster::Core
|
|
180
187
|
!(mock? || LittleMonster.disable_requests?)
|
181
188
|
end
|
182
189
|
|
190
|
+
def serialize_error(error)
|
191
|
+
{
|
192
|
+
message: error.message,
|
193
|
+
type: error.class.to_s,
|
194
|
+
retry: @retries
|
195
|
+
}
|
196
|
+
end
|
197
|
+
|
183
198
|
# callbacks definition
|
184
199
|
def on_error ; end
|
185
200
|
def on_success ; end
|
@@ -73,19 +73,20 @@ module LittleMonster::Core
|
|
73
73
|
resp.success? ? resp.body : nil
|
74
74
|
end
|
75
75
|
|
76
|
-
def
|
77
|
-
return :pending if @api_attributes[:tasks].blank?
|
76
|
+
def calculate_status_and_error
|
77
|
+
return [:pending, {}] if @api_attributes[:tasks].blank?
|
78
78
|
|
79
|
-
|
80
|
-
|
79
|
+
#FIRST we check if any callback has failed to set error status
|
80
|
+
@api_attributes.fetch(:callbacks, []).each do |callback|
|
81
|
+
return [:error, callback[:exception] || {}] if callback[:status].to_sym == :error
|
81
82
|
end
|
82
83
|
|
83
|
-
#
|
84
|
-
@api_attributes.
|
85
|
-
return :
|
84
|
+
#if no callback has fail we get the status from the tasks
|
85
|
+
@api_attributes[:tasks].sort_by! { |task| task[:order] }.each do |task|
|
86
|
+
return [task[:status].to_sym, task[:exception] || {}] if task[:status].to_sym != :success
|
86
87
|
end
|
87
88
|
|
88
|
-
:success
|
89
|
+
[:success, {}]
|
89
90
|
end
|
90
91
|
|
91
92
|
def find_current_action_and_retries
|
@@ -124,11 +125,14 @@ module LittleMonster::Core
|
|
124
125
|
|
125
126
|
return attributes if LittleMonster.disable_requests?
|
126
127
|
|
127
|
-
|
128
|
+
# these two attribute retrival methods are arranged in this way
|
129
|
+
# because each one filters the tasks based on different statuses
|
130
|
+
status, error = calculate_status_and_error
|
128
131
|
current_action, retries = find_current_action_and_retries
|
129
132
|
|
130
133
|
attributes.merge(status: status,
|
131
134
|
current_action: current_action,
|
135
|
+
error: error,
|
132
136
|
retries: retries)
|
133
137
|
end
|
134
138
|
|
@@ -124,12 +124,12 @@ module LittleMonster::Core
|
|
124
124
|
|
125
125
|
# Methods that work both on tasks and callbacks
|
126
126
|
|
127
|
-
def abort_job(
|
127
|
+
def abort_job(error)
|
128
128
|
logger.debug 'notifiying abort...'
|
129
129
|
|
130
130
|
if @job.callback_running?
|
131
131
|
logger.info "[type:finish_callback] [status:error] data: #{@job.data.to_h[:outputs]}"
|
132
|
-
@job.notify_callback :error
|
132
|
+
@job.notify_callback :error, exception: error
|
133
133
|
|
134
134
|
# if callback is not on_error, raise exception to run on_error
|
135
135
|
if @job.current_action != :on_error
|
@@ -138,26 +138,28 @@ module LittleMonster::Core
|
|
138
138
|
raise CallbackFailedError, '[type:callback_fail_error]'
|
139
139
|
end
|
140
140
|
else
|
141
|
-
@job.notify_task :error
|
141
|
+
@job.notify_task :error, exception: error
|
142
142
|
logger.info "[type:finish_task] [status:error] data: #{@job.data.to_h[:outputs]}"
|
143
143
|
end
|
144
144
|
|
145
145
|
@job.status = :error
|
146
146
|
end
|
147
147
|
|
148
|
-
def handle_error(
|
149
|
-
raise
|
150
|
-
logger.error "[type:error] [error_type:#{
|
148
|
+
def handle_error(error)
|
149
|
+
raise error if LittleMonster.env.development?
|
150
|
+
logger.error "[type:error] [error_type:#{error.class}][message:#{error.message}] \n #{error.backtrace.to_a.join("\n\t")}"
|
151
151
|
|
152
|
-
|
152
|
+
@job.error = @job.serialize_error error
|
153
|
+
|
154
|
+
if error.is_a?(FatalTaskError) || error.is_a?(NameError)
|
153
155
|
logger.debug 'error is fatal, aborting run'
|
154
|
-
return abort_job(
|
156
|
+
return abort_job(error)
|
155
157
|
end
|
156
158
|
|
157
|
-
do_retry
|
159
|
+
do_retry(error)
|
158
160
|
end
|
159
161
|
|
160
|
-
def do_retry
|
162
|
+
def do_retry(error)
|
161
163
|
if @job.retry?
|
162
164
|
logger.debug "Retry ##{@job.retries} of #{@job.max_retries}"
|
163
165
|
|
@@ -165,10 +167,10 @@ module LittleMonster::Core
|
|
165
167
|
|
166
168
|
logger.debug 'notifiying retry'
|
167
169
|
if @job.callback_running?
|
168
|
-
@job.notify_callback :pending, retries: @job.retries
|
170
|
+
@job.notify_callback :pending, retries: @job.retries, exception: error
|
169
171
|
logger.info '[type:callback_retry]'
|
170
172
|
else
|
171
|
-
@job.notify_task :pending, retries: @job.retries
|
173
|
+
@job.notify_task :pending, retries: @job.retries, exception: error
|
172
174
|
logger.info '[type:task_retry]'
|
173
175
|
end
|
174
176
|
|
@@ -186,7 +188,7 @@ module LittleMonster::Core
|
|
186
188
|
end
|
187
189
|
|
188
190
|
logger.info "[type:job_max_retries] [retries:#{@job.max_retries}]"
|
189
|
-
abort_job(
|
191
|
+
abort_job(error)
|
190
192
|
end
|
191
193
|
end
|
192
194
|
end
|
@@ -8,7 +8,7 @@ LittleMonster.configure do |conf|
|
|
8
8
|
# conf.worker_queue = 'my_sqs_queue'
|
9
9
|
end
|
10
10
|
|
11
|
-
require_relative "
|
11
|
+
require_relative "environments/#{LittleMonster.env}"
|
12
12
|
|
13
13
|
Dir["#{Dir.pwd}/lib/**/*.rb"].each { |file| require_relative file }
|
14
14
|
Dir["#{Dir.pwd}/jobs/**/*.rb"].each { |file| require_relative file }
|
data/lib/little_monster/generators/templates/config/{enviroments → environments}/development.rb
RENAMED
File without changes
|
data/lib/little_monster/generators/templates/config/{enviroments → environments}/production.rb
RENAMED
File without changes
|
File without changes
|
@@ -10,13 +10,12 @@ require_rel '../tasks'
|
|
10
10
|
RSpec.configure do |conf|
|
11
11
|
conf.color = true
|
12
12
|
conf.formatter = :documentation
|
13
|
-
|
13
|
+
|
14
14
|
conf.mock_with :rspec do |mocks|
|
15
15
|
mocks.verify_partial_doubles = true
|
16
|
-
end
|
17
|
-
|
16
|
+
end
|
17
|
+
|
18
18
|
conf.before :each do
|
19
19
|
allow_any_instance_of(Kernel).to receive(:sleep)
|
20
|
-
end
|
20
|
+
end
|
21
21
|
end
|
22
|
-
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: little_monster
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- arq
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-09-22 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: activesupport
|
@@ -295,9 +295,9 @@ files:
|
|
295
295
|
- lib/little_monster/generators/conf_gen.rb
|
296
296
|
- lib/little_monster/generators/generate.rb
|
297
297
|
- lib/little_monster/generators/templates/config/application.rb
|
298
|
-
- lib/little_monster/generators/templates/config/
|
299
|
-
- lib/little_monster/generators/templates/config/
|
300
|
-
- lib/little_monster/generators/templates/config/
|
298
|
+
- lib/little_monster/generators/templates/config/environments/development.rb
|
299
|
+
- lib/little_monster/generators/templates/config/environments/production.rb
|
300
|
+
- lib/little_monster/generators/templates/config/environments/test.rb
|
301
301
|
- lib/little_monster/generators/templates/config/toiler.yml
|
302
302
|
- lib/little_monster/generators/templates/jobs_spec_temp.erb
|
303
303
|
- lib/little_monster/generators/templates/jobs_temp.erb
|