little_monster 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.codeclimate.yml +6 -0
- data/.gitignore +11 -0
- data/.rubocop.yml +34 -0
- data/.ruby-version +1 -0
- data/.travis.yml +7 -0
- data/Gemfile +5 -0
- data/Gemfile.lock +124 -0
- data/README.md +5 -0
- data/Rakefile +6 -0
- data/bin/console +10 -0
- data/bin/setup +8 -0
- data/exe/lm +4 -0
- data/lib/little_monster.rb +67 -0
- data/lib/little_monster/all.rb +4 -0
- data/lib/little_monster/config.rb +29 -0
- data/lib/little_monster/core.rb +20 -0
- data/lib/little_monster/core/api.rb +74 -0
- data/lib/little_monster/core/counters.rb +50 -0
- data/lib/little_monster/core/errors/api_unreachable_error.rb +4 -0
- data/lib/little_monster/core/errors/callback_failed_error.rb +4 -0
- data/lib/little_monster/core/errors/cancel_error.rb +4 -0
- data/lib/little_monster/core/errors/fatal_task_error.rb +4 -0
- data/lib/little_monster/core/errors/job_already_locked_error.rb +4 -0
- data/lib/little_monster/core/errors/job_not_found_error.rb +15 -0
- data/lib/little_monster/core/errors/job_retry_error.rb +4 -0
- data/lib/little_monster/core/errors/max_retries_error.rb +4 -0
- data/lib/little_monster/core/errors/task_error.rb +4 -0
- data/lib/little_monster/core/job.rb +188 -0
- data/lib/little_monster/core/job_data.rb +47 -0
- data/lib/little_monster/core/job_factory.rb +139 -0
- data/lib/little_monster/core/job_orchrestator.rb +194 -0
- data/lib/little_monster/core/loggable.rb +7 -0
- data/lib/little_monster/core/runner.rb +39 -0
- data/lib/little_monster/core/tagged_logger.rb +66 -0
- data/lib/little_monster/core/task.rb +41 -0
- data/lib/little_monster/generators/cli.rb +75 -0
- data/lib/little_monster/generators/conf_gen.rb +28 -0
- data/lib/little_monster/generators/generate.rb +35 -0
- data/lib/little_monster/generators/templates/config/application.rb +15 -0
- data/lib/little_monster/generators/templates/config/enviroments/development.rb +1 -0
- data/lib/little_monster/generators/templates/config/enviroments/production.rb +1 -0
- data/lib/little_monster/generators/templates/config/enviroments/test.rb +1 -0
- data/lib/little_monster/generators/templates/config/toiler.yml +3 -0
- data/lib/little_monster/generators/templates/jobs_spec_temp.erb +11 -0
- data/lib/little_monster/generators/templates/jobs_temp.erb +16 -0
- data/lib/little_monster/generators/templates/lib/.keep +0 -0
- data/lib/little_monster/generators/templates/log/.keep +0 -0
- data/lib/little_monster/generators/templates/spec_helper_temp.erb +22 -0
- data/lib/little_monster/generators/templates/tasks_spec_temp.erb +11 -0
- data/lib/little_monster/generators/templates/tasks_temp.erb +5 -0
- data/lib/little_monster/rspec.rb +20 -0
- data/lib/little_monster/rspec/helpers/job_helper.rb +61 -0
- data/lib/little_monster/rspec/helpers/task_helper.rb +46 -0
- data/lib/little_monster/rspec/matchers/have_data.rb +24 -0
- data/lib/little_monster/rspec/matchers/have_ended_with_status.rb +24 -0
- data/lib/little_monster/rspec/matchers/have_run.rb +28 -0
- data/lib/little_monster/rspec/matchers/have_run_task.rb +47 -0
- data/lib/little_monster/version.rb +3 -0
- data/lib/little_monster/worker.rb +27 -0
- data/little_monster.gemspec +48 -0
- metadata +343 -0
@@ -0,0 +1,50 @@
|
|
1
|
+
require_relative '../core/loggable'
|
2
|
+
require_relative '../core/api'
|
3
|
+
|
4
|
+
module LittleMonster::Core::Counters
|
5
|
+
def increase_counter(counter_name, unique_id, type, output = '')
|
6
|
+
begin
|
7
|
+
resp = LittleMonster::Core::API.put(
|
8
|
+
"/jobs/#{job_id}/counters/#{counter_name}",
|
9
|
+
{ body: { type: type, unique_id: unique_id, output: output } },
|
10
|
+
critical: true
|
11
|
+
)
|
12
|
+
rescue LittleMonster::APIUnreachableError => e
|
13
|
+
logger.error "Could not increase counter #{counter_name}, Api unreachable"
|
14
|
+
raise e
|
15
|
+
end
|
16
|
+
raise DuplicatedCounterError if resp.code == 412
|
17
|
+
true
|
18
|
+
end
|
19
|
+
|
20
|
+
def counter(counter_name)
|
21
|
+
resp = LittleMonster::Core::API.get("/jobs/#{job_id}/counters/#{counter_name}", {}, critical: true)
|
22
|
+
raise MissedCounterError if resp.code == 404
|
23
|
+
resp.body
|
24
|
+
end
|
25
|
+
|
26
|
+
def counter_endpoint(name)
|
27
|
+
"#{LittleMonster.api_url.chomp('/')}/jobs/#{job_id}/counters/#{name}"
|
28
|
+
end
|
29
|
+
|
30
|
+
def init_counters(*counter_names)
|
31
|
+
counter_names.each do |counter_name|
|
32
|
+
resource = "/jobs/#{job_id}/counters"
|
33
|
+
values = { body: { name: counter_name } }
|
34
|
+
begin
|
35
|
+
res = LittleMonster::Core::API.post(resource, values, critical: true)
|
36
|
+
raise MissedCounterError, "Could not post to #{resource}" if !res.success? && res.code != 409 # counter already exists
|
37
|
+
rescue LittleMonster::APIUnreachableError => e
|
38
|
+
logger.error "Could not init counter #{resource} with #{values} , Api unreachable"
|
39
|
+
raise e
|
40
|
+
end
|
41
|
+
end
|
42
|
+
true
|
43
|
+
end
|
44
|
+
|
45
|
+
class MissedCounterError < StandardError
|
46
|
+
end
|
47
|
+
|
48
|
+
class DuplicatedCounterError < StandardError
|
49
|
+
end
|
50
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
module LittleMonster::Core
|
2
|
+
class JobNotFoundError < StandardError
|
3
|
+
def initialize(job_id)
|
4
|
+
params = {
|
5
|
+
body: {
|
6
|
+
status: 'error'
|
7
|
+
}
|
8
|
+
}
|
9
|
+
LittleMonster::API.put "/jobs/#{job_id}", params,
|
10
|
+
retries: LittleMonster.job_requests_retries,
|
11
|
+
retry_wait: LittleMonster.job_requests_retry_wait,
|
12
|
+
critical: true
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,188 @@
|
|
1
|
+
module LittleMonster::Core
|
2
|
+
class Job
|
3
|
+
include Loggable
|
4
|
+
|
5
|
+
ENDED_STATUS = %i(success error cancelled).freeze
|
6
|
+
CALLBACKS = %i(on_success on_error on_cancel).freeze
|
7
|
+
|
8
|
+
class << self
|
9
|
+
def task_list(*tasks)
|
10
|
+
@tasks = *tasks
|
11
|
+
end
|
12
|
+
|
13
|
+
def retries(value)
|
14
|
+
@max_retries = value
|
15
|
+
end
|
16
|
+
|
17
|
+
def task_class_for(task_name)
|
18
|
+
"#{to_s.underscore}/#{task_name}".camelcase.constantize
|
19
|
+
end
|
20
|
+
|
21
|
+
def max_retries
|
22
|
+
@max_retries ||= -1
|
23
|
+
end
|
24
|
+
|
25
|
+
def mock!
|
26
|
+
@@mock = true
|
27
|
+
end
|
28
|
+
|
29
|
+
def tasks
|
30
|
+
@tasks ||= []
|
31
|
+
end
|
32
|
+
|
33
|
+
def mock?
|
34
|
+
@@mock ||= false
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
attr_accessor :id
|
39
|
+
attr_accessor :tags
|
40
|
+
attr_accessor :status
|
41
|
+
|
42
|
+
attr_accessor :retries
|
43
|
+
attr_accessor :current_action
|
44
|
+
attr_accessor :data
|
45
|
+
|
46
|
+
attr_reader :orchrestator
|
47
|
+
|
48
|
+
def initialize(options = {})
|
49
|
+
@id = options.fetch(:id, nil)
|
50
|
+
@tags = (options[:tags] || {}).freeze
|
51
|
+
|
52
|
+
@retries = options[:retries] || 0
|
53
|
+
|
54
|
+
@current_action = options.fetch(:current_action, self.class.tasks.first)
|
55
|
+
|
56
|
+
@data = if options[:data]
|
57
|
+
Data.new(self, options[:data])
|
58
|
+
else
|
59
|
+
Data.new(self)
|
60
|
+
end
|
61
|
+
|
62
|
+
@status = options.fetch(:status, :pending)
|
63
|
+
|
64
|
+
@orchrestator = Job::Orchrestator.new(self)
|
65
|
+
|
66
|
+
if mock?
|
67
|
+
@runned_tasks = {}
|
68
|
+
self.class.send :attr_reader, :runned_tasks
|
69
|
+
end
|
70
|
+
|
71
|
+
logger.default_tags = tags.merge(
|
72
|
+
id: @id,
|
73
|
+
job: self.class.to_s,
|
74
|
+
retry: @retries
|
75
|
+
)
|
76
|
+
|
77
|
+
logger.info "[type:start_job] Starting job with data: #{data.to_h[:outputs]}"
|
78
|
+
end
|
79
|
+
|
80
|
+
def run
|
81
|
+
@orchrestator.run
|
82
|
+
end
|
83
|
+
|
84
|
+
def notify_status(options = {})
|
85
|
+
params = { body: { status: @status } }
|
86
|
+
params[:body].merge!(options)
|
87
|
+
|
88
|
+
notify_job params, retries: LittleMonster.job_requests_retries,
|
89
|
+
retry_wait: LittleMonster.job_requests_retry_wait
|
90
|
+
end
|
91
|
+
|
92
|
+
def notify_task(status, options = {})
|
93
|
+
params = { body: { tasks: [{ name: @current_action, status: status }] } }
|
94
|
+
params[:body][:data] = options[:data] if options[:data]
|
95
|
+
|
96
|
+
params[:body][:tasks].first.merge!(options.except(:data))
|
97
|
+
|
98
|
+
notify_job params, retries: LittleMonster.task_requests_retries,
|
99
|
+
retry_wait: LittleMonster.task_requests_retry_wait
|
100
|
+
end
|
101
|
+
|
102
|
+
def notify_callback(status, options = {})
|
103
|
+
return true unless should_request?
|
104
|
+
params = { body: { name: @current_action, status: status } }
|
105
|
+
params[:body].merge!(options)
|
106
|
+
|
107
|
+
resp = LittleMonster::API.put "/jobs/#{id}/callbacks/#{@current_action}", params,
|
108
|
+
retries: LittleMonster.task_requests_retries,
|
109
|
+
retry_wait: LittleMonster.task_requests_retry_wait
|
110
|
+
resp.success?
|
111
|
+
end
|
112
|
+
|
113
|
+
def notify_job(params = {}, options = {})
|
114
|
+
return true unless should_request?
|
115
|
+
options[:critical] = true
|
116
|
+
|
117
|
+
params[:body][:data] = params[:body][:data].to_h if params[:body][:data]
|
118
|
+
|
119
|
+
resp = LittleMonster::API.put "/jobs/#{id}", params, options
|
120
|
+
resp.success?
|
121
|
+
end
|
122
|
+
|
123
|
+
def is_cancelled?
|
124
|
+
return false unless should_request?
|
125
|
+
resp = LittleMonster::API.get "/jobs/#{id}"
|
126
|
+
|
127
|
+
if resp.success?
|
128
|
+
resp.body[:cancel]
|
129
|
+
else
|
130
|
+
false
|
131
|
+
end
|
132
|
+
end
|
133
|
+
|
134
|
+
def task_class_for(task_name)
|
135
|
+
self.class.task_class_for task_name
|
136
|
+
end
|
137
|
+
|
138
|
+
def max_retries
|
139
|
+
self.class.max_retries
|
140
|
+
end
|
141
|
+
|
142
|
+
def retry?
|
143
|
+
max_retries == -1 || max_retries > @retries
|
144
|
+
end
|
145
|
+
|
146
|
+
def callback_to_run
|
147
|
+
case @status
|
148
|
+
when :success
|
149
|
+
:on_success
|
150
|
+
when :error
|
151
|
+
:on_error
|
152
|
+
when :cancelled
|
153
|
+
:on_cancel
|
154
|
+
end
|
155
|
+
end
|
156
|
+
|
157
|
+
# returns the tasks that will be runned for this instance
|
158
|
+
def tasks_to_run
|
159
|
+
return [] if callback_running?
|
160
|
+
task_index = self.class.tasks.find_index(@current_action)
|
161
|
+
|
162
|
+
return [] if task_index.nil?
|
163
|
+
self.class.tasks.slice(task_index..-1)
|
164
|
+
end
|
165
|
+
|
166
|
+
def callback_running?
|
167
|
+
return false if @current_action.nil? || self.class.tasks.include?(@current_action)
|
168
|
+
CALLBACKS.include? @current_action
|
169
|
+
end
|
170
|
+
|
171
|
+
def ended_status?
|
172
|
+
Job::ENDED_STATUS.include? @status
|
173
|
+
end
|
174
|
+
|
175
|
+
def mock?
|
176
|
+
self.class.mock?
|
177
|
+
end
|
178
|
+
|
179
|
+
def should_request?
|
180
|
+
!(mock? || LittleMonster.disable_requests?)
|
181
|
+
end
|
182
|
+
|
183
|
+
# callbacks definition
|
184
|
+
def on_error ; end
|
185
|
+
def on_success ; end
|
186
|
+
def on_cancel ; end
|
187
|
+
end
|
188
|
+
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
module LittleMonster::Core
|
2
|
+
class Job::Data
|
3
|
+
def initialize(job, input = {})
|
4
|
+
@outputs = input.fetch(:outputs, {})
|
5
|
+
@key_owners = input.fetch(:owners, {})
|
6
|
+
@job = job
|
7
|
+
end
|
8
|
+
|
9
|
+
def ==(other)
|
10
|
+
return false unless is_valid?(other) && other.length == length
|
11
|
+
@outputs.each { |k, v| return false unless other[k.to_sym] == v }
|
12
|
+
true
|
13
|
+
end
|
14
|
+
|
15
|
+
def [](output_key)
|
16
|
+
@outputs[output_key.to_sym]
|
17
|
+
end
|
18
|
+
|
19
|
+
def []=(output_key, value)
|
20
|
+
raise KeyError, "The key #{output_key} already exists" if @outputs.include? output_key.to_sym
|
21
|
+
@outputs[output_key.to_sym] = value
|
22
|
+
|
23
|
+
owner = @job.current_action.to_sym
|
24
|
+
@key_owners[owner] = [] unless @key_owners[owner].is_a? Array
|
25
|
+
@key_owners[owner] << output_key.to_sym
|
26
|
+
end
|
27
|
+
|
28
|
+
def to_json
|
29
|
+
MultiJson.dump(to_h)
|
30
|
+
end
|
31
|
+
|
32
|
+
def to_h
|
33
|
+
return {} if @outputs.empty?
|
34
|
+
{ outputs: @outputs, owners: @key_owners }
|
35
|
+
end
|
36
|
+
|
37
|
+
def length
|
38
|
+
@outputs.length
|
39
|
+
end
|
40
|
+
|
41
|
+
private
|
42
|
+
|
43
|
+
def is_valid?(other)
|
44
|
+
other.instance_of?(Job::Data) || other.instance_of?(Hash)
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
@@ -0,0 +1,139 @@
|
|
1
|
+
module LittleMonster::Core
|
2
|
+
class Job::Factory
|
3
|
+
include Loggable
|
4
|
+
|
5
|
+
def initialize(message = {})
|
6
|
+
@id = message[:id]
|
7
|
+
@name = message[:name]
|
8
|
+
|
9
|
+
# it converts tags from array of hashes to a single hash
|
10
|
+
@tags = Hash[message.fetch(:tags, []).map { |h| [h.keys.first, h.values.first] }].freeze
|
11
|
+
|
12
|
+
logger.default_tags = @tags.merge(id: @id, name: @name)
|
13
|
+
|
14
|
+
@api_attributes = fetch_attributes.freeze
|
15
|
+
|
16
|
+
# this gets saved for development run and debugging purposes
|
17
|
+
@input_data = message[:data]
|
18
|
+
|
19
|
+
begin
|
20
|
+
@job_class = @name.to_s.camelcase.constantize
|
21
|
+
rescue NameError
|
22
|
+
raise JobNotFoundError.new(@id), "[type:error] job [name:#{@name}] does not exists"
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def build
|
27
|
+
if discard?
|
28
|
+
logger.info "[type:discard] discarding job with [status:#{(@api_attributes || {}).fetch(:status, 'nil')}]"
|
29
|
+
return
|
30
|
+
end
|
31
|
+
|
32
|
+
unless LittleMonster.disable_requests?
|
33
|
+
notify_job_task_list
|
34
|
+
notify_job_max_retries
|
35
|
+
end
|
36
|
+
|
37
|
+
@job_class.new job_attributes
|
38
|
+
end
|
39
|
+
|
40
|
+
def notify_job_task_list
|
41
|
+
return true unless @api_attributes[:tasks].blank?
|
42
|
+
|
43
|
+
params = {
|
44
|
+
body: {
|
45
|
+
tasks: @job_class.tasks.each_with_index.map { |task, index| { name: task, order: index } }
|
46
|
+
}
|
47
|
+
}
|
48
|
+
|
49
|
+
res = LittleMonster::API.post "/jobs/#{@id}/tasks", params, retries: LittleMonster.job_requests_retries,
|
50
|
+
retry_wait: LittleMonster.job_requests_retry_wait,
|
51
|
+
critical: true
|
52
|
+
res.success?
|
53
|
+
end
|
54
|
+
|
55
|
+
def notify_job_max_retries
|
56
|
+
return true unless @api_attributes[:max_retries].blank?
|
57
|
+
|
58
|
+
params = {
|
59
|
+
body: { max_retries: @job_class.max_retries }
|
60
|
+
}
|
61
|
+
|
62
|
+
res = LittleMonster::API.put "/jobs/#{@id}", params, retries: LittleMonster.job_requests_retries,
|
63
|
+
retry_wait: LittleMonster.job_requests_retry_wait
|
64
|
+
res.success?
|
65
|
+
end
|
66
|
+
|
67
|
+
def fetch_attributes
|
68
|
+
return {} if LittleMonster.disable_requests?
|
69
|
+
resp = API.get "/jobs/#{@id}", {}, retries: LittleMonster.job_requests_retries,
|
70
|
+
retry_wait: LittleMonster.job_requests_retry_wait,
|
71
|
+
critical: true
|
72
|
+
|
73
|
+
resp.success? ? resp.body : nil
|
74
|
+
end
|
75
|
+
|
76
|
+
def calculate_status
|
77
|
+
return :pending if @api_attributes[:tasks].blank?
|
78
|
+
|
79
|
+
@api_attributes[:tasks].sort_by! { |task| task[:order] }.each do |task|
|
80
|
+
return task[:status].to_sym if task[:status].to_sym != :success
|
81
|
+
end
|
82
|
+
|
83
|
+
# check if any callback failed
|
84
|
+
@api_attributes.fetch(:callbacks, []).each do |callback|
|
85
|
+
return :error if callback[:status].to_sym == :error
|
86
|
+
end
|
87
|
+
|
88
|
+
:success
|
89
|
+
end
|
90
|
+
|
91
|
+
def find_current_action_and_retries
|
92
|
+
return [@job_class.tasks.first, 0] if @api_attributes[:tasks].blank?
|
93
|
+
|
94
|
+
# callbacks and tasks both have name, retries and status
|
95
|
+
# that means we can search through them with the same block
|
96
|
+
|
97
|
+
search_array = if @api_attributes.fetch(:callbacks, []).blank?
|
98
|
+
# callbacks have not run yet, so we look for tasks
|
99
|
+
@api_attributes[:tasks].sort_by { |task| task[:order] }
|
100
|
+
else
|
101
|
+
@api_attributes[:callbacks]
|
102
|
+
end
|
103
|
+
|
104
|
+
current = search_array.find do |x|
|
105
|
+
!Job::ENDED_STATUS.include? x[:status].to_sym
|
106
|
+
end
|
107
|
+
return nil unless current
|
108
|
+
|
109
|
+
[current[:name].to_sym, current[:retries]]
|
110
|
+
end
|
111
|
+
|
112
|
+
def job_attributes
|
113
|
+
data = if !@api_attributes[:data].nil?
|
114
|
+
@api_attributes[:data]
|
115
|
+
else
|
116
|
+
@input_data
|
117
|
+
end
|
118
|
+
|
119
|
+
attributes = {
|
120
|
+
id: @id,
|
121
|
+
data: data,
|
122
|
+
tags: @tags
|
123
|
+
}
|
124
|
+
|
125
|
+
return attributes if LittleMonster.disable_requests?
|
126
|
+
|
127
|
+
status = calculate_status
|
128
|
+
current_action, retries = find_current_action_and_retries
|
129
|
+
|
130
|
+
attributes.merge(status: status,
|
131
|
+
current_action: current_action,
|
132
|
+
retries: retries)
|
133
|
+
end
|
134
|
+
|
135
|
+
def discard?
|
136
|
+
@api_attributes.nil? || Job::ENDED_STATUS.include?(@api_attributes.fetch(:status, :pending).to_sym)
|
137
|
+
end
|
138
|
+
end
|
139
|
+
end
|