little_monster 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. checksums.yaml +7 -0
  2. data/.codeclimate.yml +6 -0
  3. data/.gitignore +11 -0
  4. data/.rubocop.yml +34 -0
  5. data/.ruby-version +1 -0
  6. data/.travis.yml +7 -0
  7. data/Gemfile +5 -0
  8. data/Gemfile.lock +124 -0
  9. data/README.md +5 -0
  10. data/Rakefile +6 -0
  11. data/bin/console +10 -0
  12. data/bin/setup +8 -0
  13. data/exe/lm +4 -0
  14. data/lib/little_monster.rb +67 -0
  15. data/lib/little_monster/all.rb +4 -0
  16. data/lib/little_monster/config.rb +29 -0
  17. data/lib/little_monster/core.rb +20 -0
  18. data/lib/little_monster/core/api.rb +74 -0
  19. data/lib/little_monster/core/counters.rb +50 -0
  20. data/lib/little_monster/core/errors/api_unreachable_error.rb +4 -0
  21. data/lib/little_monster/core/errors/callback_failed_error.rb +4 -0
  22. data/lib/little_monster/core/errors/cancel_error.rb +4 -0
  23. data/lib/little_monster/core/errors/fatal_task_error.rb +4 -0
  24. data/lib/little_monster/core/errors/job_already_locked_error.rb +4 -0
  25. data/lib/little_monster/core/errors/job_not_found_error.rb +15 -0
  26. data/lib/little_monster/core/errors/job_retry_error.rb +4 -0
  27. data/lib/little_monster/core/errors/max_retries_error.rb +4 -0
  28. data/lib/little_monster/core/errors/task_error.rb +4 -0
  29. data/lib/little_monster/core/job.rb +188 -0
  30. data/lib/little_monster/core/job_data.rb +47 -0
  31. data/lib/little_monster/core/job_factory.rb +139 -0
  32. data/lib/little_monster/core/job_orchrestator.rb +194 -0
  33. data/lib/little_monster/core/loggable.rb +7 -0
  34. data/lib/little_monster/core/runner.rb +39 -0
  35. data/lib/little_monster/core/tagged_logger.rb +66 -0
  36. data/lib/little_monster/core/task.rb +41 -0
  37. data/lib/little_monster/generators/cli.rb +75 -0
  38. data/lib/little_monster/generators/conf_gen.rb +28 -0
  39. data/lib/little_monster/generators/generate.rb +35 -0
  40. data/lib/little_monster/generators/templates/config/application.rb +15 -0
  41. data/lib/little_monster/generators/templates/config/enviroments/development.rb +1 -0
  42. data/lib/little_monster/generators/templates/config/enviroments/production.rb +1 -0
  43. data/lib/little_monster/generators/templates/config/enviroments/test.rb +1 -0
  44. data/lib/little_monster/generators/templates/config/toiler.yml +3 -0
  45. data/lib/little_monster/generators/templates/jobs_spec_temp.erb +11 -0
  46. data/lib/little_monster/generators/templates/jobs_temp.erb +16 -0
  47. data/lib/little_monster/generators/templates/lib/.keep +0 -0
  48. data/lib/little_monster/generators/templates/log/.keep +0 -0
  49. data/lib/little_monster/generators/templates/spec_helper_temp.erb +22 -0
  50. data/lib/little_monster/generators/templates/tasks_spec_temp.erb +11 -0
  51. data/lib/little_monster/generators/templates/tasks_temp.erb +5 -0
  52. data/lib/little_monster/rspec.rb +20 -0
  53. data/lib/little_monster/rspec/helpers/job_helper.rb +61 -0
  54. data/lib/little_monster/rspec/helpers/task_helper.rb +46 -0
  55. data/lib/little_monster/rspec/matchers/have_data.rb +24 -0
  56. data/lib/little_monster/rspec/matchers/have_ended_with_status.rb +24 -0
  57. data/lib/little_monster/rspec/matchers/have_run.rb +28 -0
  58. data/lib/little_monster/rspec/matchers/have_run_task.rb +47 -0
  59. data/lib/little_monster/version.rb +3 -0
  60. data/lib/little_monster/worker.rb +27 -0
  61. data/little_monster.gemspec +48 -0
  62. metadata +343 -0
@@ -0,0 +1,50 @@
1
+ require_relative '../core/loggable'
2
+ require_relative '../core/api'
3
+
4
+ module LittleMonster::Core::Counters
5
+ def increase_counter(counter_name, unique_id, type, output = '')
6
+ begin
7
+ resp = LittleMonster::Core::API.put(
8
+ "/jobs/#{job_id}/counters/#{counter_name}",
9
+ { body: { type: type, unique_id: unique_id, output: output } },
10
+ critical: true
11
+ )
12
+ rescue LittleMonster::APIUnreachableError => e
13
+ logger.error "Could not increase counter #{counter_name}, Api unreachable"
14
+ raise e
15
+ end
16
+ raise DuplicatedCounterError if resp.code == 412
17
+ true
18
+ end
19
+
20
+ def counter(counter_name)
21
+ resp = LittleMonster::Core::API.get("/jobs/#{job_id}/counters/#{counter_name}", {}, critical: true)
22
+ raise MissedCounterError if resp.code == 404
23
+ resp.body
24
+ end
25
+
26
+ def counter_endpoint(name)
27
+ "#{LittleMonster.api_url.chomp('/')}/jobs/#{job_id}/counters/#{name}"
28
+ end
29
+
30
+ def init_counters(*counter_names)
31
+ counter_names.each do |counter_name|
32
+ resource = "/jobs/#{job_id}/counters"
33
+ values = { body: { name: counter_name } }
34
+ begin
35
+ res = LittleMonster::Core::API.post(resource, values, critical: true)
36
+ raise MissedCounterError, "Could not post to #{resource}" if !res.success? && res.code != 409 # counter already exists
37
+ rescue LittleMonster::APIUnreachableError => e
38
+ logger.error "Could not init counter #{resource} with #{values} , Api unreachable"
39
+ raise e
40
+ end
41
+ end
42
+ true
43
+ end
44
+
45
+ class MissedCounterError < StandardError
46
+ end
47
+
48
+ class DuplicatedCounterError < StandardError
49
+ end
50
+ end
@@ -0,0 +1,4 @@
1
+ module LittleMonster::Core
2
+ class APIUnreachableError < StandardError
3
+ end
4
+ end
@@ -0,0 +1,4 @@
1
+ module LittleMonster::Core
2
+ class CallbackFailedError < StandardError
3
+ end
4
+ end
@@ -0,0 +1,4 @@
1
+ module LittleMonster::Core
2
+ class CancelError < TaskError
3
+ end
4
+ end
@@ -0,0 +1,4 @@
1
+ module LittleMonster::Core
2
+ class FatalTaskError < TaskError
3
+ end
4
+ end
@@ -0,0 +1,4 @@
1
+ module LittleMonster::Core
2
+ class JobAlreadyLockedError < StandardError
3
+ end
4
+ end
@@ -0,0 +1,15 @@
1
+ module LittleMonster::Core
2
+ class JobNotFoundError < StandardError
3
+ def initialize(job_id)
4
+ params = {
5
+ body: {
6
+ status: 'error'
7
+ }
8
+ }
9
+ LittleMonster::API.put "/jobs/#{job_id}", params,
10
+ retries: LittleMonster.job_requests_retries,
11
+ retry_wait: LittleMonster.job_requests_retry_wait,
12
+ critical: true
13
+ end
14
+ end
15
+ end
@@ -0,0 +1,4 @@
1
+ module LittleMonster::Core
2
+ class JobRetryError < StandardError
3
+ end
4
+ end
@@ -0,0 +1,4 @@
1
+ module LittleMonster::Core
2
+ class MaxRetriesError < StandardError
3
+ end
4
+ end
@@ -0,0 +1,4 @@
1
+ module LittleMonster::Core
2
+ class TaskError < StandardError
3
+ end
4
+ end
@@ -0,0 +1,188 @@
1
+ module LittleMonster::Core
2
+ class Job
3
+ include Loggable
4
+
5
+ ENDED_STATUS = %i(success error cancelled).freeze
6
+ CALLBACKS = %i(on_success on_error on_cancel).freeze
7
+
8
+ class << self
9
+ def task_list(*tasks)
10
+ @tasks = *tasks
11
+ end
12
+
13
+ def retries(value)
14
+ @max_retries = value
15
+ end
16
+
17
+ def task_class_for(task_name)
18
+ "#{to_s.underscore}/#{task_name}".camelcase.constantize
19
+ end
20
+
21
+ def max_retries
22
+ @max_retries ||= -1
23
+ end
24
+
25
+ def mock!
26
+ @@mock = true
27
+ end
28
+
29
+ def tasks
30
+ @tasks ||= []
31
+ end
32
+
33
+ def mock?
34
+ @@mock ||= false
35
+ end
36
+ end
37
+
38
+ attr_accessor :id
39
+ attr_accessor :tags
40
+ attr_accessor :status
41
+
42
+ attr_accessor :retries
43
+ attr_accessor :current_action
44
+ attr_accessor :data
45
+
46
+ attr_reader :orchrestator
47
+
48
+ def initialize(options = {})
49
+ @id = options.fetch(:id, nil)
50
+ @tags = (options[:tags] || {}).freeze
51
+
52
+ @retries = options[:retries] || 0
53
+
54
+ @current_action = options.fetch(:current_action, self.class.tasks.first)
55
+
56
+ @data = if options[:data]
57
+ Data.new(self, options[:data])
58
+ else
59
+ Data.new(self)
60
+ end
61
+
62
+ @status = options.fetch(:status, :pending)
63
+
64
+ @orchrestator = Job::Orchrestator.new(self)
65
+
66
+ if mock?
67
+ @runned_tasks = {}
68
+ self.class.send :attr_reader, :runned_tasks
69
+ end
70
+
71
+ logger.default_tags = tags.merge(
72
+ id: @id,
73
+ job: self.class.to_s,
74
+ retry: @retries
75
+ )
76
+
77
+ logger.info "[type:start_job] Starting job with data: #{data.to_h[:outputs]}"
78
+ end
79
+
80
+ def run
81
+ @orchrestator.run
82
+ end
83
+
84
+ def notify_status(options = {})
85
+ params = { body: { status: @status } }
86
+ params[:body].merge!(options)
87
+
88
+ notify_job params, retries: LittleMonster.job_requests_retries,
89
+ retry_wait: LittleMonster.job_requests_retry_wait
90
+ end
91
+
92
+ def notify_task(status, options = {})
93
+ params = { body: { tasks: [{ name: @current_action, status: status }] } }
94
+ params[:body][:data] = options[:data] if options[:data]
95
+
96
+ params[:body][:tasks].first.merge!(options.except(:data))
97
+
98
+ notify_job params, retries: LittleMonster.task_requests_retries,
99
+ retry_wait: LittleMonster.task_requests_retry_wait
100
+ end
101
+
102
+ def notify_callback(status, options = {})
103
+ return true unless should_request?
104
+ params = { body: { name: @current_action, status: status } }
105
+ params[:body].merge!(options)
106
+
107
+ resp = LittleMonster::API.put "/jobs/#{id}/callbacks/#{@current_action}", params,
108
+ retries: LittleMonster.task_requests_retries,
109
+ retry_wait: LittleMonster.task_requests_retry_wait
110
+ resp.success?
111
+ end
112
+
113
+ def notify_job(params = {}, options = {})
114
+ return true unless should_request?
115
+ options[:critical] = true
116
+
117
+ params[:body][:data] = params[:body][:data].to_h if params[:body][:data]
118
+
119
+ resp = LittleMonster::API.put "/jobs/#{id}", params, options
120
+ resp.success?
121
+ end
122
+
123
+ def is_cancelled?
124
+ return false unless should_request?
125
+ resp = LittleMonster::API.get "/jobs/#{id}"
126
+
127
+ if resp.success?
128
+ resp.body[:cancel]
129
+ else
130
+ false
131
+ end
132
+ end
133
+
134
+ def task_class_for(task_name)
135
+ self.class.task_class_for task_name
136
+ end
137
+
138
+ def max_retries
139
+ self.class.max_retries
140
+ end
141
+
142
+ def retry?
143
+ max_retries == -1 || max_retries > @retries
144
+ end
145
+
146
+ def callback_to_run
147
+ case @status
148
+ when :success
149
+ :on_success
150
+ when :error
151
+ :on_error
152
+ when :cancelled
153
+ :on_cancel
154
+ end
155
+ end
156
+
157
+ # returns the tasks that will be runned for this instance
158
+ def tasks_to_run
159
+ return [] if callback_running?
160
+ task_index = self.class.tasks.find_index(@current_action)
161
+
162
+ return [] if task_index.nil?
163
+ self.class.tasks.slice(task_index..-1)
164
+ end
165
+
166
+ def callback_running?
167
+ return false if @current_action.nil? || self.class.tasks.include?(@current_action)
168
+ CALLBACKS.include? @current_action
169
+ end
170
+
171
+ def ended_status?
172
+ Job::ENDED_STATUS.include? @status
173
+ end
174
+
175
+ def mock?
176
+ self.class.mock?
177
+ end
178
+
179
+ def should_request?
180
+ !(mock? || LittleMonster.disable_requests?)
181
+ end
182
+
183
+ # callbacks definition
184
+ def on_error ; end
185
+ def on_success ; end
186
+ def on_cancel ; end
187
+ end
188
+ end
@@ -0,0 +1,47 @@
1
+ module LittleMonster::Core
2
+ class Job::Data
3
+ def initialize(job, input = {})
4
+ @outputs = input.fetch(:outputs, {})
5
+ @key_owners = input.fetch(:owners, {})
6
+ @job = job
7
+ end
8
+
9
+ def ==(other)
10
+ return false unless is_valid?(other) && other.length == length
11
+ @outputs.each { |k, v| return false unless other[k.to_sym] == v }
12
+ true
13
+ end
14
+
15
+ def [](output_key)
16
+ @outputs[output_key.to_sym]
17
+ end
18
+
19
+ def []=(output_key, value)
20
+ raise KeyError, "The key #{output_key} already exists" if @outputs.include? output_key.to_sym
21
+ @outputs[output_key.to_sym] = value
22
+
23
+ owner = @job.current_action.to_sym
24
+ @key_owners[owner] = [] unless @key_owners[owner].is_a? Array
25
+ @key_owners[owner] << output_key.to_sym
26
+ end
27
+
28
+ def to_json
29
+ MultiJson.dump(to_h)
30
+ end
31
+
32
+ def to_h
33
+ return {} if @outputs.empty?
34
+ { outputs: @outputs, owners: @key_owners }
35
+ end
36
+
37
+ def length
38
+ @outputs.length
39
+ end
40
+
41
+ private
42
+
43
+ def is_valid?(other)
44
+ other.instance_of?(Job::Data) || other.instance_of?(Hash)
45
+ end
46
+ end
47
+ end
@@ -0,0 +1,139 @@
1
+ module LittleMonster::Core
2
+ class Job::Factory
3
+ include Loggable
4
+
5
+ def initialize(message = {})
6
+ @id = message[:id]
7
+ @name = message[:name]
8
+
9
+ # it converts tags from array of hashes to a single hash
10
+ @tags = Hash[message.fetch(:tags, []).map { |h| [h.keys.first, h.values.first] }].freeze
11
+
12
+ logger.default_tags = @tags.merge(id: @id, name: @name)
13
+
14
+ @api_attributes = fetch_attributes.freeze
15
+
16
+ # this gets saved for development run and debugging purposes
17
+ @input_data = message[:data]
18
+
19
+ begin
20
+ @job_class = @name.to_s.camelcase.constantize
21
+ rescue NameError
22
+ raise JobNotFoundError.new(@id), "[type:error] job [name:#{@name}] does not exists"
23
+ end
24
+ end
25
+
26
+ def build
27
+ if discard?
28
+ logger.info "[type:discard] discarding job with [status:#{(@api_attributes || {}).fetch(:status, 'nil')}]"
29
+ return
30
+ end
31
+
32
+ unless LittleMonster.disable_requests?
33
+ notify_job_task_list
34
+ notify_job_max_retries
35
+ end
36
+
37
+ @job_class.new job_attributes
38
+ end
39
+
40
+ def notify_job_task_list
41
+ return true unless @api_attributes[:tasks].blank?
42
+
43
+ params = {
44
+ body: {
45
+ tasks: @job_class.tasks.each_with_index.map { |task, index| { name: task, order: index } }
46
+ }
47
+ }
48
+
49
+ res = LittleMonster::API.post "/jobs/#{@id}/tasks", params, retries: LittleMonster.job_requests_retries,
50
+ retry_wait: LittleMonster.job_requests_retry_wait,
51
+ critical: true
52
+ res.success?
53
+ end
54
+
55
+ def notify_job_max_retries
56
+ return true unless @api_attributes[:max_retries].blank?
57
+
58
+ params = {
59
+ body: { max_retries: @job_class.max_retries }
60
+ }
61
+
62
+ res = LittleMonster::API.put "/jobs/#{@id}", params, retries: LittleMonster.job_requests_retries,
63
+ retry_wait: LittleMonster.job_requests_retry_wait
64
+ res.success?
65
+ end
66
+
67
+ def fetch_attributes
68
+ return {} if LittleMonster.disable_requests?
69
+ resp = API.get "/jobs/#{@id}", {}, retries: LittleMonster.job_requests_retries,
70
+ retry_wait: LittleMonster.job_requests_retry_wait,
71
+ critical: true
72
+
73
+ resp.success? ? resp.body : nil
74
+ end
75
+
76
+ def calculate_status
77
+ return :pending if @api_attributes[:tasks].blank?
78
+
79
+ @api_attributes[:tasks].sort_by! { |task| task[:order] }.each do |task|
80
+ return task[:status].to_sym if task[:status].to_sym != :success
81
+ end
82
+
83
+ # check if any callback failed
84
+ @api_attributes.fetch(:callbacks, []).each do |callback|
85
+ return :error if callback[:status].to_sym == :error
86
+ end
87
+
88
+ :success
89
+ end
90
+
91
+ def find_current_action_and_retries
92
+ return [@job_class.tasks.first, 0] if @api_attributes[:tasks].blank?
93
+
94
+ # callbacks and tasks both have name, retries and status
95
+ # that means we can search through them with the same block
96
+
97
+ search_array = if @api_attributes.fetch(:callbacks, []).blank?
98
+ # callbacks have not run yet, so we look for tasks
99
+ @api_attributes[:tasks].sort_by { |task| task[:order] }
100
+ else
101
+ @api_attributes[:callbacks]
102
+ end
103
+
104
+ current = search_array.find do |x|
105
+ !Job::ENDED_STATUS.include? x[:status].to_sym
106
+ end
107
+ return nil unless current
108
+
109
+ [current[:name].to_sym, current[:retries]]
110
+ end
111
+
112
+ def job_attributes
113
+ data = if !@api_attributes[:data].nil?
114
+ @api_attributes[:data]
115
+ else
116
+ @input_data
117
+ end
118
+
119
+ attributes = {
120
+ id: @id,
121
+ data: data,
122
+ tags: @tags
123
+ }
124
+
125
+ return attributes if LittleMonster.disable_requests?
126
+
127
+ status = calculate_status
128
+ current_action, retries = find_current_action_and_retries
129
+
130
+ attributes.merge(status: status,
131
+ current_action: current_action,
132
+ retries: retries)
133
+ end
134
+
135
+ def discard?
136
+ @api_attributes.nil? || Job::ENDED_STATUS.include?(@api_attributes.fetch(:status, :pending).to_sym)
137
+ end
138
+ end
139
+ end