little_monster 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. checksums.yaml +7 -0
  2. data/.codeclimate.yml +6 -0
  3. data/.gitignore +11 -0
  4. data/.rubocop.yml +34 -0
  5. data/.ruby-version +1 -0
  6. data/.travis.yml +7 -0
  7. data/Gemfile +5 -0
  8. data/Gemfile.lock +124 -0
  9. data/README.md +5 -0
  10. data/Rakefile +6 -0
  11. data/bin/console +10 -0
  12. data/bin/setup +8 -0
  13. data/exe/lm +4 -0
  14. data/lib/little_monster.rb +67 -0
  15. data/lib/little_monster/all.rb +4 -0
  16. data/lib/little_monster/config.rb +29 -0
  17. data/lib/little_monster/core.rb +20 -0
  18. data/lib/little_monster/core/api.rb +74 -0
  19. data/lib/little_monster/core/counters.rb +50 -0
  20. data/lib/little_monster/core/errors/api_unreachable_error.rb +4 -0
  21. data/lib/little_monster/core/errors/callback_failed_error.rb +4 -0
  22. data/lib/little_monster/core/errors/cancel_error.rb +4 -0
  23. data/lib/little_monster/core/errors/fatal_task_error.rb +4 -0
  24. data/lib/little_monster/core/errors/job_already_locked_error.rb +4 -0
  25. data/lib/little_monster/core/errors/job_not_found_error.rb +15 -0
  26. data/lib/little_monster/core/errors/job_retry_error.rb +4 -0
  27. data/lib/little_monster/core/errors/max_retries_error.rb +4 -0
  28. data/lib/little_monster/core/errors/task_error.rb +4 -0
  29. data/lib/little_monster/core/job.rb +188 -0
  30. data/lib/little_monster/core/job_data.rb +47 -0
  31. data/lib/little_monster/core/job_factory.rb +139 -0
  32. data/lib/little_monster/core/job_orchrestator.rb +194 -0
  33. data/lib/little_monster/core/loggable.rb +7 -0
  34. data/lib/little_monster/core/runner.rb +39 -0
  35. data/lib/little_monster/core/tagged_logger.rb +66 -0
  36. data/lib/little_monster/core/task.rb +41 -0
  37. data/lib/little_monster/generators/cli.rb +75 -0
  38. data/lib/little_monster/generators/conf_gen.rb +28 -0
  39. data/lib/little_monster/generators/generate.rb +35 -0
  40. data/lib/little_monster/generators/templates/config/application.rb +15 -0
  41. data/lib/little_monster/generators/templates/config/enviroments/development.rb +1 -0
  42. data/lib/little_monster/generators/templates/config/enviroments/production.rb +1 -0
  43. data/lib/little_monster/generators/templates/config/enviroments/test.rb +1 -0
  44. data/lib/little_monster/generators/templates/config/toiler.yml +3 -0
  45. data/lib/little_monster/generators/templates/jobs_spec_temp.erb +11 -0
  46. data/lib/little_monster/generators/templates/jobs_temp.erb +16 -0
  47. data/lib/little_monster/generators/templates/lib/.keep +0 -0
  48. data/lib/little_monster/generators/templates/log/.keep +0 -0
  49. data/lib/little_monster/generators/templates/spec_helper_temp.erb +22 -0
  50. data/lib/little_monster/generators/templates/tasks_spec_temp.erb +11 -0
  51. data/lib/little_monster/generators/templates/tasks_temp.erb +5 -0
  52. data/lib/little_monster/rspec.rb +20 -0
  53. data/lib/little_monster/rspec/helpers/job_helper.rb +61 -0
  54. data/lib/little_monster/rspec/helpers/task_helper.rb +46 -0
  55. data/lib/little_monster/rspec/matchers/have_data.rb +24 -0
  56. data/lib/little_monster/rspec/matchers/have_ended_with_status.rb +24 -0
  57. data/lib/little_monster/rspec/matchers/have_run.rb +28 -0
  58. data/lib/little_monster/rspec/matchers/have_run_task.rb +47 -0
  59. data/lib/little_monster/version.rb +3 -0
  60. data/lib/little_monster/worker.rb +27 -0
  61. data/little_monster.gemspec +48 -0
  62. metadata +343 -0
@@ -0,0 +1,194 @@
1
+ # TODO : don't send data on callback fail
2
+ module LittleMonster::Core
3
+ class Job::Orchrestator
4
+ attr_reader :logger
5
+ attr_reader :job
6
+
7
+ def initialize(job)
8
+ @job = job
9
+ @logger = @job.logger
10
+ end
11
+
12
+ def run
13
+ # notifies status as running and then restores old_status if it is an ending status
14
+ last_status = @job.status
15
+ @job.status = :running
16
+ @job.notify_status
17
+
18
+ if Job::ENDED_STATUS.include? last_status
19
+ @job.status = last_status
20
+ else
21
+ run_tasks
22
+
23
+ logger.default_tags.delete(:current_task)
24
+ # reset retries so retries don't mix between tasks and callbacks
25
+ @job.retries = 0
26
+ end
27
+
28
+ run_callback
29
+ logger.info "[type:job_finish] [status:#{@job.status}] data: #{@job.data.to_h[:outputs]}"
30
+ ensure
31
+ options = {}
32
+ options[:data] = @job.data if @job.ended_status?
33
+ @job.notify_status options
34
+ end
35
+
36
+ def run_tasks
37
+ @job.tasks_to_run.each do |task_name|
38
+ @job.current_action = task_name
39
+ @job.notify_task :running
40
+
41
+ logger.default_tags[:current_task] = @job.current_action
42
+ logger.info "[type:start_task] data: #{@job.data.to_h[:outputs]}"
43
+
44
+ begin
45
+ raise LittleMonster::CancelError if @job.is_cancelled?
46
+
47
+ task = build_task(task_name)
48
+ task.run
49
+
50
+ # data is sent only on task success
51
+ @job.notify_task :success, data: @job.data
52
+
53
+ logger.info "[type:finish_task] [status:success] data: #{@job.data.to_h[:outputs]}"
54
+
55
+ if @job.mock?
56
+ @job.runned_tasks[task_name] = {}
57
+ @job.runned_tasks[task_name][:instance] = task
58
+ @job.runned_tasks[task_name][:data] = @job.data.to_h[:outputs].to_h.dup
59
+ end
60
+ rescue APIUnreachableError => e
61
+ logger.error "[type:api_unreachable] [message:#{e.message}]"
62
+ raise e
63
+ rescue CancelError => e
64
+ logger.info '[type:cancel] job was cancelled'
65
+ cancel
66
+ return
67
+ rescue StandardError => e
68
+ logger.debug "[type:standard_error] an error was catched with [message:#{e.message}]"
69
+ task.error e unless e.is_a? NameError
70
+ handle_error e
71
+ return
72
+ end
73
+
74
+ @job.retries = 0 # Hago esto para que despues de succesful un task resete retries
75
+ end
76
+
77
+ @job.current_action = nil
78
+ @job.status = :success
79
+ end
80
+
81
+ def run_callback
82
+ @job.current_action = @job.callback_to_run
83
+
84
+ return if @job.current_action.nil?
85
+
86
+ logger.default_tags[:callback] = @job.current_action
87
+ @job.notify_callback :running
88
+
89
+ logger.info "[type:start_callback] data: #{@job.data.to_h[:outputs]}"
90
+ begin
91
+ logger.default_tags[:type] = 'callback_log'
92
+ @job.public_send(@job.current_action)
93
+ ensure
94
+ logger.default_tags.delete(:type)
95
+ end
96
+ logger.info "[type:finish_callback] [status:success] data: #{@job.data.to_h[:outputs]}"
97
+
98
+ @job.notify_callback :success
99
+
100
+ @job.current_action = nil
101
+ @job.retries = 0
102
+ logger.default_tags.delete(:callback)
103
+ rescue APIUnreachableError => e
104
+ logger.error "[type:api_unreachable] [message:#{e.message}]"
105
+ raise e
106
+ rescue StandardError => e
107
+ logger.debug "[type:standard_error] an error was catched with [message:#{e.message}]"
108
+ handle_error e
109
+ end
110
+
111
+ def build_task(task_symbol)
112
+ task = @job.task_class_for(task_symbol).new(@job.data)
113
+ task.send(:set_default_values, @job.data, @job.id, logger, @job.method(:is_cancelled?))
114
+ task
115
+ end
116
+
117
+ def cancel
118
+ logger.debug 'notifiying cancel...'
119
+
120
+ @job.notify_task :cancelled
121
+ logger.info "[type:finish_task] [status:cancelled] data: #{@job.data.to_h[:outputs]}"
122
+
123
+ @job.status = :cancelled
124
+ end
125
+
126
+ # Methods that work both on tasks and callbacks
127
+
128
+ def abort_job(_e)
129
+ logger.debug 'notifiying abort...'
130
+
131
+ if @job.callback_running?
132
+ logger.info "[type:finish_callback] [status:error] data: #{@job.data.to_h[:outputs]}"
133
+ @job.notify_callback :error
134
+
135
+ # if callback is not on_error, raise exception to run on_error
136
+ if @job.current_action != :on_error
137
+ # set status on pending because we are sending the job back to the queue
138
+ @job.status = :pending
139
+ raise CallbackFailedError, '[type:callback_fail_error]'
140
+ end
141
+ else
142
+ @job.notify_task :error
143
+ logger.info "[type:finish_task] [status:error] data: #{@job.data.to_h[:outputs]}"
144
+ end
145
+
146
+ @job.status = :error
147
+ end
148
+
149
+ def handle_error(e)
150
+ raise e if LittleMonster.env.development?
151
+ logger.error "[type:error] [error_type:#{e.class}][message:#{e.message}] \n #{e.backtrace.to_a.join("\n\t")}"
152
+
153
+ if e.is_a?(FatalTaskError) || e.is_a?(NameError)
154
+ logger.debug 'error is fatal, aborting run'
155
+ return abort_job(e)
156
+ end
157
+
158
+ do_retry
159
+ end
160
+
161
+ def do_retry
162
+ if @job.retry?
163
+ logger.debug "Retry ##{@job.retries} of #{@job.max_retries}"
164
+
165
+ @job.retries += 1
166
+
167
+ logger.debug 'notifiying retry'
168
+ if @job.callback_running?
169
+ @job.notify_callback :pending, retries: @job.retries
170
+ logger.info '[type:callback_retry]'
171
+ else
172
+ @job.notify_task :pending, retries: @job.retries
173
+ logger.info '[type:task_retry]'
174
+ end
175
+
176
+ @job.status = :pending
177
+
178
+ logger.info "[type:job_retry] data: #{@job.data.to_h[:outputs]}"
179
+ raise JobRetryError, "doing retry #{@job.retries} of #{@job.max_retries}"
180
+ else
181
+ logger.debug 'job has reached max retries'
182
+
183
+ if @job.callback_running?
184
+ logger.info '[type:callback_max_retries]'
185
+ else
186
+ logger.info '[type:task_max_retries]'
187
+ end
188
+
189
+ logger.info "[type:job_max_retries] [retries:#{@job.max_retries}]"
190
+ abort_job(MaxRetriesError.new)
191
+ end
192
+ end
193
+ end
194
+ end
@@ -0,0 +1,7 @@
1
+ module LittleMonster::Core
2
+ module Loggable
3
+ def logger
4
+ @logger ||= TaggedLogger.new
5
+ end
6
+ end
7
+ end
@@ -0,0 +1,39 @@
1
+ module LittleMonster::Core
2
+ class Runner
3
+ include Loggable
4
+
5
+ def initialize(params)
6
+ @params = params
7
+
8
+ @heartbeat_task = Concurrent::TimerTask.new(execution_interval: LittleMonster.heartbeat_execution_interval) do
9
+ send_heartbeat!
10
+ end
11
+ end
12
+
13
+ def run
14
+ send_heartbeat!
15
+
16
+ @heartbeat_task.execute unless LittleMonster.disable_requests?
17
+
18
+ job = LittleMonster::Job::Factory.new(@params).build
19
+ job.run unless job.nil?
20
+ rescue JobNotFoundError => e
21
+ logger.error "[id:#{@params[:id]}][type:job_not_found] [message:#{e.message}] \n #{e.backtrace.to_a.join("\n\t")}"
22
+ ensure
23
+ @heartbeat_task.shutdown
24
+ end
25
+
26
+ def send_heartbeat!
27
+ return if LittleMonster.disable_requests?
28
+
29
+ res = LittleMonster::API.put "/jobs/#{@params[:id]}/worker", body: {
30
+ ip: Socket.gethostname,
31
+ host: Socket.gethostname,
32
+ pid: Process.pid
33
+ }
34
+
35
+ raise LittleMonster::JobAlreadyLockedError, "job [id:#{@params[:id]}] is already locked, discarding" if res.code == 401
36
+ res.success?
37
+ end
38
+ end
39
+ end
@@ -0,0 +1,66 @@
1
+ module LittleMonster::Core
2
+ class TaggedLogger
3
+ attr_accessor :parent_logger
4
+ attr_reader :tags
5
+
6
+ LEVELS = [:unknown, :fatal, :error, :warn, :info, :debug].freeze
7
+
8
+ def self.tags_to_string(hash)
9
+ hash.map { |k, v| "[#{k}:#{v}]" }.join
10
+ end
11
+
12
+ def initialize
13
+ @tags = Hash.new({})
14
+ @parent_logger = nil
15
+ end
16
+
17
+ def method_missing(method, *args, &block)
18
+ if method.to_s.ends_with? 'tags='
19
+ tag_key = method.to_s.split('_').first.to_sym
20
+ return public_send('tags_for', tag_key, *args) if LEVELS.include? tag_key
21
+ end
22
+
23
+ if method.to_s.ends_with? 'tags'
24
+ tag_key = method.to_s.split('_').first.to_sym
25
+ return @tags[tag_key] if LEVELS.include? tag_key
26
+ end
27
+
28
+ if LEVELS.include? method.to_sym
29
+ return LittleMonster.logger.public_send method, tag_message(method.to_sym, *args)
30
+ end
31
+
32
+ super method, *args, &block
33
+ end
34
+
35
+ def tags_for(key, t = {})
36
+ @tags[key] = t
37
+ end
38
+
39
+ def default_tags
40
+ @tags[:default]
41
+ end
42
+
43
+ def default_tags=(t)
44
+ tags_for(:default, t)
45
+ end
46
+
47
+ def tag_message(level, message = '')
48
+ prefix_string = tags_to_string @tags[:default].merge(@tags[level])
49
+ prefix_string << ' -- ' unless prefix_string.blank?
50
+
51
+ unless @parent_logger.nil?
52
+ prefix_string = @parent_logger.tag_message level, prefix_string
53
+ end
54
+
55
+ [prefix_string, message].join
56
+ end
57
+
58
+ def log_tags(level, tags_hash)
59
+ public_send(level, tags_to_string(tags_hash))
60
+ end
61
+
62
+ def tags_to_string(hash)
63
+ self.class.tags_to_string hash
64
+ end
65
+ end
66
+ end
@@ -0,0 +1,41 @@
1
+ module LittleMonster::Core
2
+ class Task
3
+ include Loggable
4
+
5
+ attr_reader :data
6
+ attr_reader :job_id
7
+
8
+ def initialize(data, job_id = nil)
9
+ @data = data
10
+ @job_id = job_id
11
+ end
12
+
13
+ def run
14
+ raise NotImplementedError, 'You must implement the run method'
15
+ end
16
+
17
+ def on_error(error)
18
+ end
19
+
20
+ def error(e)
21
+ logger.error e
22
+ on_error e
23
+ end
24
+
25
+ def is_cancelled!
26
+ is_cancelled = false
27
+ is_cancelled = @cancelled_callback.call unless @cancelled_callback.nil?
28
+ raise CancelError if is_cancelled
29
+ end
30
+
31
+ private
32
+
33
+ def set_default_values(data, job_id = nil, job_logger = nil, cancelled_callback = nil)
34
+ @cancelled_callback = cancelled_callback
35
+ @job_id = job_id
36
+ @data = data
37
+ logger.parent_logger = job_logger if job_logger
38
+ logger.default_tags.merge!(type: 'task_log')
39
+ end
40
+ end
41
+ end
@@ -0,0 +1,75 @@
1
+ require 'thor'
2
+ require_relative './conf_gen'
3
+ require_relative './generate'
4
+
5
+ module LittleMonster
6
+ class Cli < Thor
7
+ desc 'show version', 'version'
8
+ map %w(-v --version) => :version
9
+
10
+ def version
11
+ say LittleMonster::VERSION
12
+ end
13
+
14
+ desc 'exec <job>', 'runs a job'
15
+ option :message,
16
+ type: :hash,
17
+ aliases: :m,
18
+ default: {}
19
+
20
+ method_option :message,
21
+ aliases: '-m',
22
+ type: :string,
23
+ default: '{}',
24
+ desc: 'Message that will be send as parameter (must be a JSON format)'
25
+
26
+ method_option :record_mode,
27
+ aliases: '-r',
28
+ type: :string,
29
+ enum: %w(none new reload),
30
+ default: 'none',
31
+ desc: 'Recording mocks mode none|new|reload',
32
+ banner: 'Recording type could be none,new or reload on default assume none'
33
+
34
+ def exec(job)
35
+ ENV['LITTLE_MONSTER_ENV'] = options[:environment]
36
+ require_relative "#{Dir.pwd}/config/application.rb"
37
+ require 'webmock'
38
+ require 'vcr'
39
+
40
+ msg = MultiJson.load(options[:message], symbolize_keys: true)
41
+ params = { data: { outputs: msg }, name: job }
42
+ vcr_mode = { 'none' => :none,
43
+ 'new' => :new_episodes,
44
+ 'reload' => :all }.fetch(options[:record_mode], :none)
45
+ VCR.configure do |config|
46
+ config.cassette_library_dir = 'mocks/vcr_cassettes'
47
+ config.hook_into :webmock # or :fakeweb
48
+ end
49
+
50
+ VCR.use_cassette(job.to_s, record: vcr_mode) do
51
+ LittleMonster::Runner.new(params).run
52
+ end
53
+ end
54
+
55
+ desc 'start', 'starts the little monster worker'
56
+ option :daemonize,
57
+ type: :boolean,
58
+ default: false,
59
+ aliases: :d
60
+
61
+ def start
62
+ require_relative "#{Dir.pwd}/config/application.rb"
63
+
64
+ toiler_args = ['-C', "#{Dir.pwd}/config/toiler.yml"]
65
+ toiler_args += ['-d', '-L', 'log/little_monster.log'] if options[:daemonize]
66
+ Toiler::CLI.instance.run(toiler_args)
67
+ end
68
+
69
+ register(LittleMonster::ConfGen, 'init', 'init', 'Creates new Little Monster Schema app')
70
+ register(LittleMonster::Generate,
71
+ 'generate',
72
+ 'generate <job_name> <task_list>...',
73
+ 'Creates a job with his respective tasks.')
74
+ end
75
+ end
@@ -0,0 +1,28 @@
1
+ require 'thor'
2
+ require 'active_support/core_ext/string'
3
+
4
+ module LittleMonster
5
+ class ConfGen < Thor::Group
6
+ include Thor::Actions
7
+
8
+ def self.source_root
9
+ File.dirname(__FILE__)
10
+ end
11
+
12
+ def create_conf_files
13
+ directory('./templates/config', 'config')
14
+ end
15
+
16
+ def create_lib_files
17
+ directory('./templates/lib', 'lib')
18
+ end
19
+
20
+ def create_log_files
21
+ directory('./templates/log', 'log')
22
+ end
23
+
24
+ def create_specs_files
25
+ template 'templates/spec_helper_temp.erb', 'spec/spec_helper.rb'
26
+ end
27
+ end
28
+ end