little_monster 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. checksums.yaml +7 -0
  2. data/.codeclimate.yml +6 -0
  3. data/.gitignore +11 -0
  4. data/.rubocop.yml +34 -0
  5. data/.ruby-version +1 -0
  6. data/.travis.yml +7 -0
  7. data/Gemfile +5 -0
  8. data/Gemfile.lock +124 -0
  9. data/README.md +5 -0
  10. data/Rakefile +6 -0
  11. data/bin/console +10 -0
  12. data/bin/setup +8 -0
  13. data/exe/lm +4 -0
  14. data/lib/little_monster.rb +67 -0
  15. data/lib/little_monster/all.rb +4 -0
  16. data/lib/little_monster/config.rb +29 -0
  17. data/lib/little_monster/core.rb +20 -0
  18. data/lib/little_monster/core/api.rb +74 -0
  19. data/lib/little_monster/core/counters.rb +50 -0
  20. data/lib/little_monster/core/errors/api_unreachable_error.rb +4 -0
  21. data/lib/little_monster/core/errors/callback_failed_error.rb +4 -0
  22. data/lib/little_monster/core/errors/cancel_error.rb +4 -0
  23. data/lib/little_monster/core/errors/fatal_task_error.rb +4 -0
  24. data/lib/little_monster/core/errors/job_already_locked_error.rb +4 -0
  25. data/lib/little_monster/core/errors/job_not_found_error.rb +15 -0
  26. data/lib/little_monster/core/errors/job_retry_error.rb +4 -0
  27. data/lib/little_monster/core/errors/max_retries_error.rb +4 -0
  28. data/lib/little_monster/core/errors/task_error.rb +4 -0
  29. data/lib/little_monster/core/job.rb +188 -0
  30. data/lib/little_monster/core/job_data.rb +47 -0
  31. data/lib/little_monster/core/job_factory.rb +139 -0
  32. data/lib/little_monster/core/job_orchrestator.rb +194 -0
  33. data/lib/little_monster/core/loggable.rb +7 -0
  34. data/lib/little_monster/core/runner.rb +39 -0
  35. data/lib/little_monster/core/tagged_logger.rb +66 -0
  36. data/lib/little_monster/core/task.rb +41 -0
  37. data/lib/little_monster/generators/cli.rb +75 -0
  38. data/lib/little_monster/generators/conf_gen.rb +28 -0
  39. data/lib/little_monster/generators/generate.rb +35 -0
  40. data/lib/little_monster/generators/templates/config/application.rb +15 -0
  41. data/lib/little_monster/generators/templates/config/enviroments/development.rb +1 -0
  42. data/lib/little_monster/generators/templates/config/enviroments/production.rb +1 -0
  43. data/lib/little_monster/generators/templates/config/enviroments/test.rb +1 -0
  44. data/lib/little_monster/generators/templates/config/toiler.yml +3 -0
  45. data/lib/little_monster/generators/templates/jobs_spec_temp.erb +11 -0
  46. data/lib/little_monster/generators/templates/jobs_temp.erb +16 -0
  47. data/lib/little_monster/generators/templates/lib/.keep +0 -0
  48. data/lib/little_monster/generators/templates/log/.keep +0 -0
  49. data/lib/little_monster/generators/templates/spec_helper_temp.erb +22 -0
  50. data/lib/little_monster/generators/templates/tasks_spec_temp.erb +11 -0
  51. data/lib/little_monster/generators/templates/tasks_temp.erb +5 -0
  52. data/lib/little_monster/rspec.rb +20 -0
  53. data/lib/little_monster/rspec/helpers/job_helper.rb +61 -0
  54. data/lib/little_monster/rspec/helpers/task_helper.rb +46 -0
  55. data/lib/little_monster/rspec/matchers/have_data.rb +24 -0
  56. data/lib/little_monster/rspec/matchers/have_ended_with_status.rb +24 -0
  57. data/lib/little_monster/rspec/matchers/have_run.rb +28 -0
  58. data/lib/little_monster/rspec/matchers/have_run_task.rb +47 -0
  59. data/lib/little_monster/version.rb +3 -0
  60. data/lib/little_monster/worker.rb +27 -0
  61. data/little_monster.gemspec +48 -0
  62. metadata +343 -0
@@ -0,0 +1,35 @@
1
+ require 'thor'
2
+ require 'active_support/core_ext/string'
3
+
4
+ module LittleMonster
5
+ class Generate < Thor::Group
6
+ include Thor::Actions
7
+
8
+ argument :job_name,
9
+ type: :string,
10
+ banner: 'Job Name',
11
+ required: true
12
+
13
+ argument :task_names,
14
+ type: :array,
15
+ banner: 'A set of instructions for making or preparing something',
16
+ required: true
17
+
18
+ def self.source_root
19
+ File.dirname(__FILE__)
20
+ end
21
+
22
+ def create_job_file
23
+ template('templates/jobs_temp.erb', "jobs/#{job_name}.rb")
24
+ template 'templates/jobs_spec_temp.erb', "spec/jobs/#{job_name}_spec.rb"
25
+ end
26
+
27
+ def create_tasks_file
28
+ task_names.each do |task|
29
+ @current_task_name = task
30
+ template('templates/tasks_temp.erb', "tasks/#{job_name}/#{task}.rb")
31
+ template 'templates/tasks_spec_temp.erb', "spec/tasks/#{job_name}/#{task}_spec.rb"
32
+ end
33
+ end
34
+ end
35
+ end
@@ -0,0 +1,15 @@
1
+ require 'bundler'
2
+ Bundler.require(:default)
3
+
4
+ LittleMonster.configure do |conf|
5
+ conf.api_url = 'http://pepe.hongo.com'
6
+ conf.default_request_retries = 3
7
+ # conf.worker_concurrency = 200
8
+ # conf.worker_queue = 'my_sqs_queue'
9
+ end
10
+
11
+ require_relative "enviroments/#{LittleMonster.env}"
12
+
13
+ Dir["#{Dir.pwd}/lib/**/*.rb"].each { |file| require_relative file }
14
+ Dir["#{Dir.pwd}/jobs/**/*.rb"].each { |file| require_relative file }
15
+ Dir["#{Dir.pwd}/tasks/**/*.rb"].each { |file| require_relative file }
@@ -0,0 +1 @@
1
+ Bundler.require(:development)
@@ -0,0 +1 @@
1
+ Bundler.require(:production)
@@ -0,0 +1 @@
1
+ Bundler.require(:test)
@@ -0,0 +1,3 @@
1
+ aws:
2
+ region: us-east-1
3
+ wait: 20
@@ -0,0 +1,11 @@
1
+ require_relative '../spec_helper'
2
+
3
+ describe '<%=@job_name.camelize%>' do
4
+ context '.run' do
5
+ it 'runs this tasks <%=@task_names.join(" ")%>' do
6
+ expect(run_job(:<%=job_name%>)).to have_run(<%=@task_names.map{|v|":#{v}"}.join(", ")%>)
7
+ end
8
+
9
+ it 'has to run'
10
+ end
11
+ end
@@ -0,0 +1,16 @@
1
+ class <%= @job_name.camelize %> < LittleMonster::Job
2
+ task_list :<%= @task_names.join(", :") %>
3
+
4
+ # Callbacks
5
+ # def on_cancel
6
+ # #Code me!
7
+ # end
8
+ #
9
+ # def on_error
10
+ # #code me
11
+ # end
12
+
13
+ # def on_abort
14
+ # #code me
15
+ # end
16
+ end
File without changes
File without changes
@@ -0,0 +1,22 @@
1
+ ENV['RUBY_ENV'] = 'test'
2
+ require 'bundler'
3
+ Bundler.require(:default, :test)
4
+
5
+ require 'little_monster/rspec'
6
+
7
+ require_rel '../jobs'
8
+ require_rel '../tasks'
9
+
10
+ RSpec.configure do |conf|
11
+ conf.color = true
12
+ conf.formatter = :documentation
13
+
14
+ conf.mock_with :rspec do |mocks|
15
+ mocks.verify_partial_doubles = true
16
+ end
17
+
18
+ conf.before :each do
19
+ allow_any_instance_of(Kernel).to receive(:sleep)
20
+ end
21
+ end
22
+
@@ -0,0 +1,11 @@
1
+ require_relative '../../spec_helper.rb'
2
+
3
+ describe <%= "#{@job_name.camelize}::#{@current_task_name.camelize}" %> do
4
+ before :each do
5
+ @<%=@current_task_name%> = described_class.new({my_data_key:'my_data_value'})
6
+ end
7
+
8
+ describe '.run' do
9
+ it 'tell me how to run <%=@current_task_name.underscore%>!'
10
+ end
11
+ end
@@ -0,0 +1,5 @@
1
+ class <%= "#{@job_name.camelize}::#{@current_task_name.camelize}" %> < LittleMonster::Task
2
+ def run
3
+ # code me!
4
+ end
5
+ end
@@ -0,0 +1,20 @@
1
+ module LittleMonster::RSpec
2
+ require 'rspec'
3
+ require 'rspec/expectations'
4
+
5
+ require 'little_monster/rspec/helpers/job_helper'
6
+ require 'little_monster/rspec/helpers/task_helper'
7
+
8
+ # when required, this files define custom matchers on rspec
9
+ require 'little_monster/rspec/matchers/have_ended_with_status'
10
+ require 'little_monster/rspec/matchers/have_data'
11
+ require 'little_monster/rspec/matchers/have_run'
12
+ require 'little_monster/rspec/matchers/have_run_task'
13
+
14
+ # includes the run_job and run_task helper methods
15
+ ::RSpec.configure do |config|
16
+ config.include JobHelper
17
+ config.include TaskHelper
18
+ config.include Matchers
19
+ end
20
+ end
@@ -0,0 +1,61 @@
1
+ module LittleMonster::RSpec
2
+ module JobHelper
3
+ class Result
4
+ def initialize(job)
5
+ @job = job
6
+
7
+ begin
8
+ job.run
9
+ @retried = false
10
+ rescue LittleMonster::JobRetryError
11
+ @retried = true
12
+ end
13
+ end
14
+
15
+ def instance
16
+ @job
17
+ end
18
+
19
+ def status
20
+ @job.status
21
+ end
22
+
23
+ def retried?
24
+ @retried
25
+ end
26
+
27
+ def retries
28
+ @job.instance_variable_get '@retries'
29
+ end
30
+
31
+ def data
32
+ @job.data
33
+ end
34
+
35
+ def runned_tasks
36
+ @job.instance_variable_get '@runned_tasks'
37
+ end
38
+ end
39
+
40
+ def run_job(job, options = {})
41
+ Result.new(generate_job(job, options))
42
+ end
43
+
44
+ def generate_job(job, options = {})
45
+ job_class = job.class == Class ? job : job.to_s.camelcase.constantize
46
+ job_class.mock!
47
+
48
+ job_instance = job_class.new(data: { outputs: options.fetch(:data, {}) })
49
+ job_instance.define_singleton_method(:is_cancelled?) { options.fetch(:cancelled, false) }
50
+
51
+ if options[:fails]
52
+ options[:fails] = [options[:fails]] unless options[:fails].is_a? Array
53
+ options[:fails].each do |hash|
54
+ allow_any_instance_of(job_class.task_class_for(hash[:task])).to receive(:run).and_raise(hash.fetch(:error, StandardError.new))
55
+ end
56
+ end
57
+
58
+ job_instance
59
+ end
60
+ end
61
+ end
@@ -0,0 +1,46 @@
1
+ module LittleMonster::RSpec
2
+ module TaskHelper
3
+ class Result
4
+ def initialize(task)
5
+ @task = task
6
+ task.run
7
+ end
8
+
9
+ def instance
10
+ @task
11
+ end
12
+
13
+ def data
14
+ @task.data
15
+ end
16
+ end
17
+
18
+ def run_task(task, options = {})
19
+ task_instance = generate_task(task, options)
20
+
21
+ Result.new(task_instance)
22
+ end
23
+
24
+ def generate_task(task, options = {})
25
+ task_class = if task.class != Class
26
+ task.to_s.camelcase.constantize
27
+ else
28
+ task
29
+ end
30
+
31
+ task_symbol = task.to_s.underscore.split('/').last.to_sym
32
+ data = if options[:data].class == LittleMonster::Job::Data
33
+ options[:data]
34
+ else
35
+ LittleMonster::Job::Data.new(double(current_task: task_symbol),
36
+ outputs: options.fetch(:data, {}))
37
+ end
38
+
39
+ task_instance = task_class.new(data)
40
+ task_instance.send(:set_default_values, data)
41
+ task_instance.instance_variable_set('@cancelled_callback', (proc { true })) if options[:cancelled]
42
+
43
+ task_instance
44
+ end
45
+ end
46
+ end
@@ -0,0 +1,24 @@
1
+ module LittleMonster::RSpec::Matchers
2
+ class HaveData
3
+ def initialize(expected_data)
4
+ @expected_data = expected_data
5
+ end
6
+
7
+ def matches?(actual)
8
+ @actual_data = actual.data
9
+ @actual_data == @expected_data
10
+ end
11
+
12
+ def failure_message
13
+ "expected data #{@expected_data} but was #{@actual_data.instance_variable_get('@outputs') || 'nil'}"
14
+ end
15
+
16
+ def failure_message_when_negated
17
+ "expected data not to be #{@expected_data}"
18
+ end
19
+ end
20
+
21
+ def have_data(*args)
22
+ HaveData.new(*args)
23
+ end
24
+ end
@@ -0,0 +1,24 @@
1
+ module LittleMonster::RSpec::Matchers
2
+ class HaveEndedWithStatus
3
+ def initialize(expected_status)
4
+ @expected_status = expected_status
5
+ end
6
+
7
+ def matches?(job_result)
8
+ @actual_status = job_result.status
9
+ @actual_status == @expected_status
10
+ end
11
+
12
+ def failure_message
13
+ "expected job to end with status #{@expected_status} but was #{@actual_status}"
14
+ end
15
+
16
+ def failure_message_when_negated
17
+ "expected job not to end with status #{@expected_status} but instead ended that way"
18
+ end
19
+ end
20
+
21
+ def have_ended_with_status(*args)
22
+ HaveEndedWithStatus.new(*args)
23
+ end
24
+ end
@@ -0,0 +1,28 @@
1
+ module LittleMonster::RSpec::Matchers
2
+ class HaveRun
3
+ def initialize(*expected_tasks)
4
+ @expected_tasks = if expected_tasks.length == 1
5
+ expected_tasks.first
6
+ else
7
+ expected_tasks
8
+ end
9
+ end
10
+
11
+ def matches?(job_result)
12
+ @actual_tasks = job_result.runned_tasks.keys
13
+ @actual_tasks == @expected_tasks
14
+ end
15
+
16
+ def failure_message
17
+ "expected job to run #{@expected_tasks} but instead run #{@actual_tasks}"
18
+ end
19
+
20
+ def failure_message_when_negated
21
+ "expected job not to run #{@expected_tasks}"
22
+ end
23
+ end
24
+
25
+ def have_run(*args)
26
+ HaveRun.new(*args)
27
+ end
28
+ end
@@ -0,0 +1,47 @@
1
+ module LittleMonster::RSpec::Matchers
2
+ class HaveRunTask
3
+ attr_reader :expected_task
4
+ attr_reader :expected_data
5
+
6
+ def initialize(expected_task)
7
+ @expected_task = if expected_task.class == Class
8
+ expected_task.to_s.underscore.split('/').last.to_sym
9
+ else
10
+ expected_task
11
+ end
12
+ end
13
+
14
+ def matches?(job)
15
+ @task = job.runned_tasks[@expected_task][:instance]
16
+ @task_data = job.runned_tasks[@expected_task][:data]
17
+ check_task_run && check_data
18
+ end
19
+
20
+ def check_task_run
21
+ !@task.nil?
22
+ end
23
+
24
+ def check_data
25
+ if defined?(@expected_data)
26
+ @task_data == @expected_data
27
+ else
28
+ true
29
+ end
30
+ end
31
+
32
+ def with_data(data)
33
+ @expected_data = data
34
+ self
35
+ end
36
+
37
+ def failure_message
38
+ message = "task #{@expected_task} was expected to run\n"
39
+ message << "\twith data #{@expected_data} but was #{@task_data || 'nil'}\n" unless check_data
40
+ message
41
+ end
42
+ end
43
+
44
+ def have_run_task(*args)
45
+ HaveRunTask.new(*args)
46
+ end
47
+ end
@@ -0,0 +1,3 @@
1
+ module LittleMonster
2
+ VERSION = '0.1.0'.freeze
3
+ end
@@ -0,0 +1,27 @@
1
+ require 'toiler'
2
+
3
+ module LittleMonster
4
+ class Worker
5
+ include LittleMonster::Loggable
6
+ include ::Toiler::Worker
7
+
8
+ toiler_options queue: LittleMonster.worker_queue,
9
+ concurrency: LittleMonster.worker_concurrency
10
+
11
+ toiler_options auto_visibility_timeout: true,
12
+ auto_delete: true
13
+
14
+ toiler_options parser: MultiJson
15
+
16
+ def self.update_attributes
17
+ toiler_options queue: LittleMonster.worker_queue,
18
+ concurrency: LittleMonster.worker_concurrency
19
+ end
20
+
21
+ def perform(_sqs_msg, body)
22
+ message = MultiJson.load body['Message'], symbolize_keys: true
23
+
24
+ LittleMonster::Runner.new(message).run
25
+ end
26
+ end
27
+ end