little_monster 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.codeclimate.yml +6 -0
- data/.gitignore +11 -0
- data/.rubocop.yml +34 -0
- data/.ruby-version +1 -0
- data/.travis.yml +7 -0
- data/Gemfile +5 -0
- data/Gemfile.lock +124 -0
- data/README.md +5 -0
- data/Rakefile +6 -0
- data/bin/console +10 -0
- data/bin/setup +8 -0
- data/exe/lm +4 -0
- data/lib/little_monster.rb +67 -0
- data/lib/little_monster/all.rb +4 -0
- data/lib/little_monster/config.rb +29 -0
- data/lib/little_monster/core.rb +20 -0
- data/lib/little_monster/core/api.rb +74 -0
- data/lib/little_monster/core/counters.rb +50 -0
- data/lib/little_monster/core/errors/api_unreachable_error.rb +4 -0
- data/lib/little_monster/core/errors/callback_failed_error.rb +4 -0
- data/lib/little_monster/core/errors/cancel_error.rb +4 -0
- data/lib/little_monster/core/errors/fatal_task_error.rb +4 -0
- data/lib/little_monster/core/errors/job_already_locked_error.rb +4 -0
- data/lib/little_monster/core/errors/job_not_found_error.rb +15 -0
- data/lib/little_monster/core/errors/job_retry_error.rb +4 -0
- data/lib/little_monster/core/errors/max_retries_error.rb +4 -0
- data/lib/little_monster/core/errors/task_error.rb +4 -0
- data/lib/little_monster/core/job.rb +188 -0
- data/lib/little_monster/core/job_data.rb +47 -0
- data/lib/little_monster/core/job_factory.rb +139 -0
- data/lib/little_monster/core/job_orchrestator.rb +194 -0
- data/lib/little_monster/core/loggable.rb +7 -0
- data/lib/little_monster/core/runner.rb +39 -0
- data/lib/little_monster/core/tagged_logger.rb +66 -0
- data/lib/little_monster/core/task.rb +41 -0
- data/lib/little_monster/generators/cli.rb +75 -0
- data/lib/little_monster/generators/conf_gen.rb +28 -0
- data/lib/little_monster/generators/generate.rb +35 -0
- data/lib/little_monster/generators/templates/config/application.rb +15 -0
- data/lib/little_monster/generators/templates/config/enviroments/development.rb +1 -0
- data/lib/little_monster/generators/templates/config/enviroments/production.rb +1 -0
- data/lib/little_monster/generators/templates/config/enviroments/test.rb +1 -0
- data/lib/little_monster/generators/templates/config/toiler.yml +3 -0
- data/lib/little_monster/generators/templates/jobs_spec_temp.erb +11 -0
- data/lib/little_monster/generators/templates/jobs_temp.erb +16 -0
- data/lib/little_monster/generators/templates/lib/.keep +0 -0
- data/lib/little_monster/generators/templates/log/.keep +0 -0
- data/lib/little_monster/generators/templates/spec_helper_temp.erb +22 -0
- data/lib/little_monster/generators/templates/tasks_spec_temp.erb +11 -0
- data/lib/little_monster/generators/templates/tasks_temp.erb +5 -0
- data/lib/little_monster/rspec.rb +20 -0
- data/lib/little_monster/rspec/helpers/job_helper.rb +61 -0
- data/lib/little_monster/rspec/helpers/task_helper.rb +46 -0
- data/lib/little_monster/rspec/matchers/have_data.rb +24 -0
- data/lib/little_monster/rspec/matchers/have_ended_with_status.rb +24 -0
- data/lib/little_monster/rspec/matchers/have_run.rb +28 -0
- data/lib/little_monster/rspec/matchers/have_run_task.rb +47 -0
- data/lib/little_monster/version.rb +3 -0
- data/lib/little_monster/worker.rb +27 -0
- data/little_monster.gemspec +48 -0
- metadata +343 -0
@@ -0,0 +1,35 @@
|
|
1
|
+
require 'thor'
|
2
|
+
require 'active_support/core_ext/string'
|
3
|
+
|
4
|
+
module LittleMonster
|
5
|
+
class Generate < Thor::Group
|
6
|
+
include Thor::Actions
|
7
|
+
|
8
|
+
argument :job_name,
|
9
|
+
type: :string,
|
10
|
+
banner: 'Job Name',
|
11
|
+
required: true
|
12
|
+
|
13
|
+
argument :task_names,
|
14
|
+
type: :array,
|
15
|
+
banner: 'A set of instructions for making or preparing something',
|
16
|
+
required: true
|
17
|
+
|
18
|
+
def self.source_root
|
19
|
+
File.dirname(__FILE__)
|
20
|
+
end
|
21
|
+
|
22
|
+
def create_job_file
|
23
|
+
template('templates/jobs_temp.erb', "jobs/#{job_name}.rb")
|
24
|
+
template 'templates/jobs_spec_temp.erb', "spec/jobs/#{job_name}_spec.rb"
|
25
|
+
end
|
26
|
+
|
27
|
+
def create_tasks_file
|
28
|
+
task_names.each do |task|
|
29
|
+
@current_task_name = task
|
30
|
+
template('templates/tasks_temp.erb', "tasks/#{job_name}/#{task}.rb")
|
31
|
+
template 'templates/tasks_spec_temp.erb', "spec/tasks/#{job_name}/#{task}_spec.rb"
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
require 'bundler'
|
2
|
+
Bundler.require(:default)
|
3
|
+
|
4
|
+
LittleMonster.configure do |conf|
|
5
|
+
conf.api_url = 'http://pepe.hongo.com'
|
6
|
+
conf.default_request_retries = 3
|
7
|
+
# conf.worker_concurrency = 200
|
8
|
+
# conf.worker_queue = 'my_sqs_queue'
|
9
|
+
end
|
10
|
+
|
11
|
+
require_relative "enviroments/#{LittleMonster.env}"
|
12
|
+
|
13
|
+
Dir["#{Dir.pwd}/lib/**/*.rb"].each { |file| require_relative file }
|
14
|
+
Dir["#{Dir.pwd}/jobs/**/*.rb"].each { |file| require_relative file }
|
15
|
+
Dir["#{Dir.pwd}/tasks/**/*.rb"].each { |file| require_relative file }
|
@@ -0,0 +1 @@
|
|
1
|
+
Bundler.require(:development)
|
@@ -0,0 +1 @@
|
|
1
|
+
Bundler.require(:production)
|
@@ -0,0 +1 @@
|
|
1
|
+
Bundler.require(:test)
|
@@ -0,0 +1,11 @@
|
|
1
|
+
require_relative '../spec_helper'
|
2
|
+
|
3
|
+
describe '<%=@job_name.camelize%>' do
|
4
|
+
context '.run' do
|
5
|
+
it 'runs this tasks <%=@task_names.join(" ")%>' do
|
6
|
+
expect(run_job(:<%=job_name%>)).to have_run(<%=@task_names.map{|v|":#{v}"}.join(", ")%>)
|
7
|
+
end
|
8
|
+
|
9
|
+
it 'has to run'
|
10
|
+
end
|
11
|
+
end
|
File without changes
|
File without changes
|
@@ -0,0 +1,22 @@
|
|
1
|
+
ENV['RUBY_ENV'] = 'test'
|
2
|
+
require 'bundler'
|
3
|
+
Bundler.require(:default, :test)
|
4
|
+
|
5
|
+
require 'little_monster/rspec'
|
6
|
+
|
7
|
+
require_rel '../jobs'
|
8
|
+
require_rel '../tasks'
|
9
|
+
|
10
|
+
RSpec.configure do |conf|
|
11
|
+
conf.color = true
|
12
|
+
conf.formatter = :documentation
|
13
|
+
|
14
|
+
conf.mock_with :rspec do |mocks|
|
15
|
+
mocks.verify_partial_doubles = true
|
16
|
+
end
|
17
|
+
|
18
|
+
conf.before :each do
|
19
|
+
allow_any_instance_of(Kernel).to receive(:sleep)
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
@@ -0,0 +1,11 @@
|
|
1
|
+
require_relative '../../spec_helper.rb'
|
2
|
+
|
3
|
+
describe <%= "#{@job_name.camelize}::#{@current_task_name.camelize}" %> do
|
4
|
+
before :each do
|
5
|
+
@<%=@current_task_name%> = described_class.new({my_data_key:'my_data_value'})
|
6
|
+
end
|
7
|
+
|
8
|
+
describe '.run' do
|
9
|
+
it 'tell me how to run <%=@current_task_name.underscore%>!'
|
10
|
+
end
|
11
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
module LittleMonster::RSpec
|
2
|
+
require 'rspec'
|
3
|
+
require 'rspec/expectations'
|
4
|
+
|
5
|
+
require 'little_monster/rspec/helpers/job_helper'
|
6
|
+
require 'little_monster/rspec/helpers/task_helper'
|
7
|
+
|
8
|
+
# when required, this files define custom matchers on rspec
|
9
|
+
require 'little_monster/rspec/matchers/have_ended_with_status'
|
10
|
+
require 'little_monster/rspec/matchers/have_data'
|
11
|
+
require 'little_monster/rspec/matchers/have_run'
|
12
|
+
require 'little_monster/rspec/matchers/have_run_task'
|
13
|
+
|
14
|
+
# includes the run_job and run_task helper methods
|
15
|
+
::RSpec.configure do |config|
|
16
|
+
config.include JobHelper
|
17
|
+
config.include TaskHelper
|
18
|
+
config.include Matchers
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,61 @@
|
|
1
|
+
module LittleMonster::RSpec
|
2
|
+
module JobHelper
|
3
|
+
class Result
|
4
|
+
def initialize(job)
|
5
|
+
@job = job
|
6
|
+
|
7
|
+
begin
|
8
|
+
job.run
|
9
|
+
@retried = false
|
10
|
+
rescue LittleMonster::JobRetryError
|
11
|
+
@retried = true
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
def instance
|
16
|
+
@job
|
17
|
+
end
|
18
|
+
|
19
|
+
def status
|
20
|
+
@job.status
|
21
|
+
end
|
22
|
+
|
23
|
+
def retried?
|
24
|
+
@retried
|
25
|
+
end
|
26
|
+
|
27
|
+
def retries
|
28
|
+
@job.instance_variable_get '@retries'
|
29
|
+
end
|
30
|
+
|
31
|
+
def data
|
32
|
+
@job.data
|
33
|
+
end
|
34
|
+
|
35
|
+
def runned_tasks
|
36
|
+
@job.instance_variable_get '@runned_tasks'
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
def run_job(job, options = {})
|
41
|
+
Result.new(generate_job(job, options))
|
42
|
+
end
|
43
|
+
|
44
|
+
def generate_job(job, options = {})
|
45
|
+
job_class = job.class == Class ? job : job.to_s.camelcase.constantize
|
46
|
+
job_class.mock!
|
47
|
+
|
48
|
+
job_instance = job_class.new(data: { outputs: options.fetch(:data, {}) })
|
49
|
+
job_instance.define_singleton_method(:is_cancelled?) { options.fetch(:cancelled, false) }
|
50
|
+
|
51
|
+
if options[:fails]
|
52
|
+
options[:fails] = [options[:fails]] unless options[:fails].is_a? Array
|
53
|
+
options[:fails].each do |hash|
|
54
|
+
allow_any_instance_of(job_class.task_class_for(hash[:task])).to receive(:run).and_raise(hash.fetch(:error, StandardError.new))
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
job_instance
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
@@ -0,0 +1,46 @@
|
|
1
|
+
module LittleMonster::RSpec
|
2
|
+
module TaskHelper
|
3
|
+
class Result
|
4
|
+
def initialize(task)
|
5
|
+
@task = task
|
6
|
+
task.run
|
7
|
+
end
|
8
|
+
|
9
|
+
def instance
|
10
|
+
@task
|
11
|
+
end
|
12
|
+
|
13
|
+
def data
|
14
|
+
@task.data
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def run_task(task, options = {})
|
19
|
+
task_instance = generate_task(task, options)
|
20
|
+
|
21
|
+
Result.new(task_instance)
|
22
|
+
end
|
23
|
+
|
24
|
+
def generate_task(task, options = {})
|
25
|
+
task_class = if task.class != Class
|
26
|
+
task.to_s.camelcase.constantize
|
27
|
+
else
|
28
|
+
task
|
29
|
+
end
|
30
|
+
|
31
|
+
task_symbol = task.to_s.underscore.split('/').last.to_sym
|
32
|
+
data = if options[:data].class == LittleMonster::Job::Data
|
33
|
+
options[:data]
|
34
|
+
else
|
35
|
+
LittleMonster::Job::Data.new(double(current_task: task_symbol),
|
36
|
+
outputs: options.fetch(:data, {}))
|
37
|
+
end
|
38
|
+
|
39
|
+
task_instance = task_class.new(data)
|
40
|
+
task_instance.send(:set_default_values, data)
|
41
|
+
task_instance.instance_variable_set('@cancelled_callback', (proc { true })) if options[:cancelled]
|
42
|
+
|
43
|
+
task_instance
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
module LittleMonster::RSpec::Matchers
|
2
|
+
class HaveData
|
3
|
+
def initialize(expected_data)
|
4
|
+
@expected_data = expected_data
|
5
|
+
end
|
6
|
+
|
7
|
+
def matches?(actual)
|
8
|
+
@actual_data = actual.data
|
9
|
+
@actual_data == @expected_data
|
10
|
+
end
|
11
|
+
|
12
|
+
def failure_message
|
13
|
+
"expected data #{@expected_data} but was #{@actual_data.instance_variable_get('@outputs') || 'nil'}"
|
14
|
+
end
|
15
|
+
|
16
|
+
def failure_message_when_negated
|
17
|
+
"expected data not to be #{@expected_data}"
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
def have_data(*args)
|
22
|
+
HaveData.new(*args)
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
module LittleMonster::RSpec::Matchers
|
2
|
+
class HaveEndedWithStatus
|
3
|
+
def initialize(expected_status)
|
4
|
+
@expected_status = expected_status
|
5
|
+
end
|
6
|
+
|
7
|
+
def matches?(job_result)
|
8
|
+
@actual_status = job_result.status
|
9
|
+
@actual_status == @expected_status
|
10
|
+
end
|
11
|
+
|
12
|
+
def failure_message
|
13
|
+
"expected job to end with status #{@expected_status} but was #{@actual_status}"
|
14
|
+
end
|
15
|
+
|
16
|
+
def failure_message_when_negated
|
17
|
+
"expected job not to end with status #{@expected_status} but instead ended that way"
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
def have_ended_with_status(*args)
|
22
|
+
HaveEndedWithStatus.new(*args)
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
module LittleMonster::RSpec::Matchers
|
2
|
+
class HaveRun
|
3
|
+
def initialize(*expected_tasks)
|
4
|
+
@expected_tasks = if expected_tasks.length == 1
|
5
|
+
expected_tasks.first
|
6
|
+
else
|
7
|
+
expected_tasks
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
def matches?(job_result)
|
12
|
+
@actual_tasks = job_result.runned_tasks.keys
|
13
|
+
@actual_tasks == @expected_tasks
|
14
|
+
end
|
15
|
+
|
16
|
+
def failure_message
|
17
|
+
"expected job to run #{@expected_tasks} but instead run #{@actual_tasks}"
|
18
|
+
end
|
19
|
+
|
20
|
+
def failure_message_when_negated
|
21
|
+
"expected job not to run #{@expected_tasks}"
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
def have_run(*args)
|
26
|
+
HaveRun.new(*args)
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
module LittleMonster::RSpec::Matchers
|
2
|
+
class HaveRunTask
|
3
|
+
attr_reader :expected_task
|
4
|
+
attr_reader :expected_data
|
5
|
+
|
6
|
+
def initialize(expected_task)
|
7
|
+
@expected_task = if expected_task.class == Class
|
8
|
+
expected_task.to_s.underscore.split('/').last.to_sym
|
9
|
+
else
|
10
|
+
expected_task
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
def matches?(job)
|
15
|
+
@task = job.runned_tasks[@expected_task][:instance]
|
16
|
+
@task_data = job.runned_tasks[@expected_task][:data]
|
17
|
+
check_task_run && check_data
|
18
|
+
end
|
19
|
+
|
20
|
+
def check_task_run
|
21
|
+
!@task.nil?
|
22
|
+
end
|
23
|
+
|
24
|
+
def check_data
|
25
|
+
if defined?(@expected_data)
|
26
|
+
@task_data == @expected_data
|
27
|
+
else
|
28
|
+
true
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def with_data(data)
|
33
|
+
@expected_data = data
|
34
|
+
self
|
35
|
+
end
|
36
|
+
|
37
|
+
def failure_message
|
38
|
+
message = "task #{@expected_task} was expected to run\n"
|
39
|
+
message << "\twith data #{@expected_data} but was #{@task_data || 'nil'}\n" unless check_data
|
40
|
+
message
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
def have_run_task(*args)
|
45
|
+
HaveRunTask.new(*args)
|
46
|
+
end
|
47
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
require 'toiler'
|
2
|
+
|
3
|
+
module LittleMonster
|
4
|
+
class Worker
|
5
|
+
include LittleMonster::Loggable
|
6
|
+
include ::Toiler::Worker
|
7
|
+
|
8
|
+
toiler_options queue: LittleMonster.worker_queue,
|
9
|
+
concurrency: LittleMonster.worker_concurrency
|
10
|
+
|
11
|
+
toiler_options auto_visibility_timeout: true,
|
12
|
+
auto_delete: true
|
13
|
+
|
14
|
+
toiler_options parser: MultiJson
|
15
|
+
|
16
|
+
def self.update_attributes
|
17
|
+
toiler_options queue: LittleMonster.worker_queue,
|
18
|
+
concurrency: LittleMonster.worker_concurrency
|
19
|
+
end
|
20
|
+
|
21
|
+
def perform(_sqs_msg, body)
|
22
|
+
message = MultiJson.load body['Message'], symbolize_keys: true
|
23
|
+
|
24
|
+
LittleMonster::Runner.new(message).run
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|