little_monster 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.codeclimate.yml +6 -0
- data/.gitignore +11 -0
- data/.rubocop.yml +34 -0
- data/.ruby-version +1 -0
- data/.travis.yml +7 -0
- data/Gemfile +5 -0
- data/Gemfile.lock +124 -0
- data/README.md +5 -0
- data/Rakefile +6 -0
- data/bin/console +10 -0
- data/bin/setup +8 -0
- data/exe/lm +4 -0
- data/lib/little_monster.rb +67 -0
- data/lib/little_monster/all.rb +4 -0
- data/lib/little_monster/config.rb +29 -0
- data/lib/little_monster/core.rb +20 -0
- data/lib/little_monster/core/api.rb +74 -0
- data/lib/little_monster/core/counters.rb +50 -0
- data/lib/little_monster/core/errors/api_unreachable_error.rb +4 -0
- data/lib/little_monster/core/errors/callback_failed_error.rb +4 -0
- data/lib/little_monster/core/errors/cancel_error.rb +4 -0
- data/lib/little_monster/core/errors/fatal_task_error.rb +4 -0
- data/lib/little_monster/core/errors/job_already_locked_error.rb +4 -0
- data/lib/little_monster/core/errors/job_not_found_error.rb +15 -0
- data/lib/little_monster/core/errors/job_retry_error.rb +4 -0
- data/lib/little_monster/core/errors/max_retries_error.rb +4 -0
- data/lib/little_monster/core/errors/task_error.rb +4 -0
- data/lib/little_monster/core/job.rb +188 -0
- data/lib/little_monster/core/job_data.rb +47 -0
- data/lib/little_monster/core/job_factory.rb +139 -0
- data/lib/little_monster/core/job_orchrestator.rb +194 -0
- data/lib/little_monster/core/loggable.rb +7 -0
- data/lib/little_monster/core/runner.rb +39 -0
- data/lib/little_monster/core/tagged_logger.rb +66 -0
- data/lib/little_monster/core/task.rb +41 -0
- data/lib/little_monster/generators/cli.rb +75 -0
- data/lib/little_monster/generators/conf_gen.rb +28 -0
- data/lib/little_monster/generators/generate.rb +35 -0
- data/lib/little_monster/generators/templates/config/application.rb +15 -0
- data/lib/little_monster/generators/templates/config/enviroments/development.rb +1 -0
- data/lib/little_monster/generators/templates/config/enviroments/production.rb +1 -0
- data/lib/little_monster/generators/templates/config/enviroments/test.rb +1 -0
- data/lib/little_monster/generators/templates/config/toiler.yml +3 -0
- data/lib/little_monster/generators/templates/jobs_spec_temp.erb +11 -0
- data/lib/little_monster/generators/templates/jobs_temp.erb +16 -0
- data/lib/little_monster/generators/templates/lib/.keep +0 -0
- data/lib/little_monster/generators/templates/log/.keep +0 -0
- data/lib/little_monster/generators/templates/spec_helper_temp.erb +22 -0
- data/lib/little_monster/generators/templates/tasks_spec_temp.erb +11 -0
- data/lib/little_monster/generators/templates/tasks_temp.erb +5 -0
- data/lib/little_monster/rspec.rb +20 -0
- data/lib/little_monster/rspec/helpers/job_helper.rb +61 -0
- data/lib/little_monster/rspec/helpers/task_helper.rb +46 -0
- data/lib/little_monster/rspec/matchers/have_data.rb +24 -0
- data/lib/little_monster/rspec/matchers/have_ended_with_status.rb +24 -0
- data/lib/little_monster/rspec/matchers/have_run.rb +28 -0
- data/lib/little_monster/rspec/matchers/have_run_task.rb +47 -0
- data/lib/little_monster/version.rb +3 -0
- data/lib/little_monster/worker.rb +27 -0
- data/little_monster.gemspec +48 -0
- metadata +343 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: a0a4e055747390da5a8510bda07ccc837d0ee70f
|
4
|
+
data.tar.gz: f4fde5e129ab06b38e4ed5c72cf81d72097a028d
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: bd386336c5f12c4338bc6890ce38df6ea3b98d1771690cbe3bd17ced27491402ffb55498c8c71366ad54020da968da83ee6d7fc6b9a84ea219d803d33fe968c9
|
7
|
+
data.tar.gz: a9fc3185fe91e35cce693dd63a4e56923fc8c8211a9ba3dfaa1f52d135089ed70ff1907b2b0a042e813790cc302e6331b526155c1cb90e9364e68a443fa103f7
|
data/.codeclimate.yml
ADDED
data/.gitignore
ADDED
data/.rubocop.yml
ADDED
@@ -0,0 +1,34 @@
|
|
1
|
+
Documentation:
|
2
|
+
Enabled: false
|
3
|
+
|
4
|
+
Style/FrozenStringLiteralComment:
|
5
|
+
Description: >-
|
6
|
+
Add the frozen_string_literal comment to the top of files
|
7
|
+
to help transition from Ruby 2.3.0 to Ruby 3.0.
|
8
|
+
Enabled: false
|
9
|
+
|
10
|
+
Style/ClassAndModuleChildren:
|
11
|
+
Enabled: false
|
12
|
+
|
13
|
+
Metrics/LineLength:
|
14
|
+
Max: 120
|
15
|
+
|
16
|
+
MethodLength:
|
17
|
+
Max: 20
|
18
|
+
|
19
|
+
Metrics/AbcSize:
|
20
|
+
Max: 40
|
21
|
+
|
22
|
+
Metrics/ClassLength:
|
23
|
+
CountComments: false
|
24
|
+
Max: 200
|
25
|
+
|
26
|
+
AllCops:
|
27
|
+
Exclude:
|
28
|
+
- 'vendor/**/*'
|
29
|
+
- 'spec/**/*'
|
30
|
+
- 'tmp/**/*'
|
31
|
+
- 'config/**/*'
|
32
|
+
- 'bin/**'
|
33
|
+
- 'db/**/*'
|
34
|
+
TargetRubyVersion: 2.3
|
data/.ruby-version
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
2.3.0
|
data/.travis.yml
ADDED
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
@@ -0,0 +1,124 @@
|
|
1
|
+
PATH
|
2
|
+
remote: .
|
3
|
+
specs:
|
4
|
+
little_monster (0.1.0)
|
5
|
+
activesupport
|
6
|
+
multi_json
|
7
|
+
thor
|
8
|
+
tilt
|
9
|
+
toiler
|
10
|
+
typhoeus
|
11
|
+
vcr
|
12
|
+
webmock
|
13
|
+
|
14
|
+
GEM
|
15
|
+
remote: https://rubygems.org/
|
16
|
+
specs:
|
17
|
+
activesupport (5.0.0)
|
18
|
+
concurrent-ruby (~> 1.0, >= 1.0.2)
|
19
|
+
i18n (~> 0.7)
|
20
|
+
minitest (~> 5.1)
|
21
|
+
tzinfo (~> 1.1)
|
22
|
+
addressable (2.4.0)
|
23
|
+
ast (2.3.0)
|
24
|
+
aws-sdk (2.5.2)
|
25
|
+
aws-sdk-resources (= 2.5.2)
|
26
|
+
aws-sdk-core (2.5.2)
|
27
|
+
jmespath (~> 1.0)
|
28
|
+
aws-sdk-resources (2.5.2)
|
29
|
+
aws-sdk-core (= 2.5.2)
|
30
|
+
byebug (9.0.5)
|
31
|
+
codeclimate-test-reporter (0.6.0)
|
32
|
+
simplecov (>= 0.7.1, < 1.0.0)
|
33
|
+
coderay (1.1.1)
|
34
|
+
concurrent-ruby (1.0.2)
|
35
|
+
concurrent-ruby-edge (0.2.2)
|
36
|
+
concurrent-ruby (~> 1.0.2)
|
37
|
+
crack (0.4.3)
|
38
|
+
safe_yaml (~> 1.0.0)
|
39
|
+
diff-lcs (1.2.5)
|
40
|
+
docile (1.1.5)
|
41
|
+
ethon (0.9.0)
|
42
|
+
ffi (>= 1.3.0)
|
43
|
+
ffi (1.9.14)
|
44
|
+
hashdiff (0.3.0)
|
45
|
+
i18n (0.7.0)
|
46
|
+
jmespath (1.3.1)
|
47
|
+
json (2.0.2)
|
48
|
+
method_source (0.8.2)
|
49
|
+
minitest (5.9.0)
|
50
|
+
multi_json (1.12.1)
|
51
|
+
oj (2.17.1)
|
52
|
+
parser (2.3.1.2)
|
53
|
+
ast (~> 2.2)
|
54
|
+
powerpack (0.1.1)
|
55
|
+
pry (0.10.4)
|
56
|
+
coderay (~> 1.1.0)
|
57
|
+
method_source (~> 0.8.1)
|
58
|
+
slop (~> 3.4)
|
59
|
+
rainbow (2.1.0)
|
60
|
+
rake (11.2.2)
|
61
|
+
require_all (1.3.3)
|
62
|
+
rspec (3.5.0)
|
63
|
+
rspec-core (~> 3.5.0)
|
64
|
+
rspec-expectations (~> 3.5.0)
|
65
|
+
rspec-mocks (~> 3.5.0)
|
66
|
+
rspec-core (3.5.2)
|
67
|
+
rspec-support (~> 3.5.0)
|
68
|
+
rspec-expectations (3.5.0)
|
69
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
70
|
+
rspec-support (~> 3.5.0)
|
71
|
+
rspec-mocks (3.5.0)
|
72
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
73
|
+
rspec-support (~> 3.5.0)
|
74
|
+
rspec-support (3.5.0)
|
75
|
+
rubocop (0.42.0)
|
76
|
+
parser (>= 2.3.1.1, < 3.0)
|
77
|
+
powerpack (~> 0.1)
|
78
|
+
rainbow (>= 1.99.1, < 3.0)
|
79
|
+
ruby-progressbar (~> 1.7)
|
80
|
+
unicode-display_width (~> 1.0, >= 1.0.1)
|
81
|
+
ruby-progressbar (1.8.1)
|
82
|
+
safe_yaml (1.0.4)
|
83
|
+
simplecov (0.12.0)
|
84
|
+
docile (~> 1.1.0)
|
85
|
+
json (>= 1.8, < 3)
|
86
|
+
simplecov-html (~> 0.10.0)
|
87
|
+
simplecov-html (0.10.0)
|
88
|
+
slop (3.6.0)
|
89
|
+
thor (0.19.1)
|
90
|
+
thread_safe (0.3.5)
|
91
|
+
tilt (2.0.5)
|
92
|
+
toiler (0.4.1)
|
93
|
+
aws-sdk (~> 2.2, >= 2.2.10)
|
94
|
+
concurrent-ruby (~> 1.0, >= 1.0.0)
|
95
|
+
concurrent-ruby-edge (~> 0.2.0, >= 0.2.0)
|
96
|
+
typhoeus (1.1.0)
|
97
|
+
ethon (>= 0.9.0)
|
98
|
+
tzinfo (1.2.2)
|
99
|
+
thread_safe (~> 0.1)
|
100
|
+
unicode-display_width (1.1.0)
|
101
|
+
vcr (3.0.3)
|
102
|
+
webmock (2.1.0)
|
103
|
+
addressable (>= 2.3.6)
|
104
|
+
crack (>= 0.3.2)
|
105
|
+
hashdiff
|
106
|
+
|
107
|
+
PLATFORMS
|
108
|
+
ruby
|
109
|
+
|
110
|
+
DEPENDENCIES
|
111
|
+
bundler
|
112
|
+
byebug
|
113
|
+
codeclimate-test-reporter
|
114
|
+
little_monster!
|
115
|
+
oj
|
116
|
+
pry
|
117
|
+
rake
|
118
|
+
require_all
|
119
|
+
rspec
|
120
|
+
rubocop
|
121
|
+
simplecov
|
122
|
+
|
123
|
+
BUNDLED WITH
|
124
|
+
1.12.5
|
data/README.md
ADDED
@@ -0,0 +1,5 @@
|
|
1
|
+
[![Build Status](https://travis-ci.org/mercadolibre/fury-little_monster-gem.svg?branch=travis-ci)](https://travis-ci.org/mercadolibre/fury-little_monster-gem)
|
2
|
+
[![Test Coverage](https://codeclimate.com/github/mercadolibre/fury-little_monster-gem/badges/coverage.svg)](https://codeclimate.com/github/mercadolibre/fury-little_monster-gem/coverage)
|
3
|
+
[![Code Climate](https://codeclimate.com/github/mercadolibre/fury-little_monster-gem/badges/gpa.svg)](https://codeclimate.com/github/mercadolibre/fury-little_monster-gem)
|
4
|
+
# fury-little_monster
|
5
|
+
|
data/Rakefile
ADDED
data/bin/console
ADDED
@@ -0,0 +1,10 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'bundler/setup'
|
4
|
+
require 'little_monster'
|
5
|
+
|
6
|
+
# You can add fixtures and/or initialization code here to make experimenting
|
7
|
+
# with your gem easier. You can also use a different console, if you like.
|
8
|
+
|
9
|
+
require 'pry'
|
10
|
+
Pry.start
|
data/bin/setup
ADDED
data/exe/lm
ADDED
@@ -0,0 +1,67 @@
|
|
1
|
+
require 'active_support/all'
|
2
|
+
require 'multi_json'
|
3
|
+
require 'toiler'
|
4
|
+
require 'little_monster/config'
|
5
|
+
require 'little_monster/core'
|
6
|
+
|
7
|
+
module LittleMonster
|
8
|
+
include LittleMonster::Core
|
9
|
+
|
10
|
+
module_function
|
11
|
+
|
12
|
+
def init
|
13
|
+
@@config = Config.new default_config_values
|
14
|
+
|
15
|
+
@@env = ActiveSupport::StringInquirer.new(ENV['LITTLE_MONSTER_ENV'] || ENV['RUBY_ENV'] || 'development')
|
16
|
+
|
17
|
+
@@logger = @@env.test? ? Logger.new('/dev/null') : Toiler.logger
|
18
|
+
|
19
|
+
@@logger.formatter = proc do |severity, datetime, _progname, msg|
|
20
|
+
"[severity:#{severity}][origin_datetime:#{datetime.strftime('%Y-%m-%d %H:%M:%S:%L %z')}] : #{msg}\n"
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
def env
|
25
|
+
@@env
|
26
|
+
end
|
27
|
+
|
28
|
+
def disable_requests?
|
29
|
+
%w(development test).include?(@@env)
|
30
|
+
end
|
31
|
+
|
32
|
+
def configure
|
33
|
+
yield @@config
|
34
|
+
# it calls update_attributes so it can refresh and concurrency
|
35
|
+
Worker.update_attributes
|
36
|
+
end
|
37
|
+
|
38
|
+
def default_config_values
|
39
|
+
{
|
40
|
+
api_url: 'http://little_monster_api_url.com/',
|
41
|
+
worker_concurrency: 200,
|
42
|
+
worker_queue: nil,
|
43
|
+
request_timeout: 3,
|
44
|
+
default_request_retries: 4,
|
45
|
+
default_request_retry_wait: 1,
|
46
|
+
task_requests_retries: 4,
|
47
|
+
task_requests_retry_wait: 1,
|
48
|
+
job_requests_retries: 4,
|
49
|
+
job_requests_retry_wait: 1,
|
50
|
+
heartbeat_execution_interval: 10
|
51
|
+
}
|
52
|
+
end
|
53
|
+
|
54
|
+
def logger
|
55
|
+
@@logger
|
56
|
+
end
|
57
|
+
|
58
|
+
def method_missing(method, *args, &block)
|
59
|
+
return @@config.public_send(method) if @@config.respond_to? method
|
60
|
+
super method, *args, &block
|
61
|
+
end
|
62
|
+
end
|
63
|
+
|
64
|
+
LittleMonster.init
|
65
|
+
|
66
|
+
# once all the core and configs were loaded we can require the worker
|
67
|
+
require 'little_monster/worker'
|
@@ -0,0 +1,29 @@
|
|
1
|
+
module LittleMonster
|
2
|
+
class Config
|
3
|
+
attr_accessor :api_url
|
4
|
+
|
5
|
+
attr_accessor :worker_concurrency
|
6
|
+
attr_accessor :worker_queue
|
7
|
+
|
8
|
+
attr_accessor :formatter
|
9
|
+
|
10
|
+
attr_accessor :request_timeout
|
11
|
+
|
12
|
+
attr_accessor :default_request_retries
|
13
|
+
attr_accessor :default_request_retry_wait
|
14
|
+
|
15
|
+
attr_accessor :task_requests_retries
|
16
|
+
attr_accessor :task_requests_retry_wait
|
17
|
+
|
18
|
+
attr_accessor :job_requests_retries
|
19
|
+
attr_accessor :job_requests_retry_wait
|
20
|
+
|
21
|
+
attr_accessor :heartbeat_execution_interval
|
22
|
+
|
23
|
+
def initialize(params = {})
|
24
|
+
params.to_hash.each do |key, value|
|
25
|
+
instance_variable_set("@#{key}", value)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
require 'little_monster/core/errors/task_error' # must be required first to satisfy dependencies
|
2
|
+
require 'little_monster/core/errors/cancel_error'
|
3
|
+
require 'little_monster/core/errors/fatal_task_error'
|
4
|
+
require 'little_monster/core/errors/max_retries_error'
|
5
|
+
require 'little_monster/core/errors/job_not_found_error'
|
6
|
+
require 'little_monster/core/errors/job_retry_error'
|
7
|
+
require 'little_monster/core/errors/api_unreachable_error'
|
8
|
+
require 'little_monster/core/errors/job_already_locked_error'
|
9
|
+
require 'little_monster/core/errors/callback_failed_error'
|
10
|
+
|
11
|
+
require 'little_monster/core/tagged_logger'
|
12
|
+
require 'little_monster/core/loggable' # must be required first to satisfy job and task dependencies
|
13
|
+
require 'little_monster/core/api'
|
14
|
+
require 'little_monster/core/job'
|
15
|
+
require 'little_monster/core/task'
|
16
|
+
require 'little_monster/core/job_factory'
|
17
|
+
require 'little_monster/core/job_data'
|
18
|
+
require 'little_monster/core/job_orchrestator'
|
19
|
+
require 'little_monster/core/runner'
|
20
|
+
require 'little_monster/core/counters'
|
@@ -0,0 +1,74 @@
|
|
1
|
+
require 'typhoeus'
|
2
|
+
require 'multi_json'
|
3
|
+
|
4
|
+
module LittleMonster::Core
|
5
|
+
class API
|
6
|
+
extend Loggable
|
7
|
+
|
8
|
+
class << self
|
9
|
+
def get(path, params = {}, options = {})
|
10
|
+
request :get, path, params, options
|
11
|
+
end
|
12
|
+
|
13
|
+
def post(path, params = {}, options = {})
|
14
|
+
request :post, path, params, options
|
15
|
+
end
|
16
|
+
|
17
|
+
def put(path, params = {}, options = {})
|
18
|
+
request :put, path, params, options
|
19
|
+
end
|
20
|
+
|
21
|
+
def patch(path, params = {}, options = {})
|
22
|
+
request :patch, path, params, options
|
23
|
+
end
|
24
|
+
|
25
|
+
def delete(path, params = {}, options = {})
|
26
|
+
request :delete, path, params, options
|
27
|
+
end
|
28
|
+
|
29
|
+
def request(method, path, params = {}, retries: LittleMonster.default_request_retries,
|
30
|
+
retry_wait: LittleMonster.default_request_retry_wait,
|
31
|
+
critical: false)
|
32
|
+
ret = 0
|
33
|
+
res = nil
|
34
|
+
url = [LittleMonster.api_url.chomp('/'), path.sub(/\//, '')].join '/'
|
35
|
+
|
36
|
+
params[:body] = MultiJson.dump params.fetch(:body, {}) unless params[:body].is_a? String
|
37
|
+
params[:headers] ||= {}
|
38
|
+
params[:headers]['Content-Type'] = 'application/json' unless params[:headers]['Content-Type']
|
39
|
+
params[:timeout] = LittleMonster.request_timeout
|
40
|
+
|
41
|
+
begin
|
42
|
+
res = Typhoeus.public_send method, url, params
|
43
|
+
if res.code >= 500 || res.code.zero?
|
44
|
+
raise FuryHttpApiError, "request to #{res.effective_url} failed with status #{res.code} retry #{ret}"
|
45
|
+
end
|
46
|
+
rescue StandardError => e
|
47
|
+
logger.error e.message
|
48
|
+
if ret < retries
|
49
|
+
sleep(retry_wait)
|
50
|
+
ret += 1
|
51
|
+
retry
|
52
|
+
end
|
53
|
+
|
54
|
+
logger.error "[type:request_max_retries_reached][url:#{url}][retries:#{ret}] request has reached max retries"
|
55
|
+
|
56
|
+
if critical
|
57
|
+
logger.error "[type:critical_request_failed][url:#{url}][retries:#{ret}] request has reached max retries"
|
58
|
+
raise APIUnreachableError, "critical request to #{url} has fail, check little monster api"
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
res.define_singleton_method(:body) do
|
63
|
+
begin
|
64
|
+
MultiJson.load(res.options[:response_body], symbolize_keys: true)
|
65
|
+
rescue
|
66
|
+
res.options[:response_body]
|
67
|
+
end
|
68
|
+
end
|
69
|
+
res
|
70
|
+
end
|
71
|
+
end
|
72
|
+
class FuryHttpApiError < StandardError; end
|
73
|
+
end
|
74
|
+
end
|