upperkut 0.1.4 → 0.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +5 -0
- data/Gemfile +0 -3
- data/Gemfile.lock +1 -14
- data/README.md +0 -4
- data/lib/upperkut/batch_execution.rb +29 -0
- data/lib/upperkut/cli.rb +18 -3
- data/lib/upperkut/logging.rb +38 -0
- data/lib/upperkut/manager.rb +15 -2
- data/lib/upperkut/processor.rb +17 -8
- data/lib/upperkut/version.rb +1 -1
- data/lib/upperkut/worker.rb +0 -7
- metadata +6 -4
- data/.circleci/config.yml +0 -63
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 86cdd27d527086fd910ae6065716239959e2948b
|
4
|
+
data.tar.gz: 2556072af6d5b2ba93dffdcccb3c2c8f7dd7da22
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 2d758d766b00c8a094d1a83a97a1632abfdbc6406867c47379e54876a837232c526be01bb4e1aea723852c6e0af8f248afae584f3ad053dbd8311f7bd1aec019
|
7
|
+
data.tar.gz: ced16ccf12783e7a23f4cb13405da13a9c1a860b6e0b607a1822ac9feb6f3545fb2cada50a930c9df04fe8e973cbef4962d5e7f773feb2457848565138cdc90d
|
data/.travis.yml
ADDED
data/Gemfile
CHANGED
data/Gemfile.lock
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
upperkut (0.
|
4
|
+
upperkut (0.3.0)
|
5
5
|
redis (>= 3.3.5, < 5)
|
6
6
|
|
7
7
|
GEM
|
@@ -9,11 +9,8 @@ GEM
|
|
9
9
|
specs:
|
10
10
|
coderay (1.1.2)
|
11
11
|
diff-lcs (1.3)
|
12
|
-
docile (1.3.1)
|
13
12
|
fakeredis (0.7.0)
|
14
13
|
redis (>= 3.2, < 5.0)
|
15
|
-
fivemat (1.3.6)
|
16
|
-
json (2.1.0)
|
17
14
|
method_source (0.9.0)
|
18
15
|
pry (0.11.3)
|
19
16
|
coderay (~> 1.1.0)
|
@@ -33,13 +30,6 @@ GEM
|
|
33
30
|
diff-lcs (>= 1.2.0, < 2.0)
|
34
31
|
rspec-support (~> 3.7.0)
|
35
32
|
rspec-support (3.7.1)
|
36
|
-
rspec_junit_formatter (0.4.1)
|
37
|
-
rspec-core (>= 2, < 4, != 2.12.0)
|
38
|
-
simplecov (0.16.1)
|
39
|
-
docile (~> 1.1)
|
40
|
-
json (>= 1.8, < 3)
|
41
|
-
simplecov-html (~> 0.10.0)
|
42
|
-
simplecov-html (0.10.2)
|
43
33
|
|
44
34
|
PLATFORMS
|
45
35
|
ruby
|
@@ -47,12 +37,9 @@ PLATFORMS
|
|
47
37
|
DEPENDENCIES
|
48
38
|
bundler (~> 1.16)
|
49
39
|
fakeredis
|
50
|
-
fivemat
|
51
40
|
pry
|
52
41
|
rake (~> 10.0)
|
53
42
|
rspec (~> 3.0)
|
54
|
-
rspec_junit_formatter
|
55
|
-
simplecov
|
56
43
|
upperkut!
|
57
44
|
|
58
45
|
BUNDLED WITH
|
data/README.md
CHANGED
@@ -1,9 +1,5 @@
|
|
1
1
|
# Upperkut
|
2
2
|
|
3
|
-
[![CircleCI](https://circleci.com/gh/ResultadosDigitais/upperkut/tree/master.svg?style=svg&circle-token=693e512de6985be3b3db12279ba6ed508fb5c6f6)](https://circleci.com/gh/ResultadosDigitais/upperkut/tree/master)
|
4
|
-
[![Maintainability](https://api.codeclimate.com/v1/badges/ece40319b0db03af891d/maintainability)](https://codeclimate.com/repos/5b318a7c6d37b70272008676/maintainability)
|
5
|
-
[![Test Coverage](https://api.codeclimate.com/v1/badges/ece40319b0db03af891d/test_coverage)](https://codeclimate.com/repos/5b318a7c6d37b70272008676/test_coverage)
|
6
|
-
|
7
3
|
Batch background processing tool.
|
8
4
|
|
9
5
|
## Installation
|
@@ -0,0 +1,29 @@
|
|
1
|
+
require_relative 'logging'
|
2
|
+
|
3
|
+
module Upperkut
|
4
|
+
class BatchExecution
|
5
|
+
def initialize(worker, logger = Upperkut::Logging.logger)
|
6
|
+
@worker = worker
|
7
|
+
@logger = logger
|
8
|
+
end
|
9
|
+
|
10
|
+
def execute
|
11
|
+
worker_instance = @worker.new
|
12
|
+
items = @worker.fetch_items.collect! do |item|
|
13
|
+
item['body']
|
14
|
+
end
|
15
|
+
|
16
|
+
worker_instance.perform(items.dup)
|
17
|
+
rescue Exception => ex
|
18
|
+
@worker.push_items(items)
|
19
|
+
|
20
|
+
@logger.info(
|
21
|
+
action: :requeue,
|
22
|
+
ex: ex,
|
23
|
+
item_size: items.size
|
24
|
+
)
|
25
|
+
|
26
|
+
raise ex
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
data/lib/upperkut/cli.rb
CHANGED
@@ -1,14 +1,15 @@
|
|
1
1
|
require 'optparse'
|
2
2
|
require_relative '../upperkut'
|
3
3
|
require_relative 'manager'
|
4
|
+
require_relative 'logging'
|
4
5
|
|
5
6
|
module Upperkut
|
6
7
|
class CLI
|
7
8
|
def initialize(args = ARGV)
|
8
9
|
@options = {}
|
9
|
-
|
10
|
+
@logger = Upperkut::Logging.logger
|
10
11
|
|
11
|
-
|
12
|
+
parse_options(args)
|
12
13
|
end
|
13
14
|
|
14
15
|
def start
|
@@ -16,8 +17,16 @@ module Upperkut
|
|
16
17
|
require file
|
17
18
|
end
|
18
19
|
|
20
|
+
if log_level = @options[:log_level]
|
21
|
+
@logger.level = log_level
|
22
|
+
end
|
23
|
+
|
24
|
+
@options[:logger] = @logger
|
25
|
+
|
19
26
|
manager = Manager.new(@options)
|
20
27
|
|
28
|
+
@logger.info(@options)
|
29
|
+
|
21
30
|
r, w = IO.pipe
|
22
31
|
signals = %w[INT TERM]
|
23
32
|
|
@@ -34,7 +43,10 @@ module Upperkut
|
|
34
43
|
handle_signal(signal)
|
35
44
|
end
|
36
45
|
rescue Interrupt
|
37
|
-
|
46
|
+
@logger.info(
|
47
|
+
'Stopping managers, wait for 5 seconds and them kill processors'
|
48
|
+
)
|
49
|
+
|
38
50
|
manager.stop
|
39
51
|
sleep(5)
|
40
52
|
manager.kill
|
@@ -64,6 +76,9 @@ module Upperkut
|
|
64
76
|
o.on('-c', '--concurrency INT', 'Numbers of threads to spawn') do |arg|
|
65
77
|
@options[:concurrency] = Integer(arg)
|
66
78
|
end
|
79
|
+
o.on('-l', '--log-level LEVEL', 'Log level') do |arg|
|
80
|
+
@options[:log_level] = arg.to_i
|
81
|
+
end
|
67
82
|
end.parse!(args)
|
68
83
|
end
|
69
84
|
end
|
@@ -0,0 +1,38 @@
|
|
1
|
+
require 'logger'
|
2
|
+
require 'time'
|
3
|
+
require 'socket'
|
4
|
+
|
5
|
+
module Upperkut
|
6
|
+
module Logging
|
7
|
+
|
8
|
+
class DefaultFormatter < Logger::Formatter
|
9
|
+
def call(severity, time, program_name, message)
|
10
|
+
"upperkut: #{time.utc.iso8601(3)} hostname=#{Socket.gethostname} "\
|
11
|
+
"pid=#{::Process.pid} severity=#{severity} #{format_message(message)}\n"
|
12
|
+
end
|
13
|
+
|
14
|
+
private
|
15
|
+
|
16
|
+
def format_message(message)
|
17
|
+
return "msg=#{message} " unless message.is_a?(Hash)
|
18
|
+
|
19
|
+
message.each_with_object('') do |(k,v), memo|
|
20
|
+
memo << "#{k}=#{v}\s"
|
21
|
+
memo
|
22
|
+
end
|
23
|
+
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def self.initialize_logger
|
28
|
+
logger = Logger.new($stdout)
|
29
|
+
logger.level = Logger::INFO
|
30
|
+
logger.formatter = DefaultFormatter.new
|
31
|
+
logger
|
32
|
+
end
|
33
|
+
|
34
|
+
def self.logger
|
35
|
+
@logger ||= initialize_logger
|
36
|
+
end
|
37
|
+
end
|
38
|
+
end
|
data/lib/upperkut/manager.rb
CHANGED
@@ -4,19 +4,23 @@ require_relative 'processor'
|
|
4
4
|
module Upperkut
|
5
5
|
class Manager
|
6
6
|
attr_accessor :worker, :redis
|
7
|
-
attr_reader :stopped
|
7
|
+
attr_reader :stopped, :logger
|
8
8
|
|
9
9
|
def initialize(opts = {})
|
10
10
|
self.worker = opts.fetch(:worker).constantize
|
11
11
|
self.redis = worker.setup.redis
|
12
12
|
@concurrency = opts.fetch(:concurrency, 25)
|
13
|
+
@logger = opts.fetch(:logger, Upperkut::Logging.logger)
|
14
|
+
|
13
15
|
@stopped = false
|
14
16
|
@processors = []
|
15
17
|
end
|
16
18
|
|
17
19
|
def run
|
18
20
|
@concurrency.times do
|
19
|
-
|
21
|
+
processor = Processor.new(self)
|
22
|
+
@processors << processor
|
23
|
+
processor.run
|
20
24
|
end
|
21
25
|
end
|
22
26
|
|
@@ -27,5 +31,14 @@ module Upperkut
|
|
27
31
|
def kill
|
28
32
|
@processors.each(&:kill)
|
29
33
|
end
|
34
|
+
|
35
|
+
def notify_killed_processor(processor)
|
36
|
+
@processors.delete(processor)
|
37
|
+
return if @stopped
|
38
|
+
|
39
|
+
processor = Processor.new(self)
|
40
|
+
@processors << processor
|
41
|
+
processor.run
|
42
|
+
end
|
30
43
|
end
|
31
44
|
end
|
data/lib/upperkut/processor.rb
CHANGED
@@ -1,20 +1,34 @@
|
|
1
|
+
require_relative 'batch_execution'
|
2
|
+
|
1
3
|
module Upperkut
|
2
4
|
class Processor
|
3
5
|
def initialize(manager)
|
4
6
|
@manager = manager
|
5
7
|
@worker = @manager.worker
|
8
|
+
@logger = @manager.logger
|
9
|
+
|
6
10
|
@sleeping_time = 0
|
7
11
|
end
|
8
12
|
|
9
13
|
def run
|
10
14
|
@thread ||= Thread.new do
|
11
|
-
|
15
|
+
begin
|
16
|
+
process
|
17
|
+
rescue Exception => e
|
18
|
+
@logger.debug(
|
19
|
+
action: :processor_killed,
|
20
|
+
reason: e
|
21
|
+
)
|
22
|
+
|
23
|
+
@manager.notify_killed_processor(self)
|
24
|
+
end
|
12
25
|
end
|
13
26
|
end
|
14
27
|
|
15
28
|
def kill
|
16
29
|
return unless @thread
|
17
30
|
@thread.raise Upperkut::Shutdown
|
31
|
+
@thread.value # wait
|
18
32
|
end
|
19
33
|
|
20
34
|
private
|
@@ -28,6 +42,7 @@ module Upperkut
|
|
28
42
|
end
|
29
43
|
|
30
44
|
@sleeping_time += sleep(@worker.setup.polling_interval)
|
45
|
+
@logger.debug(sleeping_time: @sleeping_time)
|
31
46
|
end
|
32
47
|
end
|
33
48
|
|
@@ -43,13 +58,7 @@ module Upperkut
|
|
43
58
|
end
|
44
59
|
|
45
60
|
def process_batch
|
46
|
-
@
|
47
|
-
@worker.new.process
|
48
|
-
rescue Exception => ex
|
49
|
-
# Add to retry_queue
|
50
|
-
# if retry_limit is reached
|
51
|
-
# send to dead
|
52
|
-
raise ex
|
61
|
+
BatchExecution.new(@worker, @logger).execute
|
53
62
|
end
|
54
63
|
end
|
55
64
|
end
|
data/lib/upperkut/version.rb
CHANGED
data/lib/upperkut/worker.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: upperkut
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Nando Sousa
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-06-
|
11
|
+
date: 2018-06-28 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: redis
|
@@ -80,9 +80,9 @@ executables:
|
|
80
80
|
extensions: []
|
81
81
|
extra_rdoc_files: []
|
82
82
|
files:
|
83
|
-
- ".circleci/config.yml"
|
84
83
|
- ".gitignore"
|
85
84
|
- ".rspec"
|
85
|
+
- ".travis.yml"
|
86
86
|
- CODE_OF_CONDUCT.md
|
87
87
|
- Gemfile
|
88
88
|
- Gemfile.lock
|
@@ -92,8 +92,10 @@ files:
|
|
92
92
|
- bin/upperkut
|
93
93
|
- examples/basic.rb
|
94
94
|
- lib/upperkut.rb
|
95
|
+
- lib/upperkut/batch_execution.rb
|
95
96
|
- lib/upperkut/cli.rb
|
96
97
|
- lib/upperkut/core_ext.rb
|
98
|
+
- lib/upperkut/logging.rb
|
97
99
|
- lib/upperkut/manager.rb
|
98
100
|
- lib/upperkut/processor.rb
|
99
101
|
- lib/upperkut/strategy.rb
|
@@ -121,7 +123,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
121
123
|
version: '0'
|
122
124
|
requirements: []
|
123
125
|
rubyforge_project:
|
124
|
-
rubygems_version: 2.6.14
|
126
|
+
rubygems_version: 2.6.14
|
125
127
|
signing_key:
|
126
128
|
specification_version: 4
|
127
129
|
summary: Batch background processing tool
|
data/.circleci/config.yml
DELETED
@@ -1,63 +0,0 @@
|
|
1
|
-
# Ruby CircleCI 2.0 configuration file
|
2
|
-
#
|
3
|
-
# Check https://circleci.com/docs/2.0/language-ruby/ for more details
|
4
|
-
#
|
5
|
-
version: 2
|
6
|
-
jobs:
|
7
|
-
build:
|
8
|
-
docker:
|
9
|
-
- image: circleci/ruby:2.4.4
|
10
|
-
environment:
|
11
|
-
CC_TEST_REPORTER_ID: 03ab83a772148a577d29d4acf438d7ebdc95c632224122d0ba8dbb291eedebe6
|
12
|
-
COVERAGE: true
|
13
|
-
steps:
|
14
|
-
- checkout
|
15
|
-
|
16
|
-
# Download and cache dependencies
|
17
|
-
- restore_cache:
|
18
|
-
keys:
|
19
|
-
- v1-dependencies-{{ checksum "Gemfile.lock" }}
|
20
|
-
# fallback to using the latest cache if no exact match is found
|
21
|
-
- v1-dependencies-
|
22
|
-
|
23
|
-
- run:
|
24
|
-
name: install dependencies
|
25
|
-
command: |
|
26
|
-
bundle install --jobs=4 --retry=3 --path vendor/bundle
|
27
|
-
|
28
|
-
- save_cache:
|
29
|
-
paths:
|
30
|
-
- ./vendor/bundle
|
31
|
-
key: v1-dependencies-{{ checksum "Gemfile.lock" }}
|
32
|
-
|
33
|
-
- run:
|
34
|
-
name: Setup Code Climate test-reporter
|
35
|
-
command: |
|
36
|
-
curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
|
37
|
-
chmod +x ./cc-test-reporter
|
38
|
-
./cc-test-reporter before-build
|
39
|
-
|
40
|
-
# run tests!
|
41
|
-
- run:
|
42
|
-
name: run tests
|
43
|
-
command: |
|
44
|
-
mkdir /tmp/test-results
|
45
|
-
TEST_FILES="$(circleci tests glob "spec/**/*_spec.rb" | circleci tests split --split-by=timings)"
|
46
|
-
|
47
|
-
bundle exec rspec --format progress \
|
48
|
-
--format RspecJunitFormatter \
|
49
|
-
--out /tmp/test-results/rspec.xml \
|
50
|
-
--format progress \
|
51
|
-
$TEST_FILES
|
52
|
-
- run:
|
53
|
-
name: Submit coverage
|
54
|
-
command: |
|
55
|
-
./cc-test-reporter after-build --coverage-input-type simplecov --exit-code $?
|
56
|
-
|
57
|
-
# collect reports
|
58
|
-
- store_test_results:
|
59
|
-
path: /tmp/test-results
|
60
|
-
|
61
|
-
- store_artifacts:
|
62
|
-
path: /tmp/test-results
|
63
|
-
destination: test-results
|