upperkut 0.1.2 → 0.1.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rspec +1 -0
- data/Gemfile +3 -3
- data/Gemfile.lock +2 -2
- data/README.md +21 -21
- data/Rakefile +3 -3
- data/bin/upperkut +1 -2
- data/examples/basic.rb +1 -1
- data/lib/upperkut/cli.rb +1 -2
- data/lib/upperkut/core_ext.rb +11 -10
- data/lib/upperkut/manager.rb +1 -4
- data/lib/upperkut/processor.rb +10 -12
- data/lib/upperkut/strategy.rb +6 -1
- data/lib/upperkut/util.rb +4 -4
- data/lib/upperkut/version.rb +1 -1
- data/lib/upperkut/worker.rb +2 -3
- data/lib/upperkut.rb +13 -10
- data/upperkut.gemspec +3 -3
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 63f529ffa09940fb3f2a582ffa78481ad7d89f92
|
4
|
+
data.tar.gz: 12053d8b945ffb8de49042d65900de5733f99b5a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 51c9e0e67d3382964a5f88172765f8c940ca71c6c4532bbc180a750636d68c110509469ab963bbeb1921e7b596b6bd5e6bd47d677f6dda2eec478d0f2c5766f2
|
7
|
+
data.tar.gz: 11144703a803d44935b71f896e2d423c7f51ecdc19b30468476e0fddc7751b1429df66d016300da92bf41a20c3766fefa523b8f84f60bffe60f9a0a54759a654
|
data/.rspec
CHANGED
data/Gemfile
CHANGED
@@ -1,9 +1,9 @@
|
|
1
|
-
source
|
1
|
+
source 'https://rubygems.org'
|
2
2
|
|
3
|
-
git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
|
3
|
+
git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
|
4
4
|
|
5
5
|
# Specify your gem's dependencies in upperkut.gemspec
|
6
6
|
gemspec
|
7
7
|
|
8
|
-
gem 'pry'
|
9
8
|
gem 'fakeredis'
|
9
|
+
gem 'pry'
|
data/Gemfile.lock
CHANGED
data/README.md
CHANGED
@@ -20,45 +20,45 @@ Or install it yourself as:
|
|
20
20
|
|
21
21
|
## Usage
|
22
22
|
Examples:
|
23
|
-
|
23
|
+
|
24
24
|
1) Create a Worker class and the define how to process the batch;
|
25
|
-
```ruby
|
25
|
+
```ruby
|
26
26
|
class MyWorker
|
27
27
|
include Upperkut::Worker
|
28
|
-
|
28
|
+
|
29
29
|
# This is optional
|
30
|
-
|
31
|
-
setup_upperkut do |
|
30
|
+
|
31
|
+
setup_upperkut do |config|
|
32
32
|
# Define which redis instance you want to use
|
33
|
-
|
34
|
-
|
33
|
+
config.redis = Redis.new(url: ENV['ANOTHER_REDIS_INSTANCE_URL'])
|
34
|
+
|
35
35
|
# Define the amount of items must be accumulated
|
36
|
-
|
37
|
-
|
36
|
+
config.batch_size = 2_000 # The default value is 1_000
|
37
|
+
|
38
38
|
# How frequent the Processor should hit redis looking for elegible
|
39
|
-
# batch. The default value is 5. You can also set the env
|
39
|
+
# batch. The default value is 5 seconds. You can also set the env
|
40
40
|
# UPPERKUT_POLLING_INTERVAL.
|
41
|
-
|
42
|
-
|
43
|
-
# How long the Processor should wait to process batch
|
44
|
-
# the amount of items did not reached the batch_size.
|
45
|
-
|
41
|
+
config.polling_interval = 4
|
42
|
+
|
43
|
+
# How long the Processor should wait in seconds to process batch
|
44
|
+
# even though the amount of items did not reached the batch_size.
|
45
|
+
config.max_wait = 300
|
46
46
|
end
|
47
|
-
|
47
|
+
|
48
48
|
def perform(batch_items)
|
49
49
|
SidekiqJobA.perform_async(batch_items)
|
50
50
|
SidekiqJobB.perform_async(batch_items)
|
51
|
-
|
51
|
+
|
52
52
|
process_metrics(batch_items)
|
53
53
|
end
|
54
54
|
end
|
55
55
|
```
|
56
|
-
|
56
|
+
|
57
57
|
2) Start pushings items;
|
58
|
-
```ruby
|
59
|
-
Myworker.
|
58
|
+
```ruby
|
59
|
+
Myworker.push_items([{'id' => SecureRandom.uuid, 'name' => 'Robert C Hall', 'action' => 'EMAIL_OPENNED'}])
|
60
60
|
```
|
61
|
-
|
61
|
+
|
62
62
|
3) Start Upperkut;
|
63
63
|
```bash
|
64
64
|
$ bundle exec upperkut --worker MyWorker --concurrency 10
|
data/Rakefile
CHANGED
data/bin/upperkut
CHANGED
data/examples/basic.rb
CHANGED
data/lib/upperkut/cli.rb
CHANGED
@@ -19,7 +19,7 @@ module Upperkut
|
|
19
19
|
manager = Manager.new(@options)
|
20
20
|
|
21
21
|
r, w = IO.pipe
|
22
|
-
signals = %w
|
22
|
+
signals = %w[INT TERM]
|
23
23
|
|
24
24
|
signals.each do |signal|
|
25
25
|
trap signal do
|
@@ -64,7 +64,6 @@ module Upperkut
|
|
64
64
|
o.on('-c', '--concurrency INT', 'Numbers of threads to spawn') do |arg|
|
65
65
|
@options[:concurrency] = Integer(arg)
|
66
66
|
end
|
67
|
-
|
68
67
|
end.parse!(args)
|
69
68
|
end
|
70
69
|
end
|
data/lib/upperkut/core_ext.rb
CHANGED
@@ -1,17 +1,18 @@
|
|
1
1
|
begin
|
2
2
|
require 'active_support/core_ext/string/inflections'
|
3
3
|
rescue LoadError
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
4
|
+
unless ''.respond_to?(:constantize)
|
5
|
+
class String
|
6
|
+
def constantize
|
7
|
+
names = split('::')
|
8
|
+
names.shift if names.empty? || names.first.empty?
|
8
9
|
|
9
|
-
|
10
|
-
|
11
|
-
|
10
|
+
constant = Object
|
11
|
+
names.each do |name|
|
12
|
+
constant = constant.const_defined?(name) ? constant.const_get(name) : constant.const_missing(name)
|
13
|
+
end
|
14
|
+
constant
|
12
15
|
end
|
13
|
-
constant
|
14
16
|
end
|
15
|
-
end
|
17
|
+
end
|
16
18
|
end
|
17
|
-
|
data/lib/upperkut/manager.rb
CHANGED
@@ -3,7 +3,6 @@ require_relative 'processor'
|
|
3
3
|
|
4
4
|
module Upperkut
|
5
5
|
class Manager
|
6
|
-
|
7
6
|
attr_accessor :worker, :redis
|
8
7
|
attr_reader :stopped
|
9
8
|
|
@@ -26,9 +25,7 @@ module Upperkut
|
|
26
25
|
end
|
27
26
|
|
28
27
|
def kill
|
29
|
-
@processors.each
|
30
|
-
processor.kill
|
31
|
-
end
|
28
|
+
@processors.each(&:kill)
|
32
29
|
end
|
33
30
|
end
|
34
31
|
end
|
data/lib/upperkut/processor.rb
CHANGED
@@ -1,6 +1,5 @@
|
|
1
1
|
module Upperkut
|
2
2
|
class Processor
|
3
|
-
|
4
3
|
def initialize(manager)
|
5
4
|
@manager = manager
|
6
5
|
@worker = @manager.worker
|
@@ -12,8 +11,9 @@ module Upperkut
|
|
12
11
|
process
|
13
12
|
end
|
14
13
|
end
|
14
|
+
|
15
15
|
def kill
|
16
|
-
return
|
16
|
+
return unless @thread
|
17
17
|
@thread.raise Upperkut::Shutdown
|
18
18
|
end
|
19
19
|
|
@@ -35,7 +35,7 @@ module Upperkut
|
|
35
35
|
buffer_size = @worker.size
|
36
36
|
|
37
37
|
return false if @manager.stopped
|
38
|
-
return false if buffer_size
|
38
|
+
return false if buffer_size.zero?
|
39
39
|
|
40
40
|
# TODO: rename #setup by config
|
41
41
|
buffer_size >= @worker.setup.batch_size ||
|
@@ -43,15 +43,13 @@ module Upperkut
|
|
43
43
|
end
|
44
44
|
|
45
45
|
def process_batch
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
raise ex
|
54
|
-
end
|
46
|
+
@sleeping_time = 0
|
47
|
+
@worker.new.process
|
48
|
+
rescue Exception => ex
|
49
|
+
# Add to retry_queue
|
50
|
+
# if retry_limit is reached
|
51
|
+
# send to dead
|
52
|
+
raise ex
|
55
53
|
end
|
56
54
|
end
|
57
55
|
end
|
data/lib/upperkut/strategy.rb
CHANGED
@@ -12,8 +12,9 @@ module Upperkut
|
|
12
12
|
end
|
13
13
|
|
14
14
|
def push_items(items = [])
|
15
|
+
items = [items] if items.is_a?(Hash)
|
15
16
|
return false if items.empty?
|
16
|
-
redis.
|
17
|
+
redis.rpush(key, encode_json_items(items))
|
17
18
|
end
|
18
19
|
|
19
20
|
def fetch_items(batch_size = 1000)
|
@@ -41,6 +42,10 @@ module Upperkut
|
|
41
42
|
lat
|
42
43
|
end
|
43
44
|
|
45
|
+
def clear
|
46
|
+
redis.del(key)
|
47
|
+
end
|
48
|
+
|
44
49
|
private
|
45
50
|
|
46
51
|
def key
|
data/lib/upperkut/util.rb
CHANGED
@@ -5,9 +5,9 @@ module Upperkut
|
|
5
5
|
def to_underscore(object)
|
6
6
|
klass_name = object
|
7
7
|
klass_name.gsub!(/::/, '_')
|
8
|
-
klass_name.gsub!(/([A-Z\d]+)([A-Z][a-z])/,'\1_\2')
|
9
|
-
klass_name.gsub!(/([a-z\d])([A-Z])/,'\1_\2')
|
10
|
-
klass_name.tr!(
|
8
|
+
klass_name.gsub!(/([A-Z\d]+)([A-Z][a-z])/, '\1_\2')
|
9
|
+
klass_name.gsub!(/([a-z\d])([A-Z])/, '\1_\2')
|
10
|
+
klass_name.tr!('-', '_')
|
11
11
|
klass_name.downcase!
|
12
12
|
klass_name
|
13
13
|
end
|
@@ -22,7 +22,7 @@ module Upperkut
|
|
22
22
|
end
|
23
23
|
|
24
24
|
def decode_json_items(items)
|
25
|
-
items.collect {|i| JSON.parse(i) }
|
25
|
+
items.collect { |i| JSON.parse(i) }
|
26
26
|
end
|
27
27
|
end
|
28
28
|
end
|
data/lib/upperkut/version.rb
CHANGED
data/lib/upperkut/worker.rb
CHANGED
@@ -5,7 +5,6 @@ require_relative '../upperkut'
|
|
5
5
|
|
6
6
|
module Upperkut
|
7
7
|
module Worker
|
8
|
-
|
9
8
|
def self.included(base)
|
10
9
|
base.extend(ClassMethods)
|
11
10
|
end
|
@@ -21,7 +20,7 @@ module Upperkut
|
|
21
20
|
extend Forwardable
|
22
21
|
|
23
22
|
def_delegators :setup, :strategy
|
24
|
-
def_delegators :strategy, :push_items, :size, :latency
|
23
|
+
def_delegators :strategy, :push_items, :size, :latency, :clear
|
25
24
|
|
26
25
|
def push_items(items)
|
27
26
|
strategy.push_items(items)
|
@@ -31,7 +30,7 @@ module Upperkut
|
|
31
30
|
strategy.fetch_items(setup.batch_size)
|
32
31
|
end
|
33
32
|
|
34
|
-
def setup_upperkut
|
33
|
+
def setup_upperkut
|
35
34
|
yield(setup) if block_given?
|
36
35
|
end
|
37
36
|
|
data/lib/upperkut.rb
CHANGED
@@ -13,21 +13,21 @@ require 'redis'
|
|
13
13
|
#
|
14
14
|
# # This is optional
|
15
15
|
#
|
16
|
-
# setup_upperkut do |
|
16
|
+
# setup_upperkut do |config|
|
17
17
|
# # Define which redis instance you want to use
|
18
|
-
#
|
18
|
+
# config.redis = Redis.new(url: ENV['ANOTHER_REDIS_INSTANCE_URL'])
|
19
19
|
#
|
20
20
|
# # Define the amount of items must be accumulated
|
21
|
-
#
|
21
|
+
# config.batch_size = 2_000 # The default value is 1_000
|
22
22
|
#
|
23
23
|
# # How frequent the Processor should hit redis looking for elegible
|
24
|
-
# # batch. The default value is 5. You can also set the env
|
24
|
+
# # batch. The default value is 5 seconds. You can also set the env
|
25
25
|
# # UPPERKUT_POLLING_INTERVAL.
|
26
|
-
#
|
26
|
+
# config.polling_interval = 4
|
27
27
|
#
|
28
|
-
# # How long the Processor should wait to process batch
|
29
|
-
# # the amount of items did not reached the batch_size.
|
30
|
-
#
|
28
|
+
# # How long the Processor should wait in seconds to process batch
|
29
|
+
# # even though the amount of items did not reached the batch_size.
|
30
|
+
# config.max_wait = 300
|
31
31
|
# end
|
32
32
|
#
|
33
33
|
# def perform(batch_items)
|
@@ -40,7 +40,9 @@ require 'redis'
|
|
40
40
|
#
|
41
41
|
# 2) Start pushings items;
|
42
42
|
#
|
43
|
-
# Myworker.
|
43
|
+
# Myworker.push_items(
|
44
|
+
# [{'id' => SecureRandom.uuid, 'name' => 'Robert C Hall', 'action' => 'EMAIL_OPENNED'}]
|
45
|
+
# )
|
44
46
|
#
|
45
47
|
# 3) Start Upperkut;
|
46
48
|
#
|
@@ -61,5 +63,6 @@ module Upperkut
|
|
61
63
|
end
|
62
64
|
end
|
63
65
|
|
64
|
-
class
|
66
|
+
# Error class responsible to signal the shutdown process
|
67
|
+
class Shutdown < StandardError; end
|
65
68
|
end
|
data/upperkut.gemspec
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
|
2
|
-
lib = File.expand_path('
|
2
|
+
lib = File.expand_path('lib', __dir__)
|
3
3
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
4
|
require 'upperkut/version'
|
5
5
|
|
@@ -9,8 +9,8 @@ Gem::Specification.new do |spec|
|
|
9
9
|
spec.authors = ['Nando Sousa']
|
10
10
|
spec.email = ['nandosousafr@gmail.com']
|
11
11
|
|
12
|
-
spec.summary =
|
13
|
-
spec.description =
|
12
|
+
spec.summary = 'Batch background processing tool'
|
13
|
+
spec.description = 'Batch background processing tool'
|
14
14
|
spec.homepage = 'http://shipit.resultadosdigitais.com.br/open-source/'
|
15
15
|
spec.license = 'MIT'
|
16
16
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: upperkut
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Nando Sousa
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-06-
|
11
|
+
date: 2018-06-25 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: redis
|
@@ -121,7 +121,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
121
121
|
version: '0'
|
122
122
|
requirements: []
|
123
123
|
rubyforge_project:
|
124
|
-
rubygems_version: 2.6.14
|
124
|
+
rubygems_version: 2.6.14.1
|
125
125
|
signing_key:
|
126
126
|
specification_version: 4
|
127
127
|
summary: Batch background processing tool
|