upperkut 0.1.2 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rspec +1 -0
- data/Gemfile +3 -3
- data/Gemfile.lock +2 -2
- data/README.md +21 -21
- data/Rakefile +3 -3
- data/bin/upperkut +1 -2
- data/examples/basic.rb +1 -1
- data/lib/upperkut/cli.rb +1 -2
- data/lib/upperkut/core_ext.rb +11 -10
- data/lib/upperkut/manager.rb +1 -4
- data/lib/upperkut/processor.rb +10 -12
- data/lib/upperkut/strategy.rb +6 -1
- data/lib/upperkut/util.rb +4 -4
- data/lib/upperkut/version.rb +1 -1
- data/lib/upperkut/worker.rb +2 -3
- data/lib/upperkut.rb +13 -10
- data/upperkut.gemspec +3 -3
- metadata +3 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 63f529ffa09940fb3f2a582ffa78481ad7d89f92
|
4
|
+
data.tar.gz: 12053d8b945ffb8de49042d65900de5733f99b5a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 51c9e0e67d3382964a5f88172765f8c940ca71c6c4532bbc180a750636d68c110509469ab963bbeb1921e7b596b6bd5e6bd47d677f6dda2eec478d0f2c5766f2
|
7
|
+
data.tar.gz: 11144703a803d44935b71f896e2d423c7f51ecdc19b30468476e0fddc7751b1429df66d016300da92bf41a20c3766fefa523b8f84f60bffe60f9a0a54759a654
|
data/.rspec
CHANGED
data/Gemfile
CHANGED
@@ -1,9 +1,9 @@
|
|
1
|
-
source
|
1
|
+
source 'https://rubygems.org'
|
2
2
|
|
3
|
-
git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
|
3
|
+
git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
|
4
4
|
|
5
5
|
# Specify your gem's dependencies in upperkut.gemspec
|
6
6
|
gemspec
|
7
7
|
|
8
|
-
gem 'pry'
|
9
8
|
gem 'fakeredis'
|
9
|
+
gem 'pry'
|
data/Gemfile.lock
CHANGED
data/README.md
CHANGED
@@ -20,45 +20,45 @@ Or install it yourself as:
|
|
20
20
|
|
21
21
|
## Usage
|
22
22
|
Examples:
|
23
|
-
|
23
|
+
|
24
24
|
1) Create a Worker class and the define how to process the batch;
|
25
|
-
```ruby
|
25
|
+
```ruby
|
26
26
|
class MyWorker
|
27
27
|
include Upperkut::Worker
|
28
|
-
|
28
|
+
|
29
29
|
# This is optional
|
30
|
-
|
31
|
-
setup_upperkut do |
|
30
|
+
|
31
|
+
setup_upperkut do |config|
|
32
32
|
# Define which redis instance you want to use
|
33
|
-
|
34
|
-
|
33
|
+
config.redis = Redis.new(url: ENV['ANOTHER_REDIS_INSTANCE_URL'])
|
34
|
+
|
35
35
|
# Define the amount of items must be accumulated
|
36
|
-
|
37
|
-
|
36
|
+
config.batch_size = 2_000 # The default value is 1_000
|
37
|
+
|
38
38
|
# How frequent the Processor should hit redis looking for elegible
|
39
|
-
# batch. The default value is 5. You can also set the env
|
39
|
+
# batch. The default value is 5 seconds. You can also set the env
|
40
40
|
# UPPERKUT_POLLING_INTERVAL.
|
41
|
-
|
42
|
-
|
43
|
-
# How long the Processor should wait to process batch
|
44
|
-
# the amount of items did not reached the batch_size.
|
45
|
-
|
41
|
+
config.polling_interval = 4
|
42
|
+
|
43
|
+
# How long the Processor should wait in seconds to process batch
|
44
|
+
# even though the amount of items did not reached the batch_size.
|
45
|
+
config.max_wait = 300
|
46
46
|
end
|
47
|
-
|
47
|
+
|
48
48
|
def perform(batch_items)
|
49
49
|
SidekiqJobA.perform_async(batch_items)
|
50
50
|
SidekiqJobB.perform_async(batch_items)
|
51
|
-
|
51
|
+
|
52
52
|
process_metrics(batch_items)
|
53
53
|
end
|
54
54
|
end
|
55
55
|
```
|
56
|
-
|
56
|
+
|
57
57
|
2) Start pushings items;
|
58
|
-
```ruby
|
59
|
-
Myworker.
|
58
|
+
```ruby
|
59
|
+
Myworker.push_items([{'id' => SecureRandom.uuid, 'name' => 'Robert C Hall', 'action' => 'EMAIL_OPENNED'}])
|
60
60
|
```
|
61
|
-
|
61
|
+
|
62
62
|
3) Start Upperkut;
|
63
63
|
```bash
|
64
64
|
$ bundle exec upperkut --worker MyWorker --concurrency 10
|
data/Rakefile
CHANGED
data/bin/upperkut
CHANGED
data/examples/basic.rb
CHANGED
data/lib/upperkut/cli.rb
CHANGED
@@ -19,7 +19,7 @@ module Upperkut
|
|
19
19
|
manager = Manager.new(@options)
|
20
20
|
|
21
21
|
r, w = IO.pipe
|
22
|
-
signals = %w
|
22
|
+
signals = %w[INT TERM]
|
23
23
|
|
24
24
|
signals.each do |signal|
|
25
25
|
trap signal do
|
@@ -64,7 +64,6 @@ module Upperkut
|
|
64
64
|
o.on('-c', '--concurrency INT', 'Numbers of threads to spawn') do |arg|
|
65
65
|
@options[:concurrency] = Integer(arg)
|
66
66
|
end
|
67
|
-
|
68
67
|
end.parse!(args)
|
69
68
|
end
|
70
69
|
end
|
data/lib/upperkut/core_ext.rb
CHANGED
@@ -1,17 +1,18 @@
|
|
1
1
|
begin
|
2
2
|
require 'active_support/core_ext/string/inflections'
|
3
3
|
rescue LoadError
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
4
|
+
unless ''.respond_to?(:constantize)
|
5
|
+
class String
|
6
|
+
def constantize
|
7
|
+
names = split('::')
|
8
|
+
names.shift if names.empty? || names.first.empty?
|
8
9
|
|
9
|
-
|
10
|
-
|
11
|
-
|
10
|
+
constant = Object
|
11
|
+
names.each do |name|
|
12
|
+
constant = constant.const_defined?(name) ? constant.const_get(name) : constant.const_missing(name)
|
13
|
+
end
|
14
|
+
constant
|
12
15
|
end
|
13
|
-
constant
|
14
16
|
end
|
15
|
-
end
|
17
|
+
end
|
16
18
|
end
|
17
|
-
|
data/lib/upperkut/manager.rb
CHANGED
@@ -3,7 +3,6 @@ require_relative 'processor'
|
|
3
3
|
|
4
4
|
module Upperkut
|
5
5
|
class Manager
|
6
|
-
|
7
6
|
attr_accessor :worker, :redis
|
8
7
|
attr_reader :stopped
|
9
8
|
|
@@ -26,9 +25,7 @@ module Upperkut
|
|
26
25
|
end
|
27
26
|
|
28
27
|
def kill
|
29
|
-
@processors.each
|
30
|
-
processor.kill
|
31
|
-
end
|
28
|
+
@processors.each(&:kill)
|
32
29
|
end
|
33
30
|
end
|
34
31
|
end
|
data/lib/upperkut/processor.rb
CHANGED
@@ -1,6 +1,5 @@
|
|
1
1
|
module Upperkut
|
2
2
|
class Processor
|
3
|
-
|
4
3
|
def initialize(manager)
|
5
4
|
@manager = manager
|
6
5
|
@worker = @manager.worker
|
@@ -12,8 +11,9 @@ module Upperkut
|
|
12
11
|
process
|
13
12
|
end
|
14
13
|
end
|
14
|
+
|
15
15
|
def kill
|
16
|
-
return
|
16
|
+
return unless @thread
|
17
17
|
@thread.raise Upperkut::Shutdown
|
18
18
|
end
|
19
19
|
|
@@ -35,7 +35,7 @@ module Upperkut
|
|
35
35
|
buffer_size = @worker.size
|
36
36
|
|
37
37
|
return false if @manager.stopped
|
38
|
-
return false if buffer_size
|
38
|
+
return false if buffer_size.zero?
|
39
39
|
|
40
40
|
# TODO: rename #setup by config
|
41
41
|
buffer_size >= @worker.setup.batch_size ||
|
@@ -43,15 +43,13 @@ module Upperkut
|
|
43
43
|
end
|
44
44
|
|
45
45
|
def process_batch
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
raise ex
|
54
|
-
end
|
46
|
+
@sleeping_time = 0
|
47
|
+
@worker.new.process
|
48
|
+
rescue Exception => ex
|
49
|
+
# Add to retry_queue
|
50
|
+
# if retry_limit is reached
|
51
|
+
# send to dead
|
52
|
+
raise ex
|
55
53
|
end
|
56
54
|
end
|
57
55
|
end
|
data/lib/upperkut/strategy.rb
CHANGED
@@ -12,8 +12,9 @@ module Upperkut
|
|
12
12
|
end
|
13
13
|
|
14
14
|
def push_items(items = [])
|
15
|
+
items = [items] if items.is_a?(Hash)
|
15
16
|
return false if items.empty?
|
16
|
-
redis.
|
17
|
+
redis.rpush(key, encode_json_items(items))
|
17
18
|
end
|
18
19
|
|
19
20
|
def fetch_items(batch_size = 1000)
|
@@ -41,6 +42,10 @@ module Upperkut
|
|
41
42
|
lat
|
42
43
|
end
|
43
44
|
|
45
|
+
def clear
|
46
|
+
redis.del(key)
|
47
|
+
end
|
48
|
+
|
44
49
|
private
|
45
50
|
|
46
51
|
def key
|
data/lib/upperkut/util.rb
CHANGED
@@ -5,9 +5,9 @@ module Upperkut
|
|
5
5
|
def to_underscore(object)
|
6
6
|
klass_name = object
|
7
7
|
klass_name.gsub!(/::/, '_')
|
8
|
-
klass_name.gsub!(/([A-Z\d]+)([A-Z][a-z])/,'\1_\2')
|
9
|
-
klass_name.gsub!(/([a-z\d])([A-Z])/,'\1_\2')
|
10
|
-
klass_name.tr!(
|
8
|
+
klass_name.gsub!(/([A-Z\d]+)([A-Z][a-z])/, '\1_\2')
|
9
|
+
klass_name.gsub!(/([a-z\d])([A-Z])/, '\1_\2')
|
10
|
+
klass_name.tr!('-', '_')
|
11
11
|
klass_name.downcase!
|
12
12
|
klass_name
|
13
13
|
end
|
@@ -22,7 +22,7 @@ module Upperkut
|
|
22
22
|
end
|
23
23
|
|
24
24
|
def decode_json_items(items)
|
25
|
-
items.collect {|i| JSON.parse(i) }
|
25
|
+
items.collect { |i| JSON.parse(i) }
|
26
26
|
end
|
27
27
|
end
|
28
28
|
end
|
data/lib/upperkut/version.rb
CHANGED
data/lib/upperkut/worker.rb
CHANGED
@@ -5,7 +5,6 @@ require_relative '../upperkut'
|
|
5
5
|
|
6
6
|
module Upperkut
|
7
7
|
module Worker
|
8
|
-
|
9
8
|
def self.included(base)
|
10
9
|
base.extend(ClassMethods)
|
11
10
|
end
|
@@ -21,7 +20,7 @@ module Upperkut
|
|
21
20
|
extend Forwardable
|
22
21
|
|
23
22
|
def_delegators :setup, :strategy
|
24
|
-
def_delegators :strategy, :push_items, :size, :latency
|
23
|
+
def_delegators :strategy, :push_items, :size, :latency, :clear
|
25
24
|
|
26
25
|
def push_items(items)
|
27
26
|
strategy.push_items(items)
|
@@ -31,7 +30,7 @@ module Upperkut
|
|
31
30
|
strategy.fetch_items(setup.batch_size)
|
32
31
|
end
|
33
32
|
|
34
|
-
def setup_upperkut
|
33
|
+
def setup_upperkut
|
35
34
|
yield(setup) if block_given?
|
36
35
|
end
|
37
36
|
|
data/lib/upperkut.rb
CHANGED
@@ -13,21 +13,21 @@ require 'redis'
|
|
13
13
|
#
|
14
14
|
# # This is optional
|
15
15
|
#
|
16
|
-
# setup_upperkut do |
|
16
|
+
# setup_upperkut do |config|
|
17
17
|
# # Define which redis instance you want to use
|
18
|
-
#
|
18
|
+
# config.redis = Redis.new(url: ENV['ANOTHER_REDIS_INSTANCE_URL'])
|
19
19
|
#
|
20
20
|
# # Define the amount of items must be accumulated
|
21
|
-
#
|
21
|
+
# config.batch_size = 2_000 # The default value is 1_000
|
22
22
|
#
|
23
23
|
# # How frequent the Processor should hit redis looking for elegible
|
24
|
-
# # batch. The default value is 5. You can also set the env
|
24
|
+
# # batch. The default value is 5 seconds. You can also set the env
|
25
25
|
# # UPPERKUT_POLLING_INTERVAL.
|
26
|
-
#
|
26
|
+
# config.polling_interval = 4
|
27
27
|
#
|
28
|
-
# # How long the Processor should wait to process batch
|
29
|
-
# # the amount of items did not reached the batch_size.
|
30
|
-
#
|
28
|
+
# # How long the Processor should wait in seconds to process batch
|
29
|
+
# # even though the amount of items did not reached the batch_size.
|
30
|
+
# config.max_wait = 300
|
31
31
|
# end
|
32
32
|
#
|
33
33
|
# def perform(batch_items)
|
@@ -40,7 +40,9 @@ require 'redis'
|
|
40
40
|
#
|
41
41
|
# 2) Start pushings items;
|
42
42
|
#
|
43
|
-
# Myworker.
|
43
|
+
# Myworker.push_items(
|
44
|
+
# [{'id' => SecureRandom.uuid, 'name' => 'Robert C Hall', 'action' => 'EMAIL_OPENNED'}]
|
45
|
+
# )
|
44
46
|
#
|
45
47
|
# 3) Start Upperkut;
|
46
48
|
#
|
@@ -61,5 +63,6 @@ module Upperkut
|
|
61
63
|
end
|
62
64
|
end
|
63
65
|
|
64
|
-
class
|
66
|
+
# Error class responsible to signal the shutdown process
|
67
|
+
class Shutdown < StandardError; end
|
65
68
|
end
|
data/upperkut.gemspec
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
|
2
|
-
lib = File.expand_path('
|
2
|
+
lib = File.expand_path('lib', __dir__)
|
3
3
|
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
4
|
require 'upperkut/version'
|
5
5
|
|
@@ -9,8 +9,8 @@ Gem::Specification.new do |spec|
|
|
9
9
|
spec.authors = ['Nando Sousa']
|
10
10
|
spec.email = ['nandosousafr@gmail.com']
|
11
11
|
|
12
|
-
spec.summary =
|
13
|
-
spec.description =
|
12
|
+
spec.summary = 'Batch background processing tool'
|
13
|
+
spec.description = 'Batch background processing tool'
|
14
14
|
spec.homepage = 'http://shipit.resultadosdigitais.com.br/open-source/'
|
15
15
|
spec.license = 'MIT'
|
16
16
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: upperkut
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Nando Sousa
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-06-
|
11
|
+
date: 2018-06-25 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: redis
|
@@ -121,7 +121,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
121
121
|
version: '0'
|
122
122
|
requirements: []
|
123
123
|
rubyforge_project:
|
124
|
-
rubygems_version: 2.6.14
|
124
|
+
rubygems_version: 2.6.14.1
|
125
125
|
signing_key:
|
126
126
|
specification_version: 4
|
127
127
|
summary: Batch background processing tool
|