upperkut 0.5.2 → 0.6.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +8 -1
- data/Gemfile.lock +3 -1
- data/README.md +1 -2
- data/lib/upperkut.rb +4 -1
- data/lib/upperkut/manager.rb +1 -1
- data/lib/upperkut/redis_pool.rb +30 -0
- data/lib/upperkut/strategy.rb +31 -12
- data/lib/upperkut/version.rb +1 -1
- data/lib/upperkut/worker.rb +1 -1
- data/upperkut.gemspec +2 -1
- metadata +23 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 01ee5c00ec69139f31e74694a53163296cea3e8ee0b483f678a322d965bc76e1
|
4
|
+
data.tar.gz: df6c73408d00851951d85b5464f7ba689dcf7adc5551ce3cea3762188dd33256
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: '08bfd143bd18e122c704b44724e81c68e80f4d247611fc52ecc119ff96a4ae3a6fa3ffd03aa04582b2b4ca0e89b98fc91f56a418ed0daf5326e538cafdfac76e'
|
7
|
+
data.tar.gz: 44baf1f326ec0993fa7ff7288dbdc680bcb57189f187a7de7c850b606fae90a0f33584a37499d0cd0a81f03626023ae5169702b1edd5fda1a0a436942c59a524
|
data/CHANGELOG.md
CHANGED
@@ -1,8 +1,15 @@
|
|
1
1
|
# Upperkut changes
|
2
2
|
|
3
|
+
0.6.x
|
4
|
+
---------
|
5
|
+
- Set redis url when env var REDIS_URL is set thanks to @lucaskds #22
|
6
|
+
- Enable users to pass their own connection pool also thanks to @lucaskds #21
|
7
|
+
- Sets default concurrency to 1
|
8
|
+
- Creates connection pools by default;
|
9
|
+
- Simplifies API to enable extension via Strategy;
|
10
|
+
|
3
11
|
0.5.x
|
4
12
|
----------
|
5
|
-
|
6
13
|
- Introducing client middlewares.
|
7
14
|
- Fix issue that prevented worker to run.
|
8
15
|
- Fix thread-unsafe code that was overwriting worker configurations when
|
data/Gemfile.lock
CHANGED
@@ -1,13 +1,15 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
upperkut (0.
|
4
|
+
upperkut (0.6.0)
|
5
|
+
connection_pool (~> 2.2, >= 2.2.2)
|
5
6
|
redis (>= 3.3.3, < 5)
|
6
7
|
|
7
8
|
GEM
|
8
9
|
remote: https://rubygems.org/
|
9
10
|
specs:
|
10
11
|
coderay (1.1.2)
|
12
|
+
connection_pool (2.2.2)
|
11
13
|
diff-lcs (1.3)
|
12
14
|
docile (1.3.1)
|
13
15
|
fakeredis (0.7.0)
|
data/README.md
CHANGED
@@ -34,8 +34,7 @@ Examples:
|
|
34
34
|
|
35
35
|
setup_upperkut do |config|
|
36
36
|
# Define which redis instance you want to use
|
37
|
-
|
38
|
-
config.strategy = Upperkut::Strategy.new(self, redis)
|
37
|
+
config.strategy = Upperkut::Strategy.new(self, redis: { url: ENV['ANOTHER_REDIS_INSTANCE_URL']) })
|
39
38
|
|
40
39
|
# Define the amount of items must be accumulated
|
41
40
|
config.batch_size = 2_000 # The default value is 1_000
|
data/lib/upperkut.rb
CHANGED
@@ -15,7 +15,10 @@ require 'redis'
|
|
15
15
|
#
|
16
16
|
# setup_upperkut do |config|
|
17
17
|
# # Define which redis instance you want to use
|
18
|
-
# config.
|
18
|
+
# config.strategy = Upperkut::Strategy.new(
|
19
|
+
# self,
|
20
|
+
# redis: { url: ENV['ANOTHER_REDIS_URL'] }
|
21
|
+
# )
|
19
22
|
#
|
20
23
|
# # Define the amount of items must be accumulated
|
21
24
|
# config.batch_size = 2_000 # The default value is 1_000
|
data/lib/upperkut/manager.rb
CHANGED
@@ -9,7 +9,7 @@ module Upperkut
|
|
9
9
|
|
10
10
|
def initialize(opts = {})
|
11
11
|
self.worker = opts.fetch(:worker).constantize
|
12
|
-
@concurrency = opts.fetch(:concurrency,
|
12
|
+
@concurrency = opts.fetch(:concurrency, 1)
|
13
13
|
@logger = opts.fetch(:logger, Upperkut::Logging.logger)
|
14
14
|
|
15
15
|
@stopped = false
|
@@ -0,0 +1,30 @@
|
|
1
|
+
require 'connection_pool'
|
2
|
+
require 'redis'
|
3
|
+
|
4
|
+
module Upperkut
|
5
|
+
class RedisPool
|
6
|
+
|
7
|
+
DEFAULT_OPTIONS = {
|
8
|
+
pool_timeout: 1, # pool related option
|
9
|
+
size: 2, # pool related option
|
10
|
+
connect_timeout: 0.2,
|
11
|
+
read_timeout: 5.0,
|
12
|
+
write_timeout: 0.5,
|
13
|
+
url: ENV['REDIS_URL'],
|
14
|
+
}.freeze
|
15
|
+
|
16
|
+
def initialize(options)
|
17
|
+
@options = DEFAULT_OPTIONS.merge(options)
|
18
|
+
|
19
|
+
# Extract pool related options
|
20
|
+
@size = @options.delete(:size)
|
21
|
+
@pool_timeout = @options.delete(:pool_timeout)
|
22
|
+
end
|
23
|
+
|
24
|
+
def create
|
25
|
+
ConnectionPool.new(:timeout => @pool_timeout, :size => @size) do
|
26
|
+
Redis.new(@options)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
data/lib/upperkut/strategy.rb
CHANGED
@@ -1,28 +1,33 @@
|
|
1
1
|
require_relative 'util'
|
2
|
+
require_relative 'redis_pool'
|
2
3
|
|
3
4
|
module Upperkut
|
4
5
|
class Strategy
|
5
6
|
include Upperkut::Util
|
6
7
|
|
7
|
-
|
8
|
+
attr_reader :options
|
8
9
|
|
9
|
-
def initialize(worker,
|
10
|
-
|
11
|
-
|
10
|
+
def initialize(worker, options = {})
|
11
|
+
@options = options
|
12
|
+
@redis_options = options.fetch(:redis, {})
|
13
|
+
@redis_pool = setup_redis_pool
|
14
|
+
@worker = worker
|
12
15
|
end
|
13
16
|
|
14
17
|
def push_items(items = [])
|
15
18
|
items = [items] if items.is_a?(Hash)
|
16
19
|
return false if items.empty?
|
17
|
-
redis
|
20
|
+
redis do |conn|
|
21
|
+
conn.rpush(key, encode_json_items(items))
|
22
|
+
end
|
18
23
|
end
|
19
24
|
|
20
25
|
def fetch_items(batch_size = 1000)
|
21
26
|
stop = [batch_size, size].min
|
22
27
|
|
23
|
-
items = redis
|
24
|
-
|
25
|
-
|
28
|
+
items = redis do |conn|
|
29
|
+
conn.multi do
|
30
|
+
stop.times { conn.lpop(key) }
|
26
31
|
end
|
27
32
|
end
|
28
33
|
|
@@ -30,11 +35,13 @@ module Upperkut
|
|
30
35
|
end
|
31
36
|
|
32
37
|
def size
|
33
|
-
redis
|
38
|
+
redis do |conn|
|
39
|
+
conn.llen(key)
|
40
|
+
end
|
34
41
|
end
|
35
42
|
|
36
43
|
def latency
|
37
|
-
item = redis.lrange(key, 0, 0)
|
44
|
+
item = redis { |conn| conn.lrange(key, 0, 0) }
|
38
45
|
item = decode_json_items(item).first
|
39
46
|
return 0 unless item
|
40
47
|
now = Time.now.to_f
|
@@ -42,13 +49,25 @@ module Upperkut
|
|
42
49
|
end
|
43
50
|
|
44
51
|
def clear
|
45
|
-
redis.del(key)
|
52
|
+
redis { |conn| conn.del(key) }
|
46
53
|
end
|
47
54
|
|
48
55
|
private
|
49
56
|
|
57
|
+
def setup_redis_pool
|
58
|
+
return @redis_options if @redis_options.is_a?(ConnectionPool)
|
59
|
+
RedisPool.new(options.fetch(:redis, {})).create
|
60
|
+
end
|
61
|
+
|
62
|
+
def redis
|
63
|
+
raise ArgumentError, "requires a block" unless block_given?
|
64
|
+
@redis_pool.with do |conn|
|
65
|
+
yield conn
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
50
69
|
def key
|
51
|
-
"upperkut:buffers:#{to_underscore(worker.name)}"
|
70
|
+
"upperkut:buffers:#{to_underscore(@worker.name)}"
|
52
71
|
end
|
53
72
|
end
|
54
73
|
end
|
data/lib/upperkut/version.rb
CHANGED
data/lib/upperkut/worker.rb
CHANGED
@@ -14,7 +14,7 @@ module Upperkut
|
|
14
14
|
extend Forwardable
|
15
15
|
|
16
16
|
def_delegators :setup, :strategy, :server_middlewares, :client_middlewares
|
17
|
-
def_delegators :strategy, :push_items, :size, :latency, :clear
|
17
|
+
def_delegators :strategy, :push_items, :size, :latency, :clear
|
18
18
|
|
19
19
|
def push_items(items)
|
20
20
|
client_middlewares.invoke(self, items) do
|
data/upperkut.gemspec
CHANGED
@@ -22,7 +22,8 @@ Gem::Specification.new do |spec|
|
|
22
22
|
|
23
23
|
spec.required_ruby_version = '>= 2.2.2'
|
24
24
|
|
25
|
-
spec.add_dependency 'redis',
|
25
|
+
spec.add_dependency 'redis', '>= 3.3.3', '< 5'
|
26
|
+
spec.add_dependency 'connection_pool', '~> 2.2', '>= 2.2.2'
|
26
27
|
spec.add_development_dependency 'bundler', '~> 1.16'
|
27
28
|
spec.add_development_dependency 'rake', '~> 10.0'
|
28
29
|
spec.add_development_dependency 'rspec', '~> 3.0'
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: upperkut
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.6.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Nando Sousa
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-
|
11
|
+
date: 2018-09-12 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: redis
|
@@ -30,6 +30,26 @@ dependencies:
|
|
30
30
|
- - "<"
|
31
31
|
- !ruby/object:Gem::Version
|
32
32
|
version: '5'
|
33
|
+
- !ruby/object:Gem::Dependency
|
34
|
+
name: connection_pool
|
35
|
+
requirement: !ruby/object:Gem::Requirement
|
36
|
+
requirements:
|
37
|
+
- - "~>"
|
38
|
+
- !ruby/object:Gem::Version
|
39
|
+
version: '2.2'
|
40
|
+
- - ">="
|
41
|
+
- !ruby/object:Gem::Version
|
42
|
+
version: 2.2.2
|
43
|
+
type: :runtime
|
44
|
+
prerelease: false
|
45
|
+
version_requirements: !ruby/object:Gem::Requirement
|
46
|
+
requirements:
|
47
|
+
- - "~>"
|
48
|
+
- !ruby/object:Gem::Version
|
49
|
+
version: '2.2'
|
50
|
+
- - ">="
|
51
|
+
- !ruby/object:Gem::Version
|
52
|
+
version: 2.2.2
|
33
53
|
- !ruby/object:Gem::Dependency
|
34
54
|
name: bundler
|
35
55
|
requirement: !ruby/object:Gem::Requirement
|
@@ -103,6 +123,7 @@ files:
|
|
103
123
|
- lib/upperkut/middlewares/new_relic.rb
|
104
124
|
- lib/upperkut/middlewares/rollbar.rb
|
105
125
|
- lib/upperkut/processor.rb
|
126
|
+
- lib/upperkut/redis_pool.rb
|
106
127
|
- lib/upperkut/strategy.rb
|
107
128
|
- lib/upperkut/util.rb
|
108
129
|
- lib/upperkut/version.rb
|