upperkut 0.5.2 → 0.6.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 4710b6cb7fe25d82ec581c0500b37a02abfc22edf75f7b25a57c7b67c111e347
4
- data.tar.gz: 6a07d95e388588def062c04d229546a98396afb16ab7d929418d921ff54f50ce
3
+ metadata.gz: 01ee5c00ec69139f31e74694a53163296cea3e8ee0b483f678a322d965bc76e1
4
+ data.tar.gz: df6c73408d00851951d85b5464f7ba689dcf7adc5551ce3cea3762188dd33256
5
5
  SHA512:
6
- metadata.gz: e8c47b5c1fefbaaa0b22539eda6f0c7add81dbc484593d93fa999548ea161736f8af7fa81d3905523c3376213ebe4bb40e7b36220eb10a0b2b30dfcfb9204ef9
7
- data.tar.gz: 1755c8d9b63b21f4baf735843c21431ad86d378408e2b938591a3344e68a2db3e246d30331b7031309367e187ea20a74244c08d97a0eebe3c60b46b39a306e19
6
+ metadata.gz: '08bfd143bd18e122c704b44724e81c68e80f4d247611fc52ecc119ff96a4ae3a6fa3ffd03aa04582b2b4ca0e89b98fc91f56a418ed0daf5326e538cafdfac76e'
7
+ data.tar.gz: 44baf1f326ec0993fa7ff7288dbdc680bcb57189f187a7de7c850b606fae90a0f33584a37499d0cd0a81f03626023ae5169702b1edd5fda1a0a436942c59a524
@@ -1,8 +1,15 @@
1
1
  # Upperkut changes
2
2
 
3
+ 0.6.x
4
+ ---------
5
+ - Set redis url when env var REDIS_URL is set thanks to @lucaskds #22
6
+ - Enable users to pass their own connection pool also thanks to @lucaskds #21
7
+ - Sets default concurrency to 1
8
+ - Creates connection pools by default;
9
+ - Simplifies API to enable extension via Strategy;
10
+
3
11
  0.5.x
4
12
  ----------
5
-
6
13
  - Introducing client middlewares.
7
14
  - Fix issue that prevented worker to run.
8
15
  - Fix thread-unsafe code that was overwriting worker configurations when
@@ -1,13 +1,15 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- upperkut (0.5.2)
4
+ upperkut (0.6.0)
5
+ connection_pool (~> 2.2, >= 2.2.2)
5
6
  redis (>= 3.3.3, < 5)
6
7
 
7
8
  GEM
8
9
  remote: https://rubygems.org/
9
10
  specs:
10
11
  coderay (1.1.2)
12
+ connection_pool (2.2.2)
11
13
  diff-lcs (1.3)
12
14
  docile (1.3.1)
13
15
  fakeredis (0.7.0)
data/README.md CHANGED
@@ -34,8 +34,7 @@ Examples:
34
34
 
35
35
  setup_upperkut do |config|
36
36
  # Define which redis instance you want to use
37
- redis = Redis.new(url: ENV['ANOTHER_REDIS_INSTANCE_URL'])
38
- config.strategy = Upperkut::Strategy.new(self, redis)
37
+ config.strategy = Upperkut::Strategy.new(self, redis: { url: ENV['ANOTHER_REDIS_INSTANCE_URL']) })
39
38
 
40
39
  # Define the amount of items must be accumulated
41
40
  config.batch_size = 2_000 # The default value is 1_000
@@ -15,7 +15,10 @@ require 'redis'
15
15
  #
16
16
  # setup_upperkut do |config|
17
17
  # # Define which redis instance you want to use
18
- # config.redis = Redis.new(url: ENV['ANOTHER_REDIS_INSTANCE_URL'])
18
+ # config.strategy = Upperkut::Strategy.new(
19
+ # self,
20
+ # redis: { url: ENV['ANOTHER_REDIS_URL'] }
21
+ # )
19
22
  #
20
23
  # # Define the amount of items must be accumulated
21
24
  # config.batch_size = 2_000 # The default value is 1_000
@@ -9,7 +9,7 @@ module Upperkut
9
9
 
10
10
  def initialize(opts = {})
11
11
  self.worker = opts.fetch(:worker).constantize
12
- @concurrency = opts.fetch(:concurrency, 25)
12
+ @concurrency = opts.fetch(:concurrency, 1)
13
13
  @logger = opts.fetch(:logger, Upperkut::Logging.logger)
14
14
 
15
15
  @stopped = false
@@ -0,0 +1,30 @@
1
+ require 'connection_pool'
2
+ require 'redis'
3
+
4
+ module Upperkut
5
+ class RedisPool
6
+
7
+ DEFAULT_OPTIONS = {
8
+ pool_timeout: 1, # pool related option
9
+ size: 2, # pool related option
10
+ connect_timeout: 0.2,
11
+ read_timeout: 5.0,
12
+ write_timeout: 0.5,
13
+ url: ENV['REDIS_URL'],
14
+ }.freeze
15
+
16
+ def initialize(options)
17
+ @options = DEFAULT_OPTIONS.merge(options)
18
+
19
+ # Extract pool related options
20
+ @size = @options.delete(:size)
21
+ @pool_timeout = @options.delete(:pool_timeout)
22
+ end
23
+
24
+ def create
25
+ ConnectionPool.new(:timeout => @pool_timeout, :size => @size) do
26
+ Redis.new(@options)
27
+ end
28
+ end
29
+ end
30
+ end
@@ -1,28 +1,33 @@
1
1
  require_relative 'util'
2
+ require_relative 'redis_pool'
2
3
 
3
4
  module Upperkut
4
5
  class Strategy
5
6
  include Upperkut::Util
6
7
 
7
- attr_accessor :worker, :redis
8
+ attr_reader :options
8
9
 
9
- def initialize(worker, redis = Redis.new)
10
- self.worker = worker
11
- self.redis = redis
10
+ def initialize(worker, options = {})
11
+ @options = options
12
+ @redis_options = options.fetch(:redis, {})
13
+ @redis_pool = setup_redis_pool
14
+ @worker = worker
12
15
  end
13
16
 
14
17
  def push_items(items = [])
15
18
  items = [items] if items.is_a?(Hash)
16
19
  return false if items.empty?
17
- redis.rpush(key, encode_json_items(items))
20
+ redis do |conn|
21
+ conn.rpush(key, encode_json_items(items))
22
+ end
18
23
  end
19
24
 
20
25
  def fetch_items(batch_size = 1000)
21
26
  stop = [batch_size, size].min
22
27
 
23
- items = redis.multi do
24
- stop.times do
25
- redis.lpop(key)
28
+ items = redis do |conn|
29
+ conn.multi do
30
+ stop.times { conn.lpop(key) }
26
31
  end
27
32
  end
28
33
 
@@ -30,11 +35,13 @@ module Upperkut
30
35
  end
31
36
 
32
37
  def size
33
- redis.llen(key)
38
+ redis do |conn|
39
+ conn.llen(key)
40
+ end
34
41
  end
35
42
 
36
43
  def latency
37
- item = redis.lrange(key, 0, 0)
44
+ item = redis { |conn| conn.lrange(key, 0, 0) }
38
45
  item = decode_json_items(item).first
39
46
  return 0 unless item
40
47
  now = Time.now.to_f
@@ -42,13 +49,25 @@ module Upperkut
42
49
  end
43
50
 
44
51
  def clear
45
- redis.del(key)
52
+ redis { |conn| conn.del(key) }
46
53
  end
47
54
 
48
55
  private
49
56
 
57
+ def setup_redis_pool
58
+ return @redis_options if @redis_options.is_a?(ConnectionPool)
59
+ RedisPool.new(options.fetch(:redis, {})).create
60
+ end
61
+
62
+ def redis
63
+ raise ArgumentError, "requires a block" unless block_given?
64
+ @redis_pool.with do |conn|
65
+ yield conn
66
+ end
67
+ end
68
+
50
69
  def key
51
- "upperkut:buffers:#{to_underscore(worker.name)}"
70
+ "upperkut:buffers:#{to_underscore(@worker.name)}"
52
71
  end
53
72
  end
54
73
  end
@@ -1,3 +1,3 @@
1
1
  module Upperkut
2
- VERSION = '0.5.2'.freeze
2
+ VERSION = '0.6.0'.freeze
3
3
  end
@@ -14,7 +14,7 @@ module Upperkut
14
14
  extend Forwardable
15
15
 
16
16
  def_delegators :setup, :strategy, :server_middlewares, :client_middlewares
17
- def_delegators :strategy, :push_items, :size, :latency, :clear, :redis
17
+ def_delegators :strategy, :push_items, :size, :latency, :clear
18
18
 
19
19
  def push_items(items)
20
20
  client_middlewares.invoke(self, items) do
@@ -22,7 +22,8 @@ Gem::Specification.new do |spec|
22
22
 
23
23
  spec.required_ruby_version = '>= 2.2.2'
24
24
 
25
- spec.add_dependency 'redis', '>= 3.3.3', '< 5'
25
+ spec.add_dependency 'redis', '>= 3.3.3', '< 5'
26
+ spec.add_dependency 'connection_pool', '~> 2.2', '>= 2.2.2'
26
27
  spec.add_development_dependency 'bundler', '~> 1.16'
27
28
  spec.add_development_dependency 'rake', '~> 10.0'
28
29
  spec.add_development_dependency 'rspec', '~> 3.0'
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: upperkut
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.5.2
4
+ version: 0.6.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Nando Sousa
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2018-08-30 00:00:00.000000000 Z
11
+ date: 2018-09-12 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: redis
@@ -30,6 +30,26 @@ dependencies:
30
30
  - - "<"
31
31
  - !ruby/object:Gem::Version
32
32
  version: '5'
33
+ - !ruby/object:Gem::Dependency
34
+ name: connection_pool
35
+ requirement: !ruby/object:Gem::Requirement
36
+ requirements:
37
+ - - "~>"
38
+ - !ruby/object:Gem::Version
39
+ version: '2.2'
40
+ - - ">="
41
+ - !ruby/object:Gem::Version
42
+ version: 2.2.2
43
+ type: :runtime
44
+ prerelease: false
45
+ version_requirements: !ruby/object:Gem::Requirement
46
+ requirements:
47
+ - - "~>"
48
+ - !ruby/object:Gem::Version
49
+ version: '2.2'
50
+ - - ">="
51
+ - !ruby/object:Gem::Version
52
+ version: 2.2.2
33
53
  - !ruby/object:Gem::Dependency
34
54
  name: bundler
35
55
  requirement: !ruby/object:Gem::Requirement
@@ -103,6 +123,7 @@ files:
103
123
  - lib/upperkut/middlewares/new_relic.rb
104
124
  - lib/upperkut/middlewares/rollbar.rb
105
125
  - lib/upperkut/processor.rb
126
+ - lib/upperkut/redis_pool.rb
106
127
  - lib/upperkut/strategy.rb
107
128
  - lib/upperkut/util.rb
108
129
  - lib/upperkut/version.rb