upperkut 0.6.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 01ee5c00ec69139f31e74694a53163296cea3e8ee0b483f678a322d965bc76e1
4
- data.tar.gz: df6c73408d00851951d85b5464f7ba689dcf7adc5551ce3cea3762188dd33256
3
+ metadata.gz: 5c18a5b6516fc334c80ee3ccc35f1dab755490799581ff3bf50c6c3bcb45188c
4
+ data.tar.gz: d7b6ed2df27d4275666043cd9692cac287a0d62f5a5ca16216ebeba093838a29
5
5
  SHA512:
6
- metadata.gz: '08bfd143bd18e122c704b44724e81c68e80f4d247611fc52ecc119ff96a4ae3a6fa3ffd03aa04582b2b4ca0e89b98fc91f56a418ed0daf5326e538cafdfac76e'
7
- data.tar.gz: 44baf1f326ec0993fa7ff7288dbdc680bcb57189f187a7de7c850b606fae90a0f33584a37499d0cd0a81f03626023ae5169702b1edd5fda1a0a436942c59a524
6
+ metadata.gz: '09eacd8265373fd91432a997ac41e3b6ff120203476b3912d5bc7ccaa24dfb779d3f0817c10d4769782df1c7f563b2371eb6dfeab155a2c7442f9c3c51a75e2b'
7
+ data.tar.gz: 51dde9c9d81ddd79e81e220d2f4f38443778fd5026223b5330da74fde567b2310d67f3cdacf1dedd489730ab664fcb030a8404122f70be62f0a17ef108d4167a
data/CHANGELOG.md CHANGED
@@ -1,5 +1,9 @@
1
1
  # Upperkut changes
2
2
 
3
+ 0.7.x
4
+ ---------
5
+ - Extract Buffered Queue behavior to its own strategy #29
6
+
3
7
  0.6.x
4
8
  ---------
5
9
  - Set redis url when env var REDIS_URL is set thanks to @lucaskds #22
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- upperkut (0.6.0)
4
+ upperkut (0.7.0)
5
5
  connection_pool (~> 2.2, >= 2.2.2)
6
6
  redis (>= 3.3.3, < 5)
7
7
 
@@ -58,4 +58,4 @@ DEPENDENCIES
58
58
  upperkut!
59
59
 
60
60
  BUNDLED WITH
61
- 1.16.3
61
+ 1.16.6
data/README.md CHANGED
@@ -4,7 +4,7 @@
4
4
  [![Maintainability](https://api.codeclimate.com/v1/badges/ece40319b0db03af891d/maintainability)](https://codeclimate.com/repos/5b318a7c6d37b70272008676/maintainability)
5
5
  [![Test Coverage](https://api.codeclimate.com/v1/badges/ece40319b0db03af891d/test_coverage)](https://codeclimate.com/repos/5b318a7c6d37b70272008676/test_coverage)
6
6
 
7
- Batch background processing tool.
7
+ Background processing framework for Ruby applications.
8
8
 
9
9
  ## Installation
10
10
 
@@ -34,25 +34,23 @@ Examples:
34
34
 
35
35
  setup_upperkut do |config|
36
36
  # Define which redis instance you want to use
37
- config.strategy = Upperkut::Strategy.new(self, redis: { url: ENV['ANOTHER_REDIS_INSTANCE_URL']) })
38
-
39
- # Define the amount of items must be accumulated
40
- config.batch_size = 2_000 # The default value is 1_000
37
+ config.strategy = Upperkut::Strategies::BufferedQueue.new(
38
+ self,
39
+ redis: { url: ENV['ANOTHER_REDIS_INSTANCE_URL']) },
40
+ batch_size: 400, # How many events should be dispatched to worker.
41
+ max_wait: 300 # How long Processor wait in seconds to process batch.
42
+ # even though the amount of items did not reached the
43
+ # the batch_size.
44
+ )
41
45
 
42
46
  # How frequent the Processor should hit redis looking for elegible
43
47
  # batch. The default value is 5 seconds. You can also set the env
44
48
  # UPPERKUT_POLLING_INTERVAL.
45
49
  config.polling_interval = 4
46
-
47
- # How long the Processor should wait in seconds to process batch
48
- # even though the amount of items did not reached the batch_size.
49
- config.max_wait = 300
50
50
  end
51
51
 
52
52
  def perform(batch_items)
53
- SidekiqJobA.perform_async(batch_items)
54
- SidekiqJobB.perform_async(batch_items)
55
-
53
+ heavy_processing(batch_items)
56
54
  process_metrics(batch_items)
57
55
  end
58
56
  end
data/lib/upperkut/cli.rb CHANGED
@@ -24,7 +24,6 @@ module Upperkut
24
24
 
25
25
  require File.expand_path("#{@options[:require]}/config/environment.rb")
26
26
  else
27
- require 'sidekiq/rails'
28
27
  require File.expand_path("#{@options[:require]}/config/environment.rb")
29
28
  end
30
29
  else
@@ -63,7 +62,7 @@ module Upperkut
63
62
  )
64
63
 
65
64
  manager.stop
66
- sleep(5)
65
+ sleep(Integer(ENV['UPPERKUT_TIMEOUT'] || 8))
67
66
  manager.kill
68
67
  exit(0)
69
68
  end
@@ -3,9 +3,10 @@ require_relative 'batch_execution'
3
3
  module Upperkut
4
4
  class Processor
5
5
  def initialize(manager)
6
- @manager = manager
7
- @worker = @manager.worker
8
- @logger = @manager.logger
6
+ @manager = manager
7
+ @worker = @manager.worker
8
+ @logger = @manager.logger
9
+ @strategy = @worker.strategy
9
10
 
10
11
  @sleeping_time = 0
11
12
  end
@@ -35,7 +36,9 @@ module Upperkut
35
36
 
36
37
  def process
37
38
  loop do
38
- if should_process?
39
+ next if @manager.stopped
40
+
41
+ if @strategy.process?
39
42
  @sleeping_time = 0
40
43
  process_batch
41
44
  next
@@ -46,17 +49,6 @@ module Upperkut
46
49
  end
47
50
  end
48
51
 
49
- def should_process?
50
- buffer_size = @worker.size
51
-
52
- return false if @manager.stopped
53
- return false if buffer_size.zero?
54
-
55
- # TODO: rename #setup by config
56
- buffer_size >= @worker.setup.batch_size ||
57
- @sleeping_time >= @worker.setup.max_wait
58
- end
59
-
60
52
  def process_batch
61
53
  BatchExecution.new(@worker, @logger).execute
62
54
  end
@@ -0,0 +1,42 @@
1
+ module Upperkut
2
+ module Strategies
3
+ class Base
4
+ # Public: Ingests the event into strategy.
5
+ #
6
+ # items - The Array of items do be inserted.
7
+ #
8
+ # Returns true when success, raise when error.
9
+ def push_items(items = [])
10
+ raise NotImplementedError
11
+ end
12
+
13
+ # Public: Retrieve events from Strategy.
14
+ #
15
+ # batch_size: # of items to be retrieved.
16
+ #
17
+ # Returns an Array containing events as hash.
18
+ def fetch_items(batch_size)
19
+ raise NotImplementedError
20
+ end
21
+
22
+ # Public: Clear all data related to the strategy.
23
+ def clear
24
+ raise NotImplementedError
25
+ end
26
+
27
+ # Public: Tells when to execute the event processing,
28
+ # when this condition is met so the events are dispatched to
29
+ # the worker.
30
+ def process?
31
+ raise NotImplementedError
32
+ end
33
+
34
+ # Public: Consolidated strategy metrics.
35
+ #
36
+ # Returns hash containing metric name and values.
37
+ def metrics
38
+ raise NotImplementedError
39
+ end
40
+ end
41
+ end
42
+ end
@@ -0,0 +1,115 @@
1
+ require 'upperkut/util'
2
+ require 'upperkut/redis_pool'
3
+ require 'upperkut/strategies/base'
4
+
5
+ module Upperkut
6
+ module Strategies
7
+ class BufferedQueue < Upperkut::Strategies::Base
8
+ include Upperkut::Util
9
+
10
+ attr_reader :options
11
+
12
+ def initialize(worker, options = {})
13
+ @options = options
14
+ @redis_options = options.fetch(:redis, {})
15
+ @redis_pool = setup_redis_pool
16
+ @worker = worker
17
+ @max_wait = options.fetch(
18
+ :max_wait,
19
+ Integer(ENV['UPPERKUT_MAX_WAIT'] || 20)
20
+ )
21
+
22
+ @batch_size = options.fetch(
23
+ :batch_size,
24
+ Integer(ENV['UPPERKUT_BATCH_SIZE'] || 1000)
25
+ )
26
+
27
+ @waiting_time = 0
28
+ end
29
+
30
+ def push_items(items = [])
31
+ items = [items] if items.is_a?(Hash)
32
+ return false if items.empty?
33
+ redis do |conn|
34
+ conn.rpush(key, encode_json_items(items))
35
+ end
36
+
37
+ true
38
+ end
39
+
40
+ def fetch_items
41
+ stop = [@batch_size, size].min
42
+
43
+ items = redis do |conn|
44
+ conn.multi do
45
+ stop.times { conn.lpop(key) }
46
+ end
47
+ end
48
+
49
+ decode_json_items(items)
50
+ end
51
+
52
+ def clear
53
+ redis { |conn| conn.del(key) }
54
+ end
55
+
56
+ def metrics
57
+ {
58
+ 'latency' => latency,
59
+ 'size' => size
60
+ }
61
+ end
62
+
63
+ def process?
64
+ buff_size = size
65
+
66
+ if fulfill_condition?(buff_size)
67
+ @waiting_time = 0
68
+ return true
69
+ else
70
+ @waiting_time += @worker.setup.polling_interval
71
+ return false
72
+ end
73
+ end
74
+
75
+ private
76
+
77
+ def fulfill_condition?(buff_size)
78
+ return false if buff_size.zero?
79
+ buff_size >= @batch_size || @waiting_time >= @max_wait
80
+ end
81
+
82
+ def size
83
+ redis do |conn|
84
+ conn.llen(key)
85
+ end
86
+ end
87
+
88
+ def latency
89
+ item = redis { |conn| conn.lrange(key, 0, 0) }
90
+ item = decode_json_items(item).first
91
+ return 0 unless item
92
+ now = Time.now.to_f
93
+ now - item.fetch('enqueued_at', Time.now).to_f
94
+ end
95
+
96
+
97
+
98
+ def setup_redis_pool
99
+ return @redis_options if @redis_options.is_a?(ConnectionPool)
100
+ RedisPool.new(options.fetch(:redis, {})).create
101
+ end
102
+
103
+ def redis
104
+ raise ArgumentError, "requires a block" unless block_given?
105
+ @redis_pool.with do |conn|
106
+ yield conn
107
+ end
108
+ end
109
+
110
+ def key
111
+ "upperkut:buffers:#{to_underscore(@worker.name)}"
112
+ end
113
+ end
114
+ end
115
+ end
@@ -1,3 +1,3 @@
1
1
  module Upperkut
2
- VERSION = '0.6.0'.freeze
2
+ VERSION = '0.7.0'.freeze
3
3
  end
@@ -1,8 +1,8 @@
1
1
  require 'forwardable'
2
- require_relative 'strategy'
3
- require_relative 'middleware'
4
- require_relative './util'
5
- require_relative '../upperkut'
2
+ require 'upperkut/strategies/buffered_queue'
3
+ require 'upperkut/middleware'
4
+ require 'upperkut/util'
5
+ require 'upperkut'
6
6
 
7
7
  module Upperkut
8
8
  module Worker
@@ -14,7 +14,7 @@ module Upperkut
14
14
  extend Forwardable
15
15
 
16
16
  def_delegators :setup, :strategy, :server_middlewares, :client_middlewares
17
- def_delegators :strategy, :push_items, :size, :latency, :clear
17
+ def_delegators :strategy, :metrics, :clear
18
18
 
19
19
  def push_items(items)
20
20
  client_middlewares.invoke(self, items) do
@@ -23,7 +23,7 @@ module Upperkut
23
23
  end
24
24
 
25
25
  def fetch_items
26
- strategy.fetch_items(setup.batch_size)
26
+ strategy.fetch_items
27
27
  end
28
28
 
29
29
  def setup_upperkut
@@ -34,7 +34,7 @@ module Upperkut
34
34
  @config ||=
35
35
  begin
36
36
  config = Upperkut::Configuration.default.clone
37
- config.strategy = Upperkut::Strategy.new(self)
37
+ config.strategy = Upperkut::Strategies::BufferedQueue.new(self)
38
38
  config
39
39
  end
40
40
  end
data/lib/upperkut.rb CHANGED
@@ -54,12 +54,10 @@ require 'redis'
54
54
  # 4) That's it :)
55
55
  module Upperkut
56
56
  class Configuration
57
- attr_accessor :batch_size, :strategy, :max_wait, :polling_interval
57
+ attr_accessor :strategy, :polling_interval
58
58
 
59
59
  def self.default
60
60
  new.tap do |config|
61
- config.batch_size = 1_000
62
- config.max_wait = Integer(ENV['UPPERKUT_MAX_WAIT'] || 20)
63
61
  config.polling_interval = Integer(ENV['UPPERKUT_POLLING_INTERVAL'] || 5)
64
62
  end
65
63
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: upperkut
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.6.0
4
+ version: 0.7.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Nando Sousa
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2018-09-12 00:00:00.000000000 Z
11
+ date: 2019-04-02 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: redis
@@ -124,7 +124,8 @@ files:
124
124
  - lib/upperkut/middlewares/rollbar.rb
125
125
  - lib/upperkut/processor.rb
126
126
  - lib/upperkut/redis_pool.rb
127
- - lib/upperkut/strategy.rb
127
+ - lib/upperkut/strategies/base.rb
128
+ - lib/upperkut/strategies/buffered_queue.rb
128
129
  - lib/upperkut/util.rb
129
130
  - lib/upperkut/version.rb
130
131
  - lib/upperkut/worker.rb
@@ -149,7 +150,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
149
150
  version: '0'
150
151
  requirements: []
151
152
  rubyforge_project:
152
- rubygems_version: 2.7.7
153
+ rubygems_version: 2.7.8
153
154
  signing_key:
154
155
  specification_version: 4
155
156
  summary: Batch background processing tool
@@ -1,73 +0,0 @@
1
- require_relative 'util'
2
- require_relative 'redis_pool'
3
-
4
- module Upperkut
5
- class Strategy
6
- include Upperkut::Util
7
-
8
- attr_reader :options
9
-
10
- def initialize(worker, options = {})
11
- @options = options
12
- @redis_options = options.fetch(:redis, {})
13
- @redis_pool = setup_redis_pool
14
- @worker = worker
15
- end
16
-
17
- def push_items(items = [])
18
- items = [items] if items.is_a?(Hash)
19
- return false if items.empty?
20
- redis do |conn|
21
- conn.rpush(key, encode_json_items(items))
22
- end
23
- end
24
-
25
- def fetch_items(batch_size = 1000)
26
- stop = [batch_size, size].min
27
-
28
- items = redis do |conn|
29
- conn.multi do
30
- stop.times { conn.lpop(key) }
31
- end
32
- end
33
-
34
- decode_json_items(items)
35
- end
36
-
37
- def size
38
- redis do |conn|
39
- conn.llen(key)
40
- end
41
- end
42
-
43
- def latency
44
- item = redis { |conn| conn.lrange(key, 0, 0) }
45
- item = decode_json_items(item).first
46
- return 0 unless item
47
- now = Time.now.to_f
48
- now - item.fetch('enqueued_at', Time.now).to_f
49
- end
50
-
51
- def clear
52
- redis { |conn| conn.del(key) }
53
- end
54
-
55
- private
56
-
57
- def setup_redis_pool
58
- return @redis_options if @redis_options.is_a?(ConnectionPool)
59
- RedisPool.new(options.fetch(:redis, {})).create
60
- end
61
-
62
- def redis
63
- raise ArgumentError, "requires a block" unless block_given?
64
- @redis_pool.with do |conn|
65
- yield conn
66
- end
67
- end
68
-
69
- def key
70
- "upperkut:buffers:#{to_underscore(@worker.name)}"
71
- end
72
- end
73
- end