upperkut 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/Gemfile.lock +2 -2
- data/README.md +10 -12
- data/lib/upperkut/cli.rb +1 -2
- data/lib/upperkut/processor.rb +7 -15
- data/lib/upperkut/strategies/base.rb +42 -0
- data/lib/upperkut/strategies/buffered_queue.rb +115 -0
- data/lib/upperkut/version.rb +1 -1
- data/lib/upperkut/worker.rb +7 -7
- data/lib/upperkut.rb +1 -3
- metadata +5 -4
- data/lib/upperkut/strategy.rb +0 -73
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 5c18a5b6516fc334c80ee3ccc35f1dab755490799581ff3bf50c6c3bcb45188c
|
4
|
+
data.tar.gz: d7b6ed2df27d4275666043cd9692cac287a0d62f5a5ca16216ebeba093838a29
|
5
5
|
SHA512:
|
6
|
-
metadata.gz: '
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: '09eacd8265373fd91432a997ac41e3b6ff120203476b3912d5bc7ccaa24dfb779d3f0817c10d4769782df1c7f563b2371eb6dfeab155a2c7442f9c3c51a75e2b'
|
7
|
+
data.tar.gz: 51dde9c9d81ddd79e81e220d2f4f38443778fd5026223b5330da74fde567b2310d67f3cdacf1dedd489730ab664fcb030a8404122f70be62f0a17ef108d4167a
|
data/CHANGELOG.md
CHANGED
data/Gemfile.lock
CHANGED
data/README.md
CHANGED
@@ -4,7 +4,7 @@
|
|
4
4
|
[](https://codeclimate.com/repos/5b318a7c6d37b70272008676/maintainability)
|
5
5
|
[](https://codeclimate.com/repos/5b318a7c6d37b70272008676/test_coverage)
|
6
6
|
|
7
|
-
|
7
|
+
Background processing framework for Ruby applications.
|
8
8
|
|
9
9
|
## Installation
|
10
10
|
|
@@ -34,25 +34,23 @@ Examples:
|
|
34
34
|
|
35
35
|
setup_upperkut do |config|
|
36
36
|
# Define which redis instance you want to use
|
37
|
-
config.strategy = Upperkut::
|
38
|
-
|
39
|
-
|
40
|
-
|
37
|
+
config.strategy = Upperkut::Strategies::BufferedQueue.new(
|
38
|
+
self,
|
39
|
+
redis: { url: ENV['ANOTHER_REDIS_INSTANCE_URL']) },
|
40
|
+
batch_size: 400, # How many events should be dispatched to worker.
|
41
|
+
max_wait: 300 # How long Processor wait in seconds to process batch.
|
42
|
+
# even though the amount of items did not reached the
|
43
|
+
# the batch_size.
|
44
|
+
)
|
41
45
|
|
42
46
|
# How frequent the Processor should hit redis looking for elegible
|
43
47
|
# batch. The default value is 5 seconds. You can also set the env
|
44
48
|
# UPPERKUT_POLLING_INTERVAL.
|
45
49
|
config.polling_interval = 4
|
46
|
-
|
47
|
-
# How long the Processor should wait in seconds to process batch
|
48
|
-
# even though the amount of items did not reached the batch_size.
|
49
|
-
config.max_wait = 300
|
50
50
|
end
|
51
51
|
|
52
52
|
def perform(batch_items)
|
53
|
-
|
54
|
-
SidekiqJobB.perform_async(batch_items)
|
55
|
-
|
53
|
+
heavy_processing(batch_items)
|
56
54
|
process_metrics(batch_items)
|
57
55
|
end
|
58
56
|
end
|
data/lib/upperkut/cli.rb
CHANGED
@@ -24,7 +24,6 @@ module Upperkut
|
|
24
24
|
|
25
25
|
require File.expand_path("#{@options[:require]}/config/environment.rb")
|
26
26
|
else
|
27
|
-
require 'sidekiq/rails'
|
28
27
|
require File.expand_path("#{@options[:require]}/config/environment.rb")
|
29
28
|
end
|
30
29
|
else
|
@@ -63,7 +62,7 @@ module Upperkut
|
|
63
62
|
)
|
64
63
|
|
65
64
|
manager.stop
|
66
|
-
sleep(
|
65
|
+
sleep(Integer(ENV['UPPERKUT_TIMEOUT'] || 8))
|
67
66
|
manager.kill
|
68
67
|
exit(0)
|
69
68
|
end
|
data/lib/upperkut/processor.rb
CHANGED
@@ -3,9 +3,10 @@ require_relative 'batch_execution'
|
|
3
3
|
module Upperkut
|
4
4
|
class Processor
|
5
5
|
def initialize(manager)
|
6
|
-
@manager
|
7
|
-
@worker
|
8
|
-
@logger
|
6
|
+
@manager = manager
|
7
|
+
@worker = @manager.worker
|
8
|
+
@logger = @manager.logger
|
9
|
+
@strategy = @worker.strategy
|
9
10
|
|
10
11
|
@sleeping_time = 0
|
11
12
|
end
|
@@ -35,7 +36,9 @@ module Upperkut
|
|
35
36
|
|
36
37
|
def process
|
37
38
|
loop do
|
38
|
-
if
|
39
|
+
next if @manager.stopped
|
40
|
+
|
41
|
+
if @strategy.process?
|
39
42
|
@sleeping_time = 0
|
40
43
|
process_batch
|
41
44
|
next
|
@@ -46,17 +49,6 @@ module Upperkut
|
|
46
49
|
end
|
47
50
|
end
|
48
51
|
|
49
|
-
def should_process?
|
50
|
-
buffer_size = @worker.size
|
51
|
-
|
52
|
-
return false if @manager.stopped
|
53
|
-
return false if buffer_size.zero?
|
54
|
-
|
55
|
-
# TODO: rename #setup by config
|
56
|
-
buffer_size >= @worker.setup.batch_size ||
|
57
|
-
@sleeping_time >= @worker.setup.max_wait
|
58
|
-
end
|
59
|
-
|
60
52
|
def process_batch
|
61
53
|
BatchExecution.new(@worker, @logger).execute
|
62
54
|
end
|
@@ -0,0 +1,42 @@
|
|
1
|
+
module Upperkut
|
2
|
+
module Strategies
|
3
|
+
class Base
|
4
|
+
# Public: Ingests the event into strategy.
|
5
|
+
#
|
6
|
+
# items - The Array of items do be inserted.
|
7
|
+
#
|
8
|
+
# Returns true when success, raise when error.
|
9
|
+
def push_items(items = [])
|
10
|
+
raise NotImplementedError
|
11
|
+
end
|
12
|
+
|
13
|
+
# Public: Retrieve events from Strategy.
|
14
|
+
#
|
15
|
+
# batch_size: # of items to be retrieved.
|
16
|
+
#
|
17
|
+
# Returns an Array containing events as hash.
|
18
|
+
def fetch_items(batch_size)
|
19
|
+
raise NotImplementedError
|
20
|
+
end
|
21
|
+
|
22
|
+
# Public: Clear all data related to the strategy.
|
23
|
+
def clear
|
24
|
+
raise NotImplementedError
|
25
|
+
end
|
26
|
+
|
27
|
+
# Public: Tells when to execute the event processing,
|
28
|
+
# when this condition is met so the events are dispatched to
|
29
|
+
# the worker.
|
30
|
+
def process?
|
31
|
+
raise NotImplementedError
|
32
|
+
end
|
33
|
+
|
34
|
+
# Public: Consolidated strategy metrics.
|
35
|
+
#
|
36
|
+
# Returns hash containing metric name and values.
|
37
|
+
def metrics
|
38
|
+
raise NotImplementedError
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,115 @@
|
|
1
|
+
require 'upperkut/util'
|
2
|
+
require 'upperkut/redis_pool'
|
3
|
+
require 'upperkut/strategies/base'
|
4
|
+
|
5
|
+
module Upperkut
|
6
|
+
module Strategies
|
7
|
+
class BufferedQueue < Upperkut::Strategies::Base
|
8
|
+
include Upperkut::Util
|
9
|
+
|
10
|
+
attr_reader :options
|
11
|
+
|
12
|
+
def initialize(worker, options = {})
|
13
|
+
@options = options
|
14
|
+
@redis_options = options.fetch(:redis, {})
|
15
|
+
@redis_pool = setup_redis_pool
|
16
|
+
@worker = worker
|
17
|
+
@max_wait = options.fetch(
|
18
|
+
:max_wait,
|
19
|
+
Integer(ENV['UPPERKUT_MAX_WAIT'] || 20)
|
20
|
+
)
|
21
|
+
|
22
|
+
@batch_size = options.fetch(
|
23
|
+
:batch_size,
|
24
|
+
Integer(ENV['UPPERKUT_BATCH_SIZE'] || 1000)
|
25
|
+
)
|
26
|
+
|
27
|
+
@waiting_time = 0
|
28
|
+
end
|
29
|
+
|
30
|
+
def push_items(items = [])
|
31
|
+
items = [items] if items.is_a?(Hash)
|
32
|
+
return false if items.empty?
|
33
|
+
redis do |conn|
|
34
|
+
conn.rpush(key, encode_json_items(items))
|
35
|
+
end
|
36
|
+
|
37
|
+
true
|
38
|
+
end
|
39
|
+
|
40
|
+
def fetch_items
|
41
|
+
stop = [@batch_size, size].min
|
42
|
+
|
43
|
+
items = redis do |conn|
|
44
|
+
conn.multi do
|
45
|
+
stop.times { conn.lpop(key) }
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
decode_json_items(items)
|
50
|
+
end
|
51
|
+
|
52
|
+
def clear
|
53
|
+
redis { |conn| conn.del(key) }
|
54
|
+
end
|
55
|
+
|
56
|
+
def metrics
|
57
|
+
{
|
58
|
+
'latency' => latency,
|
59
|
+
'size' => size
|
60
|
+
}
|
61
|
+
end
|
62
|
+
|
63
|
+
def process?
|
64
|
+
buff_size = size
|
65
|
+
|
66
|
+
if fulfill_condition?(buff_size)
|
67
|
+
@waiting_time = 0
|
68
|
+
return true
|
69
|
+
else
|
70
|
+
@waiting_time += @worker.setup.polling_interval
|
71
|
+
return false
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
private
|
76
|
+
|
77
|
+
def fulfill_condition?(buff_size)
|
78
|
+
return false if buff_size.zero?
|
79
|
+
buff_size >= @batch_size || @waiting_time >= @max_wait
|
80
|
+
end
|
81
|
+
|
82
|
+
def size
|
83
|
+
redis do |conn|
|
84
|
+
conn.llen(key)
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
88
|
+
def latency
|
89
|
+
item = redis { |conn| conn.lrange(key, 0, 0) }
|
90
|
+
item = decode_json_items(item).first
|
91
|
+
return 0 unless item
|
92
|
+
now = Time.now.to_f
|
93
|
+
now - item.fetch('enqueued_at', Time.now).to_f
|
94
|
+
end
|
95
|
+
|
96
|
+
|
97
|
+
|
98
|
+
def setup_redis_pool
|
99
|
+
return @redis_options if @redis_options.is_a?(ConnectionPool)
|
100
|
+
RedisPool.new(options.fetch(:redis, {})).create
|
101
|
+
end
|
102
|
+
|
103
|
+
def redis
|
104
|
+
raise ArgumentError, "requires a block" unless block_given?
|
105
|
+
@redis_pool.with do |conn|
|
106
|
+
yield conn
|
107
|
+
end
|
108
|
+
end
|
109
|
+
|
110
|
+
def key
|
111
|
+
"upperkut:buffers:#{to_underscore(@worker.name)}"
|
112
|
+
end
|
113
|
+
end
|
114
|
+
end
|
115
|
+
end
|
data/lib/upperkut/version.rb
CHANGED
data/lib/upperkut/worker.rb
CHANGED
@@ -1,8 +1,8 @@
|
|
1
1
|
require 'forwardable'
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
2
|
+
require 'upperkut/strategies/buffered_queue'
|
3
|
+
require 'upperkut/middleware'
|
4
|
+
require 'upperkut/util'
|
5
|
+
require 'upperkut'
|
6
6
|
|
7
7
|
module Upperkut
|
8
8
|
module Worker
|
@@ -14,7 +14,7 @@ module Upperkut
|
|
14
14
|
extend Forwardable
|
15
15
|
|
16
16
|
def_delegators :setup, :strategy, :server_middlewares, :client_middlewares
|
17
|
-
def_delegators :strategy, :
|
17
|
+
def_delegators :strategy, :metrics, :clear
|
18
18
|
|
19
19
|
def push_items(items)
|
20
20
|
client_middlewares.invoke(self, items) do
|
@@ -23,7 +23,7 @@ module Upperkut
|
|
23
23
|
end
|
24
24
|
|
25
25
|
def fetch_items
|
26
|
-
strategy.fetch_items
|
26
|
+
strategy.fetch_items
|
27
27
|
end
|
28
28
|
|
29
29
|
def setup_upperkut
|
@@ -34,7 +34,7 @@ module Upperkut
|
|
34
34
|
@config ||=
|
35
35
|
begin
|
36
36
|
config = Upperkut::Configuration.default.clone
|
37
|
-
config.strategy = Upperkut::
|
37
|
+
config.strategy = Upperkut::Strategies::BufferedQueue.new(self)
|
38
38
|
config
|
39
39
|
end
|
40
40
|
end
|
data/lib/upperkut.rb
CHANGED
@@ -54,12 +54,10 @@ require 'redis'
|
|
54
54
|
# 4) That's it :)
|
55
55
|
module Upperkut
|
56
56
|
class Configuration
|
57
|
-
attr_accessor :
|
57
|
+
attr_accessor :strategy, :polling_interval
|
58
58
|
|
59
59
|
def self.default
|
60
60
|
new.tap do |config|
|
61
|
-
config.batch_size = 1_000
|
62
|
-
config.max_wait = Integer(ENV['UPPERKUT_MAX_WAIT'] || 20)
|
63
61
|
config.polling_interval = Integer(ENV['UPPERKUT_POLLING_INTERVAL'] || 5)
|
64
62
|
end
|
65
63
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: upperkut
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.7.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Nando Sousa
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2019-04-02 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: redis
|
@@ -124,7 +124,8 @@ files:
|
|
124
124
|
- lib/upperkut/middlewares/rollbar.rb
|
125
125
|
- lib/upperkut/processor.rb
|
126
126
|
- lib/upperkut/redis_pool.rb
|
127
|
-
- lib/upperkut/
|
127
|
+
- lib/upperkut/strategies/base.rb
|
128
|
+
- lib/upperkut/strategies/buffered_queue.rb
|
128
129
|
- lib/upperkut/util.rb
|
129
130
|
- lib/upperkut/version.rb
|
130
131
|
- lib/upperkut/worker.rb
|
@@ -149,7 +150,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
149
150
|
version: '0'
|
150
151
|
requirements: []
|
151
152
|
rubyforge_project:
|
152
|
-
rubygems_version: 2.7.
|
153
|
+
rubygems_version: 2.7.8
|
153
154
|
signing_key:
|
154
155
|
specification_version: 4
|
155
156
|
summary: Batch background processing tool
|
data/lib/upperkut/strategy.rb
DELETED
@@ -1,73 +0,0 @@
|
|
1
|
-
require_relative 'util'
|
2
|
-
require_relative 'redis_pool'
|
3
|
-
|
4
|
-
module Upperkut
|
5
|
-
class Strategy
|
6
|
-
include Upperkut::Util
|
7
|
-
|
8
|
-
attr_reader :options
|
9
|
-
|
10
|
-
def initialize(worker, options = {})
|
11
|
-
@options = options
|
12
|
-
@redis_options = options.fetch(:redis, {})
|
13
|
-
@redis_pool = setup_redis_pool
|
14
|
-
@worker = worker
|
15
|
-
end
|
16
|
-
|
17
|
-
def push_items(items = [])
|
18
|
-
items = [items] if items.is_a?(Hash)
|
19
|
-
return false if items.empty?
|
20
|
-
redis do |conn|
|
21
|
-
conn.rpush(key, encode_json_items(items))
|
22
|
-
end
|
23
|
-
end
|
24
|
-
|
25
|
-
def fetch_items(batch_size = 1000)
|
26
|
-
stop = [batch_size, size].min
|
27
|
-
|
28
|
-
items = redis do |conn|
|
29
|
-
conn.multi do
|
30
|
-
stop.times { conn.lpop(key) }
|
31
|
-
end
|
32
|
-
end
|
33
|
-
|
34
|
-
decode_json_items(items)
|
35
|
-
end
|
36
|
-
|
37
|
-
def size
|
38
|
-
redis do |conn|
|
39
|
-
conn.llen(key)
|
40
|
-
end
|
41
|
-
end
|
42
|
-
|
43
|
-
def latency
|
44
|
-
item = redis { |conn| conn.lrange(key, 0, 0) }
|
45
|
-
item = decode_json_items(item).first
|
46
|
-
return 0 unless item
|
47
|
-
now = Time.now.to_f
|
48
|
-
now - item.fetch('enqueued_at', Time.now).to_f
|
49
|
-
end
|
50
|
-
|
51
|
-
def clear
|
52
|
-
redis { |conn| conn.del(key) }
|
53
|
-
end
|
54
|
-
|
55
|
-
private
|
56
|
-
|
57
|
-
def setup_redis_pool
|
58
|
-
return @redis_options if @redis_options.is_a?(ConnectionPool)
|
59
|
-
RedisPool.new(options.fetch(:redis, {})).create
|
60
|
-
end
|
61
|
-
|
62
|
-
def redis
|
63
|
-
raise ArgumentError, "requires a block" unless block_given?
|
64
|
-
@redis_pool.with do |conn|
|
65
|
-
yield conn
|
66
|
-
end
|
67
|
-
end
|
68
|
-
|
69
|
-
def key
|
70
|
-
"upperkut:buffers:#{to_underscore(@worker.name)}"
|
71
|
-
end
|
72
|
-
end
|
73
|
-
end
|