upperkut 0.7.2 → 1.0.0.rc

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 9fb4b45a40c37d26aa8d96f199c79141f8b34d1edc2fd48c751a6d8567b77deb
4
- data.tar.gz: b4b91ae1d8d2737d20b97c23744cc5cc8052681ede5c68f28790c50f3a092882
3
+ metadata.gz: 497852c1c7edee902a15cb3978f2efaab5ffdd8fe5be6fa2ab3fc62e9de2b661
4
+ data.tar.gz: 39ed05ba320a38368c83a018216ab9db5ab26e760369f5b944e8ecacd24c1acc
5
5
  SHA512:
6
- metadata.gz: 2c934b63764f8a673762cdb163989b18fae91d8b2be30e8b99f5db93e4329c3c63f2beb75a8e38c46b966bffb7f4e8e7f09c4f5246e8806b001b0da2ce056f88
7
- data.tar.gz: '038a2489af4d5b8b0811f30402922e3a7ed4524dceee4b3dcdff1d13c417a037f2f7f88c61122ba65cd0ad2ee1ed9760a4210a94427e69c3d7b2f635e2c3694c'
6
+ metadata.gz: 7cccacbc6c943ca5ee5bab8bea75c67d0ba6bb039c732723f16ffb78cd98c85ee5873b04cfc485b8e3fa879a35bfe59f0aedb53ede87a620666aebbe40ef9b77
7
+ data.tar.gz: 3031919f600731f5fb2989ebe02055a1032bb3565ce471b74f6b29025b9f2417fc98a1302cc39ad6d0a05b1c4d641636cbff8d965bbd7194b5ae94f34a1096e4
@@ -1,7 +1,16 @@
1
1
  # Upperkut changes
2
+ 0.8.x
3
+ --------
4
+ - Added exponential backoff when push_items #57
5
+ - Introducing Item to avoid losing enqueued at and report wrong latency
6
+ metrics #56 thanks to @jeangnc
2
7
 
3
8
  0.7.x
4
9
  ---------
10
+ - Fix logging timeout message #54 by @jeanmatheussouto
11
+ - Add handle_error method #44
12
+ - Added Datahog Middleware (#42)
13
+ - Added Priority Queue (#39) thanks to @jeangnc and @jeanmatheussouto
5
14
  - Added Scheduled Queue Implementation thanks to @rodrigo-araujo #38
6
15
  - Added Datahog middleware #42 by @gabriel-augusto
7
16
  - Added redis to CI #40 by #henrich-m
data/Gemfile CHANGED
@@ -5,7 +5,6 @@ git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
5
5
  # Specify your gem's dependencies in upperkut.gemspec
6
6
  gemspec
7
7
 
8
- gem 'fakeredis'
9
8
  gem 'fivemat'
10
9
  gem 'pry'
11
10
  gem 'rspec_junit_formatter'
@@ -1,43 +1,41 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- upperkut (0.7.2)
4
+ upperkut (1.0.0.rc)
5
5
  connection_pool (~> 2.2, >= 2.2.2)
6
- redis (>= 3.3.3, < 5)
6
+ redis (>= 4.1.0, < 5.0.0)
7
7
 
8
8
  GEM
9
9
  remote: https://rubygems.org/
10
10
  specs:
11
11
  coderay (1.1.2)
12
- connection_pool (2.2.2)
12
+ connection_pool (2.2.3)
13
13
  diff-lcs (1.3)
14
- docile (1.3.1)
15
- fakeredis (0.7.0)
16
- redis (>= 3.2, < 5.0)
17
- fivemat (1.3.6)
18
- json (2.1.0)
19
- method_source (0.9.0)
20
- pry (0.11.3)
21
- coderay (~> 1.1.0)
22
- method_source (~> 0.9.0)
23
- rake (10.5.0)
24
- redis (4.0.1)
25
- rspec (3.7.0)
26
- rspec-core (~> 3.7.0)
27
- rspec-expectations (~> 3.7.0)
28
- rspec-mocks (~> 3.7.0)
29
- rspec-core (3.7.1)
30
- rspec-support (~> 3.7.0)
31
- rspec-expectations (3.7.0)
14
+ docile (1.3.2)
15
+ fivemat (1.3.7)
16
+ json (2.3.0)
17
+ method_source (1.0.0)
18
+ pry (0.13.1)
19
+ coderay (~> 1.1)
20
+ method_source (~> 1.0)
21
+ rake (13.0.1)
22
+ redis (4.2.1)
23
+ rspec (3.9.0)
24
+ rspec-core (~> 3.9.0)
25
+ rspec-expectations (~> 3.9.0)
26
+ rspec-mocks (~> 3.9.0)
27
+ rspec-core (3.9.0)
28
+ rspec-support (~> 3.9.0)
29
+ rspec-expectations (3.9.0)
32
30
  diff-lcs (>= 1.2.0, < 2.0)
33
- rspec-support (~> 3.7.0)
34
- rspec-mocks (3.7.0)
31
+ rspec-support (~> 3.9.0)
32
+ rspec-mocks (3.9.0)
35
33
  diff-lcs (>= 1.2.0, < 2.0)
36
- rspec-support (~> 3.7.0)
37
- rspec-support (3.7.1)
34
+ rspec-support (~> 3.9.0)
35
+ rspec-support (3.9.0)
38
36
  rspec_junit_formatter (0.4.1)
39
37
  rspec-core (>= 2, < 4, != 2.12.0)
40
- simplecov (0.16.1)
38
+ simplecov (0.17.1)
41
39
  docile (~> 1.1)
42
40
  json (>= 1.8, < 3)
43
41
  simplecov-html (~> 0.10.0)
@@ -47,15 +45,14 @@ PLATFORMS
47
45
  ruby
48
46
 
49
47
  DEPENDENCIES
50
- bundler (~> 1.16)
51
- fakeredis
48
+ bundler (>= 1.16)
52
49
  fivemat
53
50
  pry
54
- rake (~> 10.0)
51
+ rake (~> 13.0)
55
52
  rspec (~> 3.0)
56
53
  rspec_junit_formatter
57
54
  simplecov
58
55
  upperkut!
59
56
 
60
57
  BUNDLED WITH
61
- 1.16.6
58
+ 2.1.4
data/README.md CHANGED
@@ -4,6 +4,8 @@
4
4
  [![Maintainability](https://api.codeclimate.com/v1/badges/ece40319b0db03af891d/maintainability)](https://codeclimate.com/repos/5b318a7c6d37b70272008676/maintainability)
5
5
  [![Test Coverage](https://api.codeclimate.com/v1/badges/ece40319b0db03af891d/test_coverage)](https://codeclimate.com/repos/5b318a7c6d37b70272008676/test_coverage)
6
6
 
7
+ [[Docs]](https://www.rubydoc.info/gems/upperkut/0.7.2/Upperkut)
8
+
7
9
  Background processing framework for Ruby applications.
8
10
 
9
11
  ## Installation
@@ -31,25 +33,6 @@ Or install it yourself as:
31
33
  class MyWorker
32
34
  include Upperkut::Worker
33
35
 
34
- # This is optional
35
-
36
- setup_upperkut do |config|
37
- # Define which redis instance you want to use
38
- config.strategy = Upperkut::Strategies::BufferedQueue.new(
39
- self,
40
- redis: { url: ENV['ANOTHER_REDIS_INSTANCE_URL'] },
41
- batch_size: 400, # How many events should be dispatched to worker.
42
- max_wait: 300 # How long Processor wait in seconds to process batch.
43
- # even though the amount of items did not reached the
44
- # the batch_size.
45
- )
46
-
47
- # How frequent the Processor should hit redis looking for elegible
48
- # batch. The default value is 5 seconds. You can also set the env
49
- # UPPERKUT_POLLING_INTERVAL.
50
- config.polling_interval = 4
51
- end
52
-
53
36
  def perform(batch_items)
54
37
  heavy_processing(batch_items)
55
38
  process_metrics(batch_items)
@@ -57,13 +40,13 @@ Or install it yourself as:
57
40
  end
58
41
  ```
59
42
 
60
- 2) Start pushings items;
43
+ 2) Start pushing items;
61
44
  ```ruby
62
- Myworker.push_items(
45
+ MyWorker.push_items(
63
46
  [
64
47
  {
65
- 'id' => SecureRandom.uuid,
66
- 'name' => 'Robert C Hall',
48
+ 'id' => SecureRandom.uuid,
49
+ 'name' => 'Robert C Hall',
67
50
  'action' => 'EMAIL_OPENNED'
68
51
  }
69
52
  ]
@@ -94,15 +77,15 @@ Or install it yourself as:
94
77
  end
95
78
  ```
96
79
 
97
- 2) Start pushings items with `timestamp` param;
80
+ 2) Start pushing items with `timestamp` parameter;
98
81
  ```ruby
99
82
  # timestamp is 'Thu, 10 May 2019 23:43:58 GMT'
100
- Myworker.push_items(
83
+ MyWorker.push_items(
101
84
  [
102
85
  {
103
- 'timestamp' => '1557531838',
104
- 'id' => SecureRandom.uuid,
105
- 'name' => 'Robert C Hall',
86
+ 'timestamp' => '1557531838',
87
+ 'id' => SecureRandom.uuid,
88
+ 'name' => 'Robert C Hall',
106
89
  'action' => 'SEND_NOTIFICATION'
107
90
  }
108
91
  ]
@@ -114,6 +97,52 @@ Or install it yourself as:
114
97
  $ bundle exec upperkut --worker MyWorker --concurrency 10
115
98
  ```
116
99
 
100
+ ### Example 3 - Priority Queue:
101
+
102
+ Note: priority queues requires redis 5.0.0+ as it uses ZPOP* commands.
103
+
104
+ 1) Create a Worker class and the define how to process the batch;
105
+ ```ruby
106
+ require 'upperkut/strategies/priority_queue'
107
+
108
+ class MyWorker
109
+ include Upperkut::Worker
110
+
111
+ setup_upperkut do |config|
112
+ config.strategy = Upperkut::Strategies::PriorityQueue.new(
113
+ self,
114
+ priority_key: -> { |item| item['tenant_id'] }
115
+ )
116
+ end
117
+
118
+ def perform(items)
119
+ items.each do |item|
120
+ puts "event dispatched: #{item.inspect}"
121
+ end
122
+ end
123
+ end
124
+ ```
125
+
126
+ 2) So you can enqueue items from different tenants;
127
+ ```ruby
128
+ MyWorker.push_items(
129
+ [
130
+ { 'tenant_id' => 1, 'id' => 1 },
131
+ { 'tenant_id' => 1, 'id' => 2 },
132
+ { 'tenant_id' => 1, 'id' => 3 },
133
+ { 'tenant_id' => 2, 'id' => 4 },
134
+ { 'tenant_id' => 3, 'id' => 5 },
135
+ ]
136
+ )
137
+ ```
138
+
139
+ The code above will enqueue items as follows `1, 4, 5, 2, 3`
140
+
141
+ 3) Start Upperkut;
142
+ ```bash
143
+ $ bundle exec upperkut --worker MyWorker --concurrency 10
144
+ ```
145
+
117
146
  ## Development
118
147
 
119
148
  After checking out the repo, run `bundle install` to install dependencies. Then, run `bundle exec rspec` to run the tests.
@@ -0,0 +1,21 @@
1
+ require_relative '../lib/upperkut/worker'
2
+ require_relative '../lib/upperkut/strategies/priority_queue'
3
+
4
+ class PriorityWorker
5
+ include Upperkut::Worker
6
+
7
+ setup_upperkut do |config|
8
+ config.strategy = Upperkut::Strategies::PriorityQueue.new(
9
+ self,
10
+ priority_key: -> { |item| item['tenant_id'] },
11
+ batch_size: 1
12
+ )
13
+ end
14
+
15
+ def perform(items)
16
+ items.each do |item|
17
+ puts "event dispatched: #{item.inspect}"
18
+ sleep 1
19
+ end
20
+ end
21
+ end
@@ -57,12 +57,13 @@ module Upperkut
57
57
  handle_signal(signal)
58
58
  end
59
59
  rescue Interrupt
60
+ timeout = Integer(ENV['UPPERKUT_TIMEOUT'] || 8)
60
61
  @logger.info(
61
- 'Stopping managers, wait for 5 seconds and them kill processors'
62
+ "Stopping managers, wait for #{timeout} seconds and them kill processors"
62
63
  )
63
64
 
64
65
  manager.stop
65
- sleep(Integer(ENV['UPPERKUT_TIMEOUT'] || 8))
66
+ sleep(timeout)
66
67
  manager.kill
67
68
  exit(0)
68
69
  end
@@ -0,0 +1,50 @@
1
+ require 'securerandom'
2
+
3
+ module Upperkut
4
+ class Item
5
+ attr_reader :id, :body, :enqueued_at
6
+
7
+ def initialize(body:, id: nil, enqueued_at: nil)
8
+ raise ArgumentError, 'Body should be a Hash' unless body.is_a?(Hash)
9
+
10
+ @body = body
11
+ @id = id || SecureRandom.uuid
12
+ @enqueued_at = enqueued_at || Time.now.utc.to_i
13
+ @nacked = false
14
+ end
15
+
16
+ def [](key)
17
+ @body[key]
18
+ end
19
+
20
+ def []=(key, value)
21
+ @body[key] = value
22
+ end
23
+
24
+ def key?(key)
25
+ @body.key?(key)
26
+ end
27
+
28
+ def nack
29
+ @nacked = true
30
+ end
31
+
32
+ def nacked?
33
+ @nacked
34
+ end
35
+
36
+ def to_json
37
+ JSON.generate(
38
+ 'id' => @id,
39
+ 'body' => @body,
40
+ 'enqueued_at' => @enqueued_at
41
+ )
42
+ end
43
+
44
+ def self.from_json(item_json)
45
+ hash = JSON.parse(item_json)
46
+ id, body, enqueued_at = hash.values_at('id', 'body', 'enqueued_at')
47
+ new(id: id, body: body, enqueued_at: enqueued_at)
48
+ end
49
+ end
50
+ end
@@ -1,44 +1,50 @@
1
1
  require_relative 'core_ext'
2
- require_relative 'processor'
2
+ require_relative 'worker_thread'
3
+ require_relative 'logging'
3
4
  require_relative 'worker'
4
5
 
5
6
  module Upperkut
6
7
  class Manager
7
8
  attr_accessor :worker
8
- attr_reader :stopped, :logger, :concurrency, :processors
9
+ attr_reader :stopped, :logger, :concurrency
9
10
 
10
11
  def initialize(opts = {})
11
12
  self.worker = opts.fetch(:worker).constantize
12
13
  @concurrency = opts.fetch(:concurrency, 1)
13
- @logger = opts.fetch(:logger, Upperkut::Logging.logger)
14
+ @logger = opts.fetch(:logger, Logging.logger)
14
15
 
15
16
  @stopped = false
16
- @processors = []
17
+ @threads = []
17
18
  end
18
19
 
19
20
  def run
20
21
  @concurrency.times do
21
- processor = Processor.new(self)
22
- @processors << processor
23
- processor.run
22
+ spawn_thread
24
23
  end
25
24
  end
26
25
 
27
26
  def stop
28
27
  @stopped = true
28
+ @threads.each(&:stop)
29
29
  end
30
30
 
31
31
  def kill
32
- @processors.each(&:kill)
32
+ @threads.each(&:kill)
33
33
  end
34
34
 
35
- def notify_killed_processor(processor)
36
- @processors.delete(processor)
37
- return if @stopped
35
+ def notify_killed_processor(thread)
36
+ @threads.delete(thread)
37
+ spawn_thread unless @stopped
38
+ end
39
+
40
+ private
41
+
42
+ def spawn_thread
43
+ processor = Processor.new(worker, logger)
38
44
 
39
- processor = Processor.new(self)
40
- @processors << processor
41
- processor.run
45
+ thread = WorkerThread.new(self, processor)
46
+ @threads << thread
47
+ thread.run
42
48
  end
43
49
  end
44
50
  end
@@ -1,58 +1,63 @@
1
- require_relative 'batch_execution'
1
+ require_relative 'logging'
2
2
 
3
3
  module Upperkut
4
4
  class Processor
5
- def initialize(manager)
6
- @manager = manager
7
- @worker = @manager.worker
8
- @logger = @manager.logger
9
- @strategy = @worker.strategy
10
-
11
- @sleeping_time = 0
5
+ def initialize(worker, logger = Logging.logger)
6
+ @worker = worker
7
+ @strategy = worker.strategy
8
+ @worker_instance = worker.new
9
+ @logger = logger
12
10
  end
13
11
 
14
- def run
15
- @thread ||= Thread.new do
16
- begin
17
- process
18
- rescue Exception => e
19
- @logger.debug(
20
- action: :processor_killed,
21
- reason: e,
22
- stacktrace: e.backtrace
23
- )
24
-
25
- @manager.notify_killed_processor(self)
26
- end
12
+ def process
13
+ items = @worker.fetch_items.freeze
14
+
15
+ @worker.server_middlewares.invoke(@worker, items) do
16
+ @worker_instance.perform(items)
27
17
  end
28
- end
29
18
 
30
- def kill
31
- return unless @thread
19
+ nacked_items, pending_ack_items = items.partition(&:nacked?)
20
+ @strategy.nack(nacked_items) if nacked_items.any?
21
+ @strategy.ack(pending_ack_items) if pending_ack_items.any?
22
+ rescue StandardError => error
23
+ @logger.error(
24
+ action: :handle_execution_error,
25
+ ex: error.to_s,
26
+ backtrace: error.backtrace.join("\n"),
27
+ item_size: Array(items).size
28
+ )
29
+
30
+ if items
31
+ if @worker_instance.respond_to?(:handle_error)
32
+ @worker_instance.handle_error(error, items)
33
+ return
34
+ end
35
+
36
+ @strategy.nack(items)
37
+ end
32
38
 
33
- @thread.raise Upperkut::Shutdown
34
- @thread.value # wait
39
+ raise error
35
40
  end
36
41
 
37
- private
42
+ def blocking_process
43
+ sleeping_time = 0
38
44
 
39
- def process
40
45
  loop do
41
- next if @manager.stopped
46
+ break if @stopped
42
47
 
43
48
  if @strategy.process?
44
- @sleeping_time = 0
45
- process_batch
49
+ sleeping_time = 0
50
+ process
46
51
  next
47
52
  end
48
53
 
49
- @sleeping_time += sleep(@worker.setup.polling_interval)
50
- @logger.debug(sleeping_time: @sleeping_time)
54
+ sleeping_time += sleep(@worker.setup.polling_interval)
55
+ @logger.debug(sleeping_time: sleeping_time)
51
56
  end
52
57
  end
53
58
 
54
- def process_batch
55
- BatchExecution.new(@worker, @logger).execute
59
+ def stop
60
+ @stopped = true
56
61
  end
57
62
  end
58
63
  end