upperkut 0.7.2 → 1.0.0.rc

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 9fb4b45a40c37d26aa8d96f199c79141f8b34d1edc2fd48c751a6d8567b77deb
4
- data.tar.gz: b4b91ae1d8d2737d20b97c23744cc5cc8052681ede5c68f28790c50f3a092882
3
+ metadata.gz: 497852c1c7edee902a15cb3978f2efaab5ffdd8fe5be6fa2ab3fc62e9de2b661
4
+ data.tar.gz: 39ed05ba320a38368c83a018216ab9db5ab26e760369f5b944e8ecacd24c1acc
5
5
  SHA512:
6
- metadata.gz: 2c934b63764f8a673762cdb163989b18fae91d8b2be30e8b99f5db93e4329c3c63f2beb75a8e38c46b966bffb7f4e8e7f09c4f5246e8806b001b0da2ce056f88
7
- data.tar.gz: '038a2489af4d5b8b0811f30402922e3a7ed4524dceee4b3dcdff1d13c417a037f2f7f88c61122ba65cd0ad2ee1ed9760a4210a94427e69c3d7b2f635e2c3694c'
6
+ metadata.gz: 7cccacbc6c943ca5ee5bab8bea75c67d0ba6bb039c732723f16ffb78cd98c85ee5873b04cfc485b8e3fa879a35bfe59f0aedb53ede87a620666aebbe40ef9b77
7
+ data.tar.gz: 3031919f600731f5fb2989ebe02055a1032bb3565ce471b74f6b29025b9f2417fc98a1302cc39ad6d0a05b1c4d641636cbff8d965bbd7194b5ae94f34a1096e4
@@ -1,7 +1,16 @@
1
1
  # Upperkut changes
2
+ 0.8.x
3
+ --------
4
+ - Added exponential backoff when push_items #57
5
+ - Introducing Item to avoid losing enqueued at and report wrong latency
6
+ metrics #56 thanks to @jeangnc
2
7
 
3
8
  0.7.x
4
9
  ---------
10
+ - Fix logging timeout message #54 by @jeanmatheussouto
11
+ - Add handle_error method #44
12
+ - Added Datahog Middleware (#42)
13
+ - Added Priority Queue (#39) thanks to @jeangnc and @jeanmatheussouto
5
14
  - Added Scheduled Queue Implementation thanks to @rodrigo-araujo #38
6
15
  - Added Datahog middleware #42 by @gabriel-augusto
7
16
  - Added redis to CI #40 by #henrich-m
data/Gemfile CHANGED
@@ -5,7 +5,6 @@ git_source(:github) { |repo_name| "https://github.com/#{repo_name}" }
5
5
  # Specify your gem's dependencies in upperkut.gemspec
6
6
  gemspec
7
7
 
8
- gem 'fakeredis'
9
8
  gem 'fivemat'
10
9
  gem 'pry'
11
10
  gem 'rspec_junit_formatter'
@@ -1,43 +1,41 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- upperkut (0.7.2)
4
+ upperkut (1.0.0.rc)
5
5
  connection_pool (~> 2.2, >= 2.2.2)
6
- redis (>= 3.3.3, < 5)
6
+ redis (>= 4.1.0, < 5.0.0)
7
7
 
8
8
  GEM
9
9
  remote: https://rubygems.org/
10
10
  specs:
11
11
  coderay (1.1.2)
12
- connection_pool (2.2.2)
12
+ connection_pool (2.2.3)
13
13
  diff-lcs (1.3)
14
- docile (1.3.1)
15
- fakeredis (0.7.0)
16
- redis (>= 3.2, < 5.0)
17
- fivemat (1.3.6)
18
- json (2.1.0)
19
- method_source (0.9.0)
20
- pry (0.11.3)
21
- coderay (~> 1.1.0)
22
- method_source (~> 0.9.0)
23
- rake (10.5.0)
24
- redis (4.0.1)
25
- rspec (3.7.0)
26
- rspec-core (~> 3.7.0)
27
- rspec-expectations (~> 3.7.0)
28
- rspec-mocks (~> 3.7.0)
29
- rspec-core (3.7.1)
30
- rspec-support (~> 3.7.0)
31
- rspec-expectations (3.7.0)
14
+ docile (1.3.2)
15
+ fivemat (1.3.7)
16
+ json (2.3.0)
17
+ method_source (1.0.0)
18
+ pry (0.13.1)
19
+ coderay (~> 1.1)
20
+ method_source (~> 1.0)
21
+ rake (13.0.1)
22
+ redis (4.2.1)
23
+ rspec (3.9.0)
24
+ rspec-core (~> 3.9.0)
25
+ rspec-expectations (~> 3.9.0)
26
+ rspec-mocks (~> 3.9.0)
27
+ rspec-core (3.9.0)
28
+ rspec-support (~> 3.9.0)
29
+ rspec-expectations (3.9.0)
32
30
  diff-lcs (>= 1.2.0, < 2.0)
33
- rspec-support (~> 3.7.0)
34
- rspec-mocks (3.7.0)
31
+ rspec-support (~> 3.9.0)
32
+ rspec-mocks (3.9.0)
35
33
  diff-lcs (>= 1.2.0, < 2.0)
36
- rspec-support (~> 3.7.0)
37
- rspec-support (3.7.1)
34
+ rspec-support (~> 3.9.0)
35
+ rspec-support (3.9.0)
38
36
  rspec_junit_formatter (0.4.1)
39
37
  rspec-core (>= 2, < 4, != 2.12.0)
40
- simplecov (0.16.1)
38
+ simplecov (0.17.1)
41
39
  docile (~> 1.1)
42
40
  json (>= 1.8, < 3)
43
41
  simplecov-html (~> 0.10.0)
@@ -47,15 +45,14 @@ PLATFORMS
47
45
  ruby
48
46
 
49
47
  DEPENDENCIES
50
- bundler (~> 1.16)
51
- fakeredis
48
+ bundler (>= 1.16)
52
49
  fivemat
53
50
  pry
54
- rake (~> 10.0)
51
+ rake (~> 13.0)
55
52
  rspec (~> 3.0)
56
53
  rspec_junit_formatter
57
54
  simplecov
58
55
  upperkut!
59
56
 
60
57
  BUNDLED WITH
61
- 1.16.6
58
+ 2.1.4
data/README.md CHANGED
@@ -4,6 +4,8 @@
4
4
  [![Maintainability](https://api.codeclimate.com/v1/badges/ece40319b0db03af891d/maintainability)](https://codeclimate.com/repos/5b318a7c6d37b70272008676/maintainability)
5
5
  [![Test Coverage](https://api.codeclimate.com/v1/badges/ece40319b0db03af891d/test_coverage)](https://codeclimate.com/repos/5b318a7c6d37b70272008676/test_coverage)
6
6
 
7
+ [[Docs]](https://www.rubydoc.info/gems/upperkut/0.7.2/Upperkut)
8
+
7
9
  Background processing framework for Ruby applications.
8
10
 
9
11
  ## Installation
@@ -31,25 +33,6 @@ Or install it yourself as:
31
33
  class MyWorker
32
34
  include Upperkut::Worker
33
35
 
34
- # This is optional
35
-
36
- setup_upperkut do |config|
37
- # Define which redis instance you want to use
38
- config.strategy = Upperkut::Strategies::BufferedQueue.new(
39
- self,
40
- redis: { url: ENV['ANOTHER_REDIS_INSTANCE_URL'] },
41
- batch_size: 400, # How many events should be dispatched to worker.
42
- max_wait: 300 # How long Processor wait in seconds to process batch.
43
- # even though the amount of items did not reached the
44
- # the batch_size.
45
- )
46
-
47
- # How frequent the Processor should hit redis looking for elegible
48
- # batch. The default value is 5 seconds. You can also set the env
49
- # UPPERKUT_POLLING_INTERVAL.
50
- config.polling_interval = 4
51
- end
52
-
53
36
  def perform(batch_items)
54
37
  heavy_processing(batch_items)
55
38
  process_metrics(batch_items)
@@ -57,13 +40,13 @@ Or install it yourself as:
57
40
  end
58
41
  ```
59
42
 
60
- 2) Start pushings items;
43
+ 2) Start pushing items;
61
44
  ```ruby
62
- Myworker.push_items(
45
+ MyWorker.push_items(
63
46
  [
64
47
  {
65
- 'id' => SecureRandom.uuid,
66
- 'name' => 'Robert C Hall',
48
+ 'id' => SecureRandom.uuid,
49
+ 'name' => 'Robert C Hall',
67
50
  'action' => 'EMAIL_OPENNED'
68
51
  }
69
52
  ]
@@ -94,15 +77,15 @@ Or install it yourself as:
94
77
  end
95
78
  ```
96
79
 
97
- 2) Start pushings items with `timestamp` param;
80
+ 2) Start pushing items with `timestamp` parameter;
98
81
  ```ruby
99
82
  # timestamp is 'Thu, 10 May 2019 23:43:58 GMT'
100
- Myworker.push_items(
83
+ MyWorker.push_items(
101
84
  [
102
85
  {
103
- 'timestamp' => '1557531838',
104
- 'id' => SecureRandom.uuid,
105
- 'name' => 'Robert C Hall',
86
+ 'timestamp' => '1557531838',
87
+ 'id' => SecureRandom.uuid,
88
+ 'name' => 'Robert C Hall',
106
89
  'action' => 'SEND_NOTIFICATION'
107
90
  }
108
91
  ]
@@ -114,6 +97,52 @@ Or install it yourself as:
114
97
  $ bundle exec upperkut --worker MyWorker --concurrency 10
115
98
  ```
116
99
 
100
+ ### Example 3 - Priority Queue:
101
+
102
+ Note: priority queues requires redis 5.0.0+ as it uses ZPOP* commands.
103
+
104
+ 1) Create a Worker class and the define how to process the batch;
105
+ ```ruby
106
+ require 'upperkut/strategies/priority_queue'
107
+
108
+ class MyWorker
109
+ include Upperkut::Worker
110
+
111
+ setup_upperkut do |config|
112
+ config.strategy = Upperkut::Strategies::PriorityQueue.new(
113
+ self,
114
+ priority_key: -> { |item| item['tenant_id'] }
115
+ )
116
+ end
117
+
118
+ def perform(items)
119
+ items.each do |item|
120
+ puts "event dispatched: #{item.inspect}"
121
+ end
122
+ end
123
+ end
124
+ ```
125
+
126
+ 2) So you can enqueue items from different tenants;
127
+ ```ruby
128
+ MyWorker.push_items(
129
+ [
130
+ { 'tenant_id' => 1, 'id' => 1 },
131
+ { 'tenant_id' => 1, 'id' => 2 },
132
+ { 'tenant_id' => 1, 'id' => 3 },
133
+ { 'tenant_id' => 2, 'id' => 4 },
134
+ { 'tenant_id' => 3, 'id' => 5 },
135
+ ]
136
+ )
137
+ ```
138
+
139
+ The code above will enqueue items as follows `1, 4, 5, 2, 3`
140
+
141
+ 3) Start Upperkut;
142
+ ```bash
143
+ $ bundle exec upperkut --worker MyWorker --concurrency 10
144
+ ```
145
+
117
146
  ## Development
118
147
 
119
148
  After checking out the repo, run `bundle install` to install dependencies. Then, run `bundle exec rspec` to run the tests.
@@ -0,0 +1,21 @@
1
+ require_relative '../lib/upperkut/worker'
2
+ require_relative '../lib/upperkut/strategies/priority_queue'
3
+
4
+ class PriorityWorker
5
+ include Upperkut::Worker
6
+
7
+ setup_upperkut do |config|
8
+ config.strategy = Upperkut::Strategies::PriorityQueue.new(
9
+ self,
10
+ priority_key: -> { |item| item['tenant_id'] },
11
+ batch_size: 1
12
+ )
13
+ end
14
+
15
+ def perform(items)
16
+ items.each do |item|
17
+ puts "event dispatched: #{item.inspect}"
18
+ sleep 1
19
+ end
20
+ end
21
+ end
@@ -57,12 +57,13 @@ module Upperkut
57
57
  handle_signal(signal)
58
58
  end
59
59
  rescue Interrupt
60
+ timeout = Integer(ENV['UPPERKUT_TIMEOUT'] || 8)
60
61
  @logger.info(
61
- 'Stopping managers, wait for 5 seconds and them kill processors'
62
+ "Stopping managers, wait for #{timeout} seconds and them kill processors"
62
63
  )
63
64
 
64
65
  manager.stop
65
- sleep(Integer(ENV['UPPERKUT_TIMEOUT'] || 8))
66
+ sleep(timeout)
66
67
  manager.kill
67
68
  exit(0)
68
69
  end
@@ -0,0 +1,50 @@
1
+ require 'securerandom'
2
+
3
+ module Upperkut
4
+ class Item
5
+ attr_reader :id, :body, :enqueued_at
6
+
7
+ def initialize(body:, id: nil, enqueued_at: nil)
8
+ raise ArgumentError, 'Body should be a Hash' unless body.is_a?(Hash)
9
+
10
+ @body = body
11
+ @id = id || SecureRandom.uuid
12
+ @enqueued_at = enqueued_at || Time.now.utc.to_i
13
+ @nacked = false
14
+ end
15
+
16
+ def [](key)
17
+ @body[key]
18
+ end
19
+
20
+ def []=(key, value)
21
+ @body[key] = value
22
+ end
23
+
24
+ def key?(key)
25
+ @body.key?(key)
26
+ end
27
+
28
+ def nack
29
+ @nacked = true
30
+ end
31
+
32
+ def nacked?
33
+ @nacked
34
+ end
35
+
36
+ def to_json
37
+ JSON.generate(
38
+ 'id' => @id,
39
+ 'body' => @body,
40
+ 'enqueued_at' => @enqueued_at
41
+ )
42
+ end
43
+
44
+ def self.from_json(item_json)
45
+ hash = JSON.parse(item_json)
46
+ id, body, enqueued_at = hash.values_at('id', 'body', 'enqueued_at')
47
+ new(id: id, body: body, enqueued_at: enqueued_at)
48
+ end
49
+ end
50
+ end
@@ -1,44 +1,50 @@
1
1
  require_relative 'core_ext'
2
- require_relative 'processor'
2
+ require_relative 'worker_thread'
3
+ require_relative 'logging'
3
4
  require_relative 'worker'
4
5
 
5
6
  module Upperkut
6
7
  class Manager
7
8
  attr_accessor :worker
8
- attr_reader :stopped, :logger, :concurrency, :processors
9
+ attr_reader :stopped, :logger, :concurrency
9
10
 
10
11
  def initialize(opts = {})
11
12
  self.worker = opts.fetch(:worker).constantize
12
13
  @concurrency = opts.fetch(:concurrency, 1)
13
- @logger = opts.fetch(:logger, Upperkut::Logging.logger)
14
+ @logger = opts.fetch(:logger, Logging.logger)
14
15
 
15
16
  @stopped = false
16
- @processors = []
17
+ @threads = []
17
18
  end
18
19
 
19
20
  def run
20
21
  @concurrency.times do
21
- processor = Processor.new(self)
22
- @processors << processor
23
- processor.run
22
+ spawn_thread
24
23
  end
25
24
  end
26
25
 
27
26
  def stop
28
27
  @stopped = true
28
+ @threads.each(&:stop)
29
29
  end
30
30
 
31
31
  def kill
32
- @processors.each(&:kill)
32
+ @threads.each(&:kill)
33
33
  end
34
34
 
35
- def notify_killed_processor(processor)
36
- @processors.delete(processor)
37
- return if @stopped
35
+ def notify_killed_processor(thread)
36
+ @threads.delete(thread)
37
+ spawn_thread unless @stopped
38
+ end
39
+
40
+ private
41
+
42
+ def spawn_thread
43
+ processor = Processor.new(worker, logger)
38
44
 
39
- processor = Processor.new(self)
40
- @processors << processor
41
- processor.run
45
+ thread = WorkerThread.new(self, processor)
46
+ @threads << thread
47
+ thread.run
42
48
  end
43
49
  end
44
50
  end
@@ -1,58 +1,63 @@
1
- require_relative 'batch_execution'
1
+ require_relative 'logging'
2
2
 
3
3
  module Upperkut
4
4
  class Processor
5
- def initialize(manager)
6
- @manager = manager
7
- @worker = @manager.worker
8
- @logger = @manager.logger
9
- @strategy = @worker.strategy
10
-
11
- @sleeping_time = 0
5
+ def initialize(worker, logger = Logging.logger)
6
+ @worker = worker
7
+ @strategy = worker.strategy
8
+ @worker_instance = worker.new
9
+ @logger = logger
12
10
  end
13
11
 
14
- def run
15
- @thread ||= Thread.new do
16
- begin
17
- process
18
- rescue Exception => e
19
- @logger.debug(
20
- action: :processor_killed,
21
- reason: e,
22
- stacktrace: e.backtrace
23
- )
24
-
25
- @manager.notify_killed_processor(self)
26
- end
12
+ def process
13
+ items = @worker.fetch_items.freeze
14
+
15
+ @worker.server_middlewares.invoke(@worker, items) do
16
+ @worker_instance.perform(items)
27
17
  end
28
- end
29
18
 
30
- def kill
31
- return unless @thread
19
+ nacked_items, pending_ack_items = items.partition(&:nacked?)
20
+ @strategy.nack(nacked_items) if nacked_items.any?
21
+ @strategy.ack(pending_ack_items) if pending_ack_items.any?
22
+ rescue StandardError => error
23
+ @logger.error(
24
+ action: :handle_execution_error,
25
+ ex: error.to_s,
26
+ backtrace: error.backtrace.join("\n"),
27
+ item_size: Array(items).size
28
+ )
29
+
30
+ if items
31
+ if @worker_instance.respond_to?(:handle_error)
32
+ @worker_instance.handle_error(error, items)
33
+ return
34
+ end
35
+
36
+ @strategy.nack(items)
37
+ end
32
38
 
33
- @thread.raise Upperkut::Shutdown
34
- @thread.value # wait
39
+ raise error
35
40
  end
36
41
 
37
- private
42
+ def blocking_process
43
+ sleeping_time = 0
38
44
 
39
- def process
40
45
  loop do
41
- next if @manager.stopped
46
+ break if @stopped
42
47
 
43
48
  if @strategy.process?
44
- @sleeping_time = 0
45
- process_batch
49
+ sleeping_time = 0
50
+ process
46
51
  next
47
52
  end
48
53
 
49
- @sleeping_time += sleep(@worker.setup.polling_interval)
50
- @logger.debug(sleeping_time: @sleeping_time)
54
+ sleeping_time += sleep(@worker.setup.polling_interval)
55
+ @logger.debug(sleeping_time: sleeping_time)
51
56
  end
52
57
  end
53
58
 
54
- def process_batch
55
- BatchExecution.new(@worker, @logger).execute
59
+ def stop
60
+ @stopped = true
56
61
  end
57
62
  end
58
63
  end