upperkut 0.7.2 → 1.0.0.rc

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -28,19 +28,23 @@ module Upperkut
28
28
 
29
29
  def initialize(worker, options = {})
30
30
  @options = options
31
- initialize_options
32
- @redis_pool = setup_redis_pool
31
+ @redis_options = @options.fetch(:redis, {})
33
32
  @worker = worker
33
+
34
+ @batch_size = @options.fetch(
35
+ :batch_size,
36
+ Integer(ENV['UPPERKUT_BATCH_SIZE'] || 1000)
37
+ )
34
38
  end
35
39
 
36
40
  def push_items(items = [])
37
- items = [items] if items.is_a?(Hash)
41
+ items = normalize_items(items)
38
42
  return false if items.empty?
39
43
 
40
44
  redis do |conn|
41
45
  items.each do |item|
42
46
  ensure_timestamp_attr(item)
43
- conn.zadd(key, item['timestamp'], encode_json_item(item))
47
+ conn.zadd(key, item['timestamp'], item.to_json)
44
48
  end
45
49
  end
46
50
 
@@ -66,6 +70,12 @@ module Upperkut
66
70
  redis { |conn| conn.del(key) }
67
71
  end
68
72
 
73
+ def ack(_items); end
74
+
75
+ def nack(items)
76
+ push_items(items)
77
+ end
78
+
69
79
  def metrics
70
80
  {
71
81
  'latency' => latency,
@@ -82,13 +92,12 @@ module Upperkut
82
92
 
83
93
  private
84
94
 
85
- def initialize_options
86
- @redis_options = @options.fetch(:redis, {})
95
+ def key
96
+ "upperkut:queued:#{to_underscore(@worker.name)}"
97
+ end
87
98
 
88
- @batch_size = @options.fetch(
89
- :batch_size,
90
- Integer(ENV['UPPERKUT_BATCH_SIZE'] || 1000)
91
- )
99
+ def ensure_timestamp_attr(item)
100
+ item['timestamp'] = Time.now.utc.to_i unless item.key?('timestamp')
92
101
  end
93
102
 
94
103
  def pop_values(redis_client, args)
@@ -120,36 +129,27 @@ module Upperkut
120
129
 
121
130
  return 0 unless job
122
131
 
123
- now_timestamp - job['body'].fetch('timestamp', now).to_f
124
- end
125
-
126
- def setup_redis_pool
127
- return @redis_options if @redis_options.is_a?(ConnectionPool)
128
-
129
- RedisPool.new(options.fetch(:redis, {})).create
132
+ now_timestamp - job['timestamp'].to_f
130
133
  end
131
134
 
132
135
  def redis
133
136
  raise ArgumentError, 'requires a block' unless block_given?
134
137
 
135
- @redis_pool.with do |conn|
136
- yield conn
138
+ retry_block do
139
+ redis_pool.with do |conn|
140
+ yield conn
141
+ end
137
142
  end
138
143
  end
139
144
 
140
- def key
141
- "upperkut:queued:#{to_underscore(@worker.name)}"
142
- end
143
-
144
- def ensure_timestamp_attr(item)
145
- item['timestamp'] = Time.now.utc.to_i unless item.key?('timestamp')
146
- end
147
-
148
- def encode_json_item(item)
149
- JSON.generate(
150
- 'enqueued_at' => Time.now.utc.to_i,
151
- 'body' => item
152
- )
145
+ def redis_pool
146
+ @redis_pool ||= begin
147
+ if @redis_options.is_a?(ConnectionPool)
148
+ @redis_options
149
+ else
150
+ RedisPool.new(@redis_options).create
151
+ end
152
+ end
153
153
  end
154
154
  end
155
155
  end
@@ -1,4 +1,5 @@
1
1
  require 'json'
2
+ require 'upperkut/item'
2
3
 
3
4
  module Upperkut
4
5
  module Util
@@ -12,22 +13,45 @@ module Upperkut
12
13
  klass_name
13
14
  end
14
15
 
15
- def encode_json_items(items)
16
- items = items.collect do |i|
17
- JSON.generate(
18
- 'enqueued_at' => Time.now.to_i,
19
- 'body' => i
20
- )
16
+ # Public:
17
+ # Normalize hash and hash arrays into a hash of Items.
18
+ # An Item object contains metadata, for example the timestamp from the moment it was enqueued,
19
+ # that we need to carry through multiple execution tries.
20
+ #
21
+ # When the execution fails, we need to schedule the whole batch for retry, and scheduling
22
+ # an Item will make Upperkut understand that we're not dealing with a new batch,
23
+ # so metrics like latency will increase.
24
+ def normalize_items(items)
25
+ items = [items] unless items.is_a?(Array)
26
+
27
+ items.map do |item|
28
+ next item if item.is_a?(Item)
29
+
30
+ Item.new(body: item)
21
31
  end
22
32
  end
23
33
 
24
34
  def decode_json_items(items)
25
- items.collect! do |i|
26
- JSON.parse(i) if i
35
+ items.each_with_object([]) do |item, memo|
36
+ memo << Item.from_json(item) if item
27
37
  end
38
+ end
28
39
 
29
- items.compact!
30
- items
40
+ def retry_block(retries_limit = 3, base_sleep = 2)
41
+ retries = 0
42
+
43
+ begin
44
+ yield
45
+ rescue StandardError => err
46
+ if retries < retries_limit
47
+ retries += 1
48
+ sleep_time = base_sleep**retries
49
+ Kernel.sleep(sleep_time)
50
+ retry
51
+ end
52
+
53
+ raise err
54
+ end
31
55
  end
32
56
  end
33
57
  end
@@ -1,3 +1,3 @@
1
1
  module Upperkut
2
- VERSION = '0.7.2'.freeze
2
+ VERSION = '1.0.0.rc'.freeze
3
3
  end
@@ -1,7 +1,6 @@
1
1
  require 'forwardable'
2
2
  require 'upperkut/strategies/buffered_queue'
3
3
  require 'upperkut/middleware'
4
- require 'upperkut/util'
5
4
  require 'upperkut'
6
5
 
7
6
  module Upperkut
@@ -0,0 +1,37 @@
1
+ require_relative 'processor'
2
+
3
+ module Upperkut
4
+ class WorkerThread
5
+ def initialize(manager, processor)
6
+ @manager = manager
7
+ @processor = processor
8
+ end
9
+
10
+ def run
11
+ @thread ||= Thread.new do
12
+ begin
13
+ @processor.blocking_process
14
+ rescue Exception => e
15
+ @manager.logger.debug(
16
+ action: :processor_killed,
17
+ reason: e,
18
+ stacktrace: e.backtrace
19
+ )
20
+
21
+ @manager.notify_killed_processor(self)
22
+ end
23
+ end
24
+ end
25
+
26
+ def stop
27
+ @processor.stop
28
+ end
29
+
30
+ def kill
31
+ return unless @thread
32
+
33
+ @thread.raise Upperkut::Shutdown
34
+ @thread.value # wait
35
+ end
36
+ end
37
+ end
@@ -22,8 +22,8 @@ Gem::Specification.new do |spec|
22
22
  spec.required_ruby_version = '>= 2.2.2'
23
23
 
24
24
  spec.add_dependency 'connection_pool', '~> 2.2', '>= 2.2.2'
25
- spec.add_dependency 'redis', '>= 3.3.3', '< 5'
26
- spec.add_development_dependency 'bundler', '~> 1.16'
27
- spec.add_development_dependency 'rake', '~> 10.0'
25
+ spec.add_dependency 'redis', '>= 4.1.0', '< 5.0.0'
26
+ spec.add_development_dependency 'bundler', '>= 1.16'
27
+ spec.add_development_dependency 'rake', '~> 13.0'
28
28
  spec.add_development_dependency 'rspec', '~> 3.0'
29
29
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: upperkut
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.7.2
4
+ version: 1.0.0.rc
5
5
  platform: ruby
6
6
  authors:
7
7
  - Nando Sousa
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2019-04-29 00:00:00.000000000 Z
11
+ date: 2020-07-31 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: connection_pool
@@ -36,32 +36,32 @@ dependencies:
36
36
  requirements:
37
37
  - - ">="
38
38
  - !ruby/object:Gem::Version
39
- version: 3.3.3
39
+ version: 4.1.0
40
40
  - - "<"
41
41
  - !ruby/object:Gem::Version
42
- version: '5'
42
+ version: 5.0.0
43
43
  type: :runtime
44
44
  prerelease: false
45
45
  version_requirements: !ruby/object:Gem::Requirement
46
46
  requirements:
47
47
  - - ">="
48
48
  - !ruby/object:Gem::Version
49
- version: 3.3.3
49
+ version: 4.1.0
50
50
  - - "<"
51
51
  - !ruby/object:Gem::Version
52
- version: '5'
52
+ version: 5.0.0
53
53
  - !ruby/object:Gem::Dependency
54
54
  name: bundler
55
55
  requirement: !ruby/object:Gem::Requirement
56
56
  requirements:
57
- - - "~>"
57
+ - - ">="
58
58
  - !ruby/object:Gem::Version
59
59
  version: '1.16'
60
60
  type: :development
61
61
  prerelease: false
62
62
  version_requirements: !ruby/object:Gem::Requirement
63
63
  requirements:
64
- - - "~>"
64
+ - - ">="
65
65
  - !ruby/object:Gem::Version
66
66
  version: '1.16'
67
67
  - !ruby/object:Gem::Dependency
@@ -70,14 +70,14 @@ dependencies:
70
70
  requirements:
71
71
  - - "~>"
72
72
  - !ruby/object:Gem::Version
73
- version: '10.0'
73
+ version: '13.0'
74
74
  type: :development
75
75
  prerelease: false
76
76
  version_requirements: !ruby/object:Gem::Requirement
77
77
  requirements:
78
78
  - - "~>"
79
79
  - !ruby/object:Gem::Version
80
- version: '10.0'
80
+ version: '13.0'
81
81
  - !ruby/object:Gem::Dependency
82
82
  name: rspec
83
83
  requirement: !ruby/object:Gem::Requirement
@@ -113,12 +113,13 @@ files:
113
113
  - Rakefile
114
114
  - bin/upperkut
115
115
  - examples/basic.rb
116
+ - examples/priority_worker.rb
116
117
  - examples/scheduled_worker.rb
117
118
  - examples/with_middlewares.rb
118
119
  - lib/upperkut.rb
119
- - lib/upperkut/batch_execution.rb
120
120
  - lib/upperkut/cli.rb
121
121
  - lib/upperkut/core_ext.rb
122
+ - lib/upperkut/item.rb
122
123
  - lib/upperkut/logging.rb
123
124
  - lib/upperkut/manager.rb
124
125
  - lib/upperkut/middleware.rb
@@ -129,10 +130,12 @@ files:
129
130
  - lib/upperkut/redis_pool.rb
130
131
  - lib/upperkut/strategies/base.rb
131
132
  - lib/upperkut/strategies/buffered_queue.rb
133
+ - lib/upperkut/strategies/priority_queue.rb
132
134
  - lib/upperkut/strategies/scheduled_queue.rb
133
135
  - lib/upperkut/util.rb
134
136
  - lib/upperkut/version.rb
135
137
  - lib/upperkut/worker.rb
138
+ - lib/upperkut/worker_thread.rb
136
139
  - upperkut.gemspec
137
140
  homepage: http://shipit.resultadosdigitais.com.br/open-source/
138
141
  licenses:
@@ -149,12 +152,11 @@ required_ruby_version: !ruby/object:Gem::Requirement
149
152
  version: 2.2.2
150
153
  required_rubygems_version: !ruby/object:Gem::Requirement
151
154
  requirements:
152
- - - ">="
155
+ - - ">"
153
156
  - !ruby/object:Gem::Version
154
- version: '0'
157
+ version: 1.3.1
155
158
  requirements: []
156
- rubyforge_project:
157
- rubygems_version: 2.7.8
159
+ rubygems_version: 3.1.2
158
160
  signing_key:
159
161
  specification_version: 4
160
162
  summary: Batch background processing tool
@@ -1,36 +0,0 @@
1
- require_relative 'logging'
2
-
3
- module Upperkut
4
- class BatchExecution
5
- include Upperkut::Util
6
-
7
- def initialize(worker, logger = Upperkut::Logging.logger)
8
- @worker = worker
9
- @logger = logger
10
- end
11
-
12
- def execute
13
- worker_instance = @worker.new
14
- items = @worker.fetch_items.freeze
15
-
16
- items_body = items.collect do |item|
17
- item['body']
18
- end
19
-
20
- @worker.server_middlewares.invoke(@worker, items) do
21
- worker_instance.perform(items_body.dup)
22
- end
23
- rescue Exception => ex
24
- @worker.push_items(items_body)
25
-
26
- @logger.info(
27
- action: :requeue,
28
- ex: ex,
29
- item_size: items_body.size
30
- )
31
-
32
- @logger.error(ex.backtrace.join("\n"))
33
- raise ex
34
- end
35
- end
36
- end