upperkut 0.7.1 → 0.7.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: f8a422257ff9efd3c1cc0f5cc8ca940c31b77df381a549ca1ad9e0fd59919eba
4
- data.tar.gz: 3e6bfc18b1c2890a033dc5b301e4009eeeccdd026ae6c040be5f8d7f551bd8b5
3
+ metadata.gz: 9fb4b45a40c37d26aa8d96f199c79141f8b34d1edc2fd48c751a6d8567b77deb
4
+ data.tar.gz: b4b91ae1d8d2737d20b97c23744cc5cc8052681ede5c68f28790c50f3a092882
5
5
  SHA512:
6
- metadata.gz: 1d8b525f86a878faae9cd8297c75120c1a883b9b6652f6f39669fa6c428055966ae151b1de7115d5e54c5e593fe5e15df499985757e1ea619298aa3098dd71b7
7
- data.tar.gz: 403e4c6cdd0da351bedbd0daeddf96ab8f06269ce3d09a4468c0c996a6a22b55a9af7d3f75fd7aafdca9e867b4ddffc83e0bb55cd24b05519e654421c2df7f8c
6
+ metadata.gz: 2c934b63764f8a673762cdb163989b18fae91d8b2be30e8b99f5db93e4329c3c63f2beb75a8e38c46b966bffb7f4e8e7f09c4f5246e8806b001b0da2ce056f88
7
+ data.tar.gz: '038a2489af4d5b8b0811f30402922e3a7ed4524dceee4b3dcdff1d13c417a037f2f7f88c61122ba65cd0ad2ee1ed9760a4210a94427e69c3d7b2f635e2c3694c'
data/CHANGELOG.md CHANGED
@@ -2,13 +2,12 @@
2
2
 
3
3
  0.7.x
4
4
  ---------
5
+ - Added Scheduled Queue Implementation thanks to @rodrigo-araujo #38
5
6
  - Added Datahog middleware #42 by @gabriel-augusto
6
7
  - Added redis to CI #40 by #henrich-m
7
8
  - Specs improvements #34 and #35, #37 by @gabriel-augusto
8
9
  - Enable Rubocop #32 by @henrich-m
9
10
  - Added codeclimate #31 by @henrich-m
10
-
11
-
12
11
  - Extract Buffered Queue behavior to its own strategy #29
13
12
 
14
13
  0.6.x
data/Gemfile.lock CHANGED
@@ -1,7 +1,7 @@
1
1
  PATH
2
2
  remote: .
3
3
  specs:
4
- upperkut (0.7.1)
4
+ upperkut (0.7.2)
5
5
  connection_pool (~> 2.2, >= 2.2.2)
6
6
  redis (>= 3.3.3, < 5)
7
7
 
data/README.md CHANGED
@@ -23,7 +23,8 @@ Or install it yourself as:
23
23
  $ gem install upperkut
24
24
 
25
25
  ## Usage
26
- Examples:
26
+
27
+ ### Example 1 - Buffered Queue:
27
28
 
28
29
  1) Create a Worker class and the define how to process the batch;
29
30
  ```ruby
@@ -36,7 +37,7 @@ Examples:
36
37
  # Define which redis instance you want to use
37
38
  config.strategy = Upperkut::Strategies::BufferedQueue.new(
38
39
  self,
39
- redis: { url: ENV['ANOTHER_REDIS_INSTANCE_URL']) },
40
+ redis: { url: ENV['ANOTHER_REDIS_INSTANCE_URL'] },
40
41
  batch_size: 400, # How many events should be dispatched to worker.
41
42
  max_wait: 300 # How long Processor wait in seconds to process batch.
42
43
  # even though the amount of items did not reached the
@@ -58,7 +59,54 @@ Examples:
58
59
 
59
60
  2) Start pushings items;
60
61
  ```ruby
61
- Myworker.push_items([{'id' => SecureRandom.uuid, 'name' => 'Robert C Hall', 'action' => 'EMAIL_OPENNED'}])
62
+ Myworker.push_items(
63
+ [
64
+ {
65
+ 'id' => SecureRandom.uuid,
66
+ 'name' => 'Robert C Hall',
67
+ 'action' => 'EMAIL_OPENNED'
68
+ }
69
+ ]
70
+ )
71
+ ```
72
+
73
+ 3) Start Upperkut;
74
+ ```bash
75
+ $ bundle exec upperkut --worker MyWorker --concurrency 10
76
+ ```
77
+
78
+ ### Example 2 - Scheduled Queue:
79
+
80
+ 1) Create a Worker class and the define how to process the batch;
81
+ ```ruby
82
+ require 'upperkut/strategies/scheduled_queue'
83
+ class MyWorker
84
+ include Upperkut::Worker
85
+
86
+ setup_upperkut do |config|
87
+ config.strategy = Upperkut::Strategies::ScheduledQueue.new(self)
88
+ end
89
+
90
+ def perform(batch_items)
91
+ heavy_processing(batch_items)
92
+ process_metrics(batch_items)
93
+ end
94
+ end
95
+ ```
96
+
97
+ 2) Start pushings items with `timestamp` param;
98
+ ```ruby
99
+ # timestamp is 'Thu, 10 May 2019 23:43:58 GMT'
100
+ Myworker.push_items(
101
+ [
102
+ {
103
+ 'timestamp' => '1557531838',
104
+ 'id' => SecureRandom.uuid,
105
+ 'name' => 'Robert C Hall',
106
+ 'action' => 'SEND_NOTIFICATION'
107
+ }
108
+ ]
109
+ )
62
110
  ```
63
111
 
64
112
  3) Start Upperkut;
@@ -0,0 +1,19 @@
1
+ require_relative '../lib/upperkut/worker'
2
+ require_relative '../lib/upperkut/strategies/scheduled_queue'
3
+
4
+ class ScheduledWorker
5
+ include Upperkut::Worker
6
+
7
+ setup_upperkut do |config|
8
+ config.strategy = Upperkut::Strategies::ScheduledQueue.new(
9
+ self,
10
+ batch_size: 200
11
+ )
12
+ end
13
+
14
+ def perform(items)
15
+ items.each do |item|
16
+ puts "event dispatched: #{item.inspect}"
17
+ end
18
+ end
19
+ end
@@ -18,7 +18,8 @@ module Upperkut
18
18
  rescue Exception => e
19
19
  @logger.debug(
20
20
  action: :processor_killed,
21
- reason: e
21
+ reason: e,
22
+ stacktrace: e.backtrace
22
23
  )
23
24
 
24
25
  @manager.notify_killed_processor(self)
@@ -0,0 +1,156 @@
1
+ require 'time'
2
+ require 'upperkut/util'
3
+ require 'upperkut/redis_pool'
4
+ require 'upperkut/strategies/base'
5
+
6
+ module Upperkut
7
+ module Strategies
8
+ # Public: Encapsulates methods required to build a Scheculed Queue
9
+ # Items are queued, but are only fetched at a specific point in time.
10
+ class ScheduledQueue < Upperkut::Strategies::Base
11
+ include Upperkut::Util
12
+
13
+ ZPOPBYRANGE = %(
14
+ local score_from = ARGV[1]
15
+ local score_to = ARGV[2]
16
+ local limit = ARGV[3]
17
+
18
+ local values = redis.call('zrangebyscore', KEYS[1], score_from, score_to, 'LIMIT', '0', limit)
19
+
20
+ if table.getn(values) > 0 then
21
+ redis.call('zrem', KEYS[1], unpack(values))
22
+ end
23
+
24
+ return values
25
+ ).freeze
26
+
27
+ attr_reader :options
28
+
29
+ def initialize(worker, options = {})
30
+ @options = options
31
+ initialize_options
32
+ @redis_pool = setup_redis_pool
33
+ @worker = worker
34
+ end
35
+
36
+ def push_items(items = [])
37
+ items = [items] if items.is_a?(Hash)
38
+ return false if items.empty?
39
+
40
+ redis do |conn|
41
+ items.each do |item|
42
+ ensure_timestamp_attr(item)
43
+ conn.zadd(key, item['timestamp'], encode_json_item(item))
44
+ end
45
+ end
46
+
47
+ true
48
+ end
49
+
50
+ def fetch_items
51
+ args = {
52
+ value_from: '-inf'.freeze,
53
+ value_to: Time.now.utc.to_f.to_s,
54
+ limit: @batch_size
55
+ }
56
+ items = []
57
+
58
+ redis do |conn|
59
+ items = pop_values(conn, args)
60
+ end
61
+
62
+ decode_json_items(items)
63
+ end
64
+
65
+ def clear
66
+ redis { |conn| conn.del(key) }
67
+ end
68
+
69
+ def metrics
70
+ {
71
+ 'latency' => latency,
72
+ 'size' => size
73
+ }
74
+ end
75
+
76
+ def process?
77
+ buff_size = size('-inf', Time.now.utc.to_i)
78
+ return true if fulfill_condition?(buff_size)
79
+
80
+ false
81
+ end
82
+
83
+ private
84
+
85
+ def initialize_options
86
+ @redis_options = @options.fetch(:redis, {})
87
+
88
+ @batch_size = @options.fetch(
89
+ :batch_size,
90
+ Integer(ENV['UPPERKUT_BATCH_SIZE'] || 1000)
91
+ )
92
+ end
93
+
94
+ def pop_values(redis_client, args)
95
+ value_from = args[:value_from]
96
+ value_to = args[:value_to]
97
+ limit = args[:limit]
98
+ redis_client.eval(ZPOPBYRANGE, keys: [key], argv: [value_from, value_to, limit])
99
+ end
100
+
101
+ def fulfill_condition?(buff_size)
102
+ !buff_size.zero?
103
+ end
104
+
105
+ def size(min = '-inf', max = '+inf')
106
+ redis do |conn|
107
+ conn.zcount(key, min, max)
108
+ end
109
+ end
110
+
111
+ def latency
112
+ now = Time.now.utc
113
+ now_timestamp = now.to_f
114
+ job = nil
115
+
116
+ redis do |conn|
117
+ job = conn.zrangebyscore(key, '-inf'.freeze, now_timestamp.to_s, limit: [0, 1]).first
118
+ job = decode_json_items([job]).first
119
+ end
120
+
121
+ return 0 unless job
122
+
123
+ now_timestamp - job['body'].fetch('timestamp', now).to_f
124
+ end
125
+
126
+ def setup_redis_pool
127
+ return @redis_options if @redis_options.is_a?(ConnectionPool)
128
+
129
+ RedisPool.new(options.fetch(:redis, {})).create
130
+ end
131
+
132
+ def redis
133
+ raise ArgumentError, 'requires a block' unless block_given?
134
+
135
+ @redis_pool.with do |conn|
136
+ yield conn
137
+ end
138
+ end
139
+
140
+ def key
141
+ "upperkut:queued:#{to_underscore(@worker.name)}"
142
+ end
143
+
144
+ def ensure_timestamp_attr(item)
145
+ item['timestamp'] = Time.now.utc.to_i unless item.key?('timestamp')
146
+ end
147
+
148
+ def encode_json_item(item)
149
+ JSON.generate(
150
+ 'enqueued_at' => Time.now.utc.to_i,
151
+ 'body' => item
152
+ )
153
+ end
154
+ end
155
+ end
156
+ end
@@ -1,3 +1,3 @@
1
1
  module Upperkut
2
- VERSION = '0.7.1'.freeze
2
+ VERSION = '0.7.2'.freeze
3
3
  end
@@ -34,7 +34,7 @@ module Upperkut
34
34
  @config ||=
35
35
  begin
36
36
  config = Upperkut::Configuration.default.clone
37
- config.strategy = Upperkut::Strategies::BufferedQueue.new(self)
37
+ config.strategy ||= Upperkut::Strategies::BufferedQueue.new(self)
38
38
  config
39
39
  end
40
40
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: upperkut
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.7.1
4
+ version: 0.7.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Nando Sousa
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2019-04-26 00:00:00.000000000 Z
11
+ date: 2019-04-29 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: connection_pool
@@ -113,6 +113,7 @@ files:
113
113
  - Rakefile
114
114
  - bin/upperkut
115
115
  - examples/basic.rb
116
+ - examples/scheduled_worker.rb
116
117
  - examples/with_middlewares.rb
117
118
  - lib/upperkut.rb
118
119
  - lib/upperkut/batch_execution.rb
@@ -128,6 +129,7 @@ files:
128
129
  - lib/upperkut/redis_pool.rb
129
130
  - lib/upperkut/strategies/base.rb
130
131
  - lib/upperkut/strategies/buffered_queue.rb
132
+ - lib/upperkut/strategies/scheduled_queue.rb
131
133
  - lib/upperkut/util.rb
132
134
  - lib/upperkut/version.rb
133
135
  - lib/upperkut/worker.rb