hutch-schedule 0.6.1 → 0.7.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +24 -0
- data/README.md +4 -14
- data/hutch-schedule.gemspec +10 -7
- data/lib/active_job/queue_adapters/hutch_adapter.rb +14 -13
- data/lib/hutch/enqueue.rb +14 -12
- data/lib/hutch/error_handlers/max_retry.rb +7 -7
- data/lib/hutch/patch/config.rb +28 -0
- data/lib/hutch/patch/worker.rb +99 -0
- data/lib/hutch/schedule.rb +16 -7
- data/lib/hutch/schedule/core.rb +11 -11
- data/lib/hutch/schedule/version.rb +1 -1
- data/lib/hutch/threshold.rb +77 -0
- metadata +56 -12
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 67a8a0f9f0a464729740d0bb57ae9d1d10d5005927de1e746fc440ea9c15bbc0
|
4
|
+
data.tar.gz: 5b66aca0219764dfefc211cf4ee55bdd17dc13d9cb02a61e5cce39fbb03ee470
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4b2db45314d63e5ed2b2a12810146e1d09aa85194a2a575246d83fc4cd850de06f055dcff4e489bcf1889058fbd1891e0479c53a796efb8e7cfa3eb577dcd5cb
|
7
|
+
data.tar.gz: ac1bd4cd5e6469d1d4d80690c29119a5929f947669753f54504a619b69a93f07549147198fcb669f3315703ce6e18646e6a8267eead5415abee9f14c9150fab3
|
data/CHANGELOG.md
CHANGED
@@ -1,6 +1,30 @@
|
|
1
1
|
# Change Log
|
2
2
|
All notable changes to this project will be documented in this file.
|
3
3
|
|
4
|
+
## [0.7.0] - 2020-04-13
|
5
|
+
### Fixed
|
6
|
+
- Use monkey patch to support Conumser ratelimit through Hutch::Threshold
|
7
|
+
- Use monkey patch to expand Hutch::Config add more work pool relate configurations
|
8
|
+
- Upgrade Hutch to 1.0
|
9
|
+
|
10
|
+
## [0.6.1] - 2018-05-23
|
11
|
+
### Fixed
|
12
|
+
- When message requeue merge the message headers properties
|
13
|
+
|
14
|
+
## [0.6.0] - 2018-04-17
|
15
|
+
### Fixed
|
16
|
+
- Reuse Hutch::Enqueue in HutchAdapter#dynamic_consumer
|
17
|
+
|
18
|
+
## [0.5.1] - 2018-04-15
|
19
|
+
### Fixed
|
20
|
+
- Fix dynamic_consumer name/inspect/to_s method issue
|
21
|
+
- Add delay time from seconds to days
|
22
|
+
|
23
|
+
## [0.4.3] - 2017-06-06
|
24
|
+
### Fixed
|
25
|
+
- Support #dynamic_consumer for HutchAdapter to adoption ActiveJob
|
26
|
+
- Provider correct #name for dynamic_consumer
|
27
|
+
|
4
28
|
## [0.4.2] - 2017-05-23
|
5
29
|
### Fixed
|
6
30
|
- Reuse consumer when register ActiveJob queue to consumer
|
data/README.md
CHANGED
@@ -84,9 +84,7 @@ Add an `hutch.rb` to `conf/initializers`:
|
|
84
84
|
Hutch::Config.load_from_file(Rails.root.join('config', 'config.yaml'))
|
85
85
|
# replace error_handlers with Hutch::ErrorHandlers::MaxRetry
|
86
86
|
Hutch::Config.error_handlers = [Hutch::ErrorHandlers::MaxRetry.new]
|
87
|
-
# Init Hutch
|
88
|
-
Hutch.connect
|
89
|
-
# Init Hutch::Schedule
|
87
|
+
# Init Hutch and Hutch::Schedule
|
90
88
|
Hutch::Schedule.connect
|
91
89
|
```
|
92
90
|
|
@@ -128,14 +126,6 @@ The gem is available as open source under the terms of the [MIT License](http://
|
|
128
126
|
## Performance
|
129
127
|
Use the repo: https://github.com/wppurking/hutch-schedule-demo
|
130
128
|
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
#### Hutch publish message
|
135
|
-
![Hutch publish message](http://ofooyx8i9.bkt.clouddn.com/enqueue_raw.jpg)
|
136
|
-
|
137
|
-
#### ActiveJob dequeue to execute
|
138
|
-
![ActiveJob dequeue to execute](http://ofooyx8i9.bkt.clouddn.com/consume.jpg)
|
139
|
-
|
140
|
-
#### Hutch dequeue to execute
|
141
|
-
![Hutch dequeue to execute](http://ofooyx8i9.bkt.clouddn.com/consume2.jpg)
|
129
|
+
# TODO
|
130
|
+
* add cron job support
|
131
|
+
* add unique job support
|
data/hutch-schedule.gemspec
CHANGED
@@ -8,25 +8,28 @@ Gem::Specification.new do |spec|
|
|
8
8
|
spec.version = Hutch::Schedule::VERSION
|
9
9
|
spec.authors = ["Wyatt pan"]
|
10
10
|
spec.email = ["wppurking@gmail.com"]
|
11
|
-
|
11
|
+
|
12
12
|
spec.summary = %q{Add Schedule and Error Retry To Hutch.}
|
13
13
|
spec.description = %q{Add Schedule and Error Retry To Hutch.}
|
14
14
|
spec.homepage = "https://github.com/wppurking/hutch-schedule"
|
15
15
|
spec.license = "MIT"
|
16
|
-
|
16
|
+
|
17
17
|
spec.files = `git ls-files -z`.split("\x0").reject do |f|
|
18
18
|
f.match(%r{^(test|spec|features)/})
|
19
19
|
end
|
20
20
|
spec.bindir = "exe"
|
21
21
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
22
22
|
spec.require_paths = ["lib"]
|
23
|
-
|
24
|
-
spec.add_runtime_dependency 'hutch', '~> 0
|
23
|
+
|
24
|
+
spec.add_runtime_dependency 'hutch', '~> 1.0'
|
25
25
|
spec.add_runtime_dependency "multi_json"
|
26
|
-
|
26
|
+
spec.add_runtime_dependency "concurrent-ruby", "~> 1.1"
|
27
|
+
spec.add_runtime_dependency "ratelimit", "~> 1.0"
|
28
|
+
spec.add_runtime_dependency "activesupport", ">= 5.0"
|
29
|
+
|
27
30
|
spec.add_development_dependency "activejob"
|
28
|
-
spec.add_development_dependency "bundler", "
|
29
|
-
spec.add_development_dependency "rake", "
|
31
|
+
spec.add_development_dependency "bundler", ">= 1.14"
|
32
|
+
spec.add_development_dependency "rake", ">= 12.3.3"
|
30
33
|
spec.add_development_dependency "rspec", "~> 3.0"
|
31
34
|
spec.add_development_dependency "timecop", "~> 0.8"
|
32
35
|
end
|
@@ -10,30 +10,30 @@ module ActiveJob
|
|
10
10
|
class HutchAdapter
|
11
11
|
# All activejob Message will routing to one RabbitMQ Queue.
|
12
12
|
# Because Hutch will one Consumer per Queue
|
13
|
-
AJ_ROUTING_KEY
|
13
|
+
AJ_ROUTING_KEY = "active_job"
|
14
14
|
@@queue_consumers = {}
|
15
|
-
|
15
|
+
|
16
16
|
def initialize
|
17
17
|
@monitor = Monitor.new
|
18
18
|
end
|
19
|
-
|
19
|
+
|
20
20
|
def enqueue(job) #:nodoc:
|
21
21
|
@monitor.synchronize do
|
22
22
|
@@queue_consumers[job.queue_name].enqueue(job.serialize)
|
23
23
|
end
|
24
24
|
end
|
25
|
-
|
25
|
+
|
26
26
|
def enqueue_at(job, timestamp) #:nodoc:
|
27
27
|
@monitor.synchronize do
|
28
28
|
@@queue_consumers[job.queue_name].enqueue_at(timestamp, job.serialize)
|
29
29
|
end
|
30
30
|
end
|
31
|
-
|
31
|
+
|
32
32
|
# Get an routing_key
|
33
33
|
def self.routing_key(job)
|
34
34
|
"#{AJ_ROUTING_KEY}.#{job.queue_name}"
|
35
35
|
end
|
36
|
-
|
36
|
+
|
37
37
|
# Register all ActiveJob Class to Hutch. (per queue per consumer)
|
38
38
|
def self.register_actice_job_classes
|
39
39
|
# TODO: 需要考虑如何将 AJ 的 Proc queue_name 动态注册到 Hutch
|
@@ -47,32 +47,33 @@ module ActiveJob
|
|
47
47
|
Hutch.register_consumer(@@queue_consumers[job.queue_name])
|
48
48
|
end
|
49
49
|
end
|
50
|
-
|
50
|
+
|
51
51
|
private
|
52
|
+
|
52
53
|
def self.dynamic_consumer(job_instance)
|
53
54
|
Class.new do
|
54
55
|
# don't include Hutch::Consumer, we should change the name of consumer to registe
|
55
56
|
extend Hutch::Consumer::ClassMethods
|
56
57
|
include Hutch::Enqueue
|
57
|
-
|
58
|
+
|
58
59
|
attr_accessor :broker, :delivery_info
|
59
|
-
|
60
|
+
|
60
61
|
queue_name job_instance.queue_name
|
61
62
|
consume HutchAdapter.routing_key(job_instance)
|
62
|
-
|
63
|
+
|
63
64
|
def process(job_data)
|
64
65
|
ActiveJob::Base.execute(job_data)
|
65
66
|
end
|
66
|
-
|
67
|
+
|
67
68
|
define_singleton_method :name do
|
68
69
|
"#{job_instance.queue_name}_dynamic_consumer".camelize
|
69
70
|
end
|
70
|
-
|
71
|
+
|
71
72
|
# inspect name
|
72
73
|
define_singleton_method :inspect do
|
73
74
|
"#{job_instance.queue_name}_dynamic_consumer".camelize
|
74
75
|
end
|
75
|
-
|
76
|
+
|
76
77
|
define_singleton_method :to_s do
|
77
78
|
"#{job_instance.queue_name}_dynamic_consumer".camelize
|
78
79
|
end
|
data/lib/hutch/enqueue.rb
CHANGED
@@ -1,37 +1,39 @@
|
|
1
1
|
require 'active_support/concern'
|
2
2
|
require 'active_support/core_ext/numeric/time'
|
3
3
|
require 'hutch/schedule'
|
4
|
+
require 'hutch/threshold'
|
4
5
|
|
5
6
|
module Hutch
|
6
7
|
# If consumer need `enqueue`, just include this module
|
7
8
|
module Enqueue
|
8
9
|
extend ActiveSupport::Concern
|
9
|
-
|
10
|
+
include Hutch::Threshold
|
11
|
+
|
10
12
|
# Add Consumer methods
|
11
13
|
class_methods do
|
12
14
|
# Publish the message to this consumer with one routing_key
|
13
15
|
def enqueue(message)
|
14
16
|
Hutch.publish(enqueue_routing_key, message)
|
15
17
|
end
|
16
|
-
|
18
|
+
|
17
19
|
# publish message at a delay times
|
18
20
|
# interval: delay interval seconds
|
19
21
|
# message: publish message
|
20
22
|
def enqueue_in(interval, message, props = {})
|
21
23
|
# TODO: 超过 3h 的延迟也会接收, 但是不会延迟那么长时间, 但给予 warn
|
22
24
|
delay_seconds = delay_seconds_level(interval)
|
23
|
-
|
25
|
+
|
24
26
|
# 设置固定的延迟, 利用 headers 中的 CC, 以及区分的 topic, 将消息重新投递进入队列
|
25
|
-
prop_headers
|
26
|
-
properties
|
27
|
+
prop_headers = props[:headers] || {}
|
28
|
+
properties = props.merge(
|
27
29
|
expiration: (delay_seconds * 1000).to_i,
|
28
|
-
headers:
|
30
|
+
headers: prop_headers.merge(CC: [enqueue_routing_key])
|
29
31
|
)
|
30
32
|
delay_routing_key = Hutch::Schedule.delay_routing_key("#{delay_seconds}s")
|
31
|
-
|
33
|
+
|
32
34
|
Hutch::Schedule.publish(delay_routing_key, message, properties)
|
33
35
|
end
|
34
|
-
|
36
|
+
|
35
37
|
# delay at exatly time point
|
36
38
|
def enqueue_at(time, message, props = {})
|
37
39
|
# compatible with with ActiveJob API
|
@@ -40,22 +42,22 @@ module Hutch
|
|
40
42
|
interval = [(time_or_timestamp - Time.now.utc.to_f), 1.second].max
|
41
43
|
enqueue_in(interval, message, props)
|
42
44
|
end
|
43
|
-
|
45
|
+
|
44
46
|
# routing_key: the purpose is to send message to hutch exchange and then routing to the correct queue,
|
45
47
|
# so can use any of them routing_key that the consumer is consuming.
|
46
48
|
def enqueue_routing_key
|
47
49
|
raise "Routing Keys is not set!" if routing_keys.size < 1
|
48
50
|
routing_keys.to_a.last
|
49
51
|
end
|
50
|
-
|
52
|
+
|
51
53
|
def attempts(times)
|
52
54
|
@max_retries = [times, 0].max
|
53
55
|
end
|
54
|
-
|
56
|
+
|
55
57
|
def max_attempts
|
56
58
|
@max_retries || 0
|
57
59
|
end
|
58
|
-
|
60
|
+
|
59
61
|
# 计算 delay 的 level
|
60
62
|
# 5s 10s 20s 30s
|
61
63
|
# 60s 120s 180s 240s 300s 360s 420s 480s 540s 600s 1200s 1800s 2400s
|
@@ -3,12 +3,12 @@ require 'active_support/core_ext/object/blank'
|
|
3
3
|
|
4
4
|
module Hutch
|
5
5
|
module ErrorHandlers
|
6
|
-
|
6
|
+
|
7
7
|
# When reach the Max Attempts, republish this message to RabbitMQ,
|
8
8
|
# And persisted the properties[:headers] to tell RabbitMQ the `x-dead.count`
|
9
9
|
class MaxRetry
|
10
10
|
include Logging
|
11
|
-
|
11
|
+
|
12
12
|
# properties.headers example:
|
13
13
|
# {
|
14
14
|
# "x-death": [
|
@@ -40,7 +40,7 @@ module Hutch
|
|
40
40
|
logger.warn("Consumer: #{consumer} is not include Hutch::Enqueue can`t use #enqueue_in`")
|
41
41
|
return false
|
42
42
|
end
|
43
|
-
|
43
|
+
|
44
44
|
prop_headers = properties[:headers] || {}
|
45
45
|
attempts = failure_count(prop_headers, consumer) + 1
|
46
46
|
if attempts <= consumer.max_attempts
|
@@ -51,13 +51,13 @@ module Hutch
|
|
51
51
|
logger.debug("failing, retry_count=#{attempts}, ex:#{ex}")
|
52
52
|
end
|
53
53
|
end
|
54
|
-
|
54
|
+
|
55
55
|
# becareful with the RabbitMQ fixed delay level, this retry_dealy seconds will fit to one fixed delay level.
|
56
56
|
# so the max delay time is limit to 3 hours(10800s, error times 11: 14643)
|
57
57
|
def retry_delay(executes)
|
58
|
-
(executes**4) + 2
|
58
|
+
(executes ** 4) + 2
|
59
59
|
end
|
60
|
-
|
60
|
+
|
61
61
|
def failure_count(headers, consumer)
|
62
62
|
if headers.nil? || headers['x-death'].nil?
|
63
63
|
0
|
@@ -66,7 +66,7 @@ module Hutch
|
|
66
66
|
# http://ruby-doc.org/stdlib-2.2.3/libdoc/set/rdoc/Set.html#method-i-intersect-3F
|
67
67
|
(x_death['routing-keys'].presence || []).to_set.intersect?(consumer.routing_keys)
|
68
68
|
end
|
69
|
-
|
69
|
+
|
70
70
|
if x_death_array.count > 0 && x_death_array.first['count']
|
71
71
|
# Newer versions of RabbitMQ return headers with a count key
|
72
72
|
x_death_array.inject(0) { |sum, x_death| sum + x_death['count'] }
|
@@ -0,0 +1,28 @@
|
|
1
|
+
module Hutch
|
2
|
+
module Config
|
3
|
+
# Hutch Schedule work pool size
|
4
|
+
number_setting :worker_pool_size, 20
|
5
|
+
|
6
|
+
# Hutch Schedule exceeded checker and poller interval seconds
|
7
|
+
# Hutch Worker Pool heartbeat interval (share interval config)
|
8
|
+
number_setting :poller_interval, 1
|
9
|
+
|
10
|
+
# Hutch Schedule poller batch size
|
11
|
+
number_setting :poller_batch_size, 100
|
12
|
+
|
13
|
+
# Ratelimit redis url
|
14
|
+
string_setting :ratelimit_redis_url, "redis://127.0.0.1:6379/0"
|
15
|
+
|
16
|
+
# Ratelimit bucket interval
|
17
|
+
number_setting :ratelimit_bucket_interval, 1
|
18
|
+
|
19
|
+
initialize(
|
20
|
+
worker_pool_size: 20,
|
21
|
+
poller_interval: 1,
|
22
|
+
poller_batch_size: 100,
|
23
|
+
# @see Redis::Client
|
24
|
+
ratelimit_redis_url: "redis://127.0.0.1:6379/0",
|
25
|
+
ratelimit_bucket_interval: 1
|
26
|
+
)
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,99 @@
|
|
1
|
+
require 'concurrent'
|
2
|
+
|
3
|
+
module Hutch
|
4
|
+
# Monkey patch worker. 因为 hutch 是借用的底层的 bunny 的 ConsumerWorkPool 来完成的并发任务处理,
|
5
|
+
# 但这个 Pool 太过于通用, 而我们需要针对 rabbitmq 传入过来的 Message 进行处理, 需要在任务执行的过程中
|
6
|
+
# 添加额外的处理信息, 所以我们不由 ConsumerWorkPool 来处理任务, 改为由 ConsumerWorkPool 执行一次任务提交,
|
7
|
+
# 由 Bunny::ConsumerWorkPool 将需要执行的 block 提交给自己的 WorkerPool 来进行最终执行.
|
8
|
+
# 因为 RabbitMQ 队列中的任务是需要手动 Ack 才会标记完成, 并且根据 Channel 会有 Prefetch, 所以结合这两个特性
|
9
|
+
# 则可以利用本地进程中的 Queue 进行缓存任务, 只要没有执行会有 Prefetch 控制当前节点缓存的总任务数, 根据 Ack 会
|
10
|
+
# 明确告知 RabbitMQ 此任务完成.
|
11
|
+
class Worker
|
12
|
+
def initialize(broker, consumers, setup_procs)
|
13
|
+
@broker = broker
|
14
|
+
self.consumers = consumers
|
15
|
+
self.setup_procs = setup_procs
|
16
|
+
|
17
|
+
@message_worker = Concurrent::FixedThreadPool.new(Hutch::Config.get(:worker_pool_size))
|
18
|
+
@timer_worker = Concurrent::TimerTask.execute(execution_interval: Hutch::Config.get(:poller_interval)) do
|
19
|
+
heartbeat_connection
|
20
|
+
retry_buffer_queue
|
21
|
+
end
|
22
|
+
@buffer_queue = ::Queue.new
|
23
|
+
@batch_size = Hutch::Config.get(:poller_batch_size)
|
24
|
+
@connected = Hutch.connected?
|
25
|
+
end
|
26
|
+
|
27
|
+
# 停止两个线程池
|
28
|
+
def shutdown
|
29
|
+
@message_worker.shutdown
|
30
|
+
@timer_worker.shutdown
|
31
|
+
end
|
32
|
+
|
33
|
+
# Bind a consumer's routing keys to its queue, and set up a subscription to
|
34
|
+
# receive messages sent to the queue.
|
35
|
+
def setup_queue(consumer)
|
36
|
+
logger.info "setting up queue: #{consumer.get_queue_name}"
|
37
|
+
|
38
|
+
queue = @broker.queue(consumer.get_queue_name, consumer.get_arguments)
|
39
|
+
@broker.bind_queue(queue, consumer.routing_keys)
|
40
|
+
|
41
|
+
queue.subscribe(consumer_tag: unique_consumer_tag, manual_ack: true) do |*args|
|
42
|
+
delivery_info, properties, payload = Hutch::Adapter.decode_message(*args)
|
43
|
+
handle_message_with_limits(consumer, delivery_info, properties, payload)
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
def handle_message_with_limits(consumer, delivery_info, properties, payload)
|
48
|
+
# 1. consumer.limit?
|
49
|
+
# 2. yes: make and ConsumerMsg to queue
|
50
|
+
# 3. no: post handle
|
51
|
+
@message_worker.post do
|
52
|
+
if consumer.ratelimit_exceeded?
|
53
|
+
@buffer_queue.push(ConsumerMsg.new(consumer, delivery_info, properties, payload))
|
54
|
+
else
|
55
|
+
# if Hutch disconnect skip do work let message timeout in rabbitmq waiting message push again
|
56
|
+
return unless @connected
|
57
|
+
consumer.ratelimit_add
|
58
|
+
handle_message(consumer, delivery_info, properties, payload)
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
# 心跳检查 Hutch 的连接
|
64
|
+
def heartbeat_connection
|
65
|
+
@connected = Hutch.connected?
|
66
|
+
end
|
67
|
+
|
68
|
+
# 每隔一段时间, 从 buffer queue 中转移任务到执行
|
69
|
+
def retry_buffer_queue
|
70
|
+
@batch_size.times do
|
71
|
+
cmsg = peak
|
72
|
+
return if cmsg.blank?
|
73
|
+
handle_message_with_limits(cmsg.consumer, cmsg.delivery_info, cmsg.properties, cmsg.payload)
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
# non-blocking pop message, if empty return nil. other error raise exception
|
78
|
+
def peak
|
79
|
+
@buffer_queue.pop(true)
|
80
|
+
rescue ThreadError => e
|
81
|
+
nil if e.to_s == "queue empty"
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
# Consumer Message wrap rabbitmq message infomation
|
86
|
+
class ConsumerMsg
|
87
|
+
attr_reader :consumer, :delivery_info, :properties, :payload
|
88
|
+
|
89
|
+
def initialize(consumer, delivery_info, properties, payload)
|
90
|
+
@consumer = consumer
|
91
|
+
@delivery_info = delivery_info
|
92
|
+
@properties = properties
|
93
|
+
@payload = payload
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
|
99
|
+
|
data/lib/hutch/schedule.rb
CHANGED
@@ -1,9 +1,12 @@
|
|
1
1
|
require 'active_support/core_ext/module/delegation'
|
2
2
|
require 'active_support/core_ext/object/blank'
|
3
3
|
require 'hutch'
|
4
|
+
require 'hutch/patch/config'
|
5
|
+
require 'hutch/patch/worker'
|
6
|
+
require 'hutch/schedule/core'
|
4
7
|
require 'hutch/enqueue'
|
8
|
+
require 'hutch/threshold'
|
5
9
|
require 'hutch/error_handlers/max_retry'
|
6
|
-
require 'hutch/schedule/core'
|
7
10
|
|
8
11
|
# If ActiveJob is requried then required the adapter
|
9
12
|
if defined?(ActiveJob)
|
@@ -15,13 +18,13 @@ module Hutch
|
|
15
18
|
# If you want use it, just do `Hutch::Schedule.connect(Hutch.broker)` to initialize it
|
16
19
|
# and then just use like Hutch to publish message `Hutch::Schedule.publish`
|
17
20
|
module Schedule
|
18
|
-
|
21
|
+
|
19
22
|
# fixed delay levels
|
20
23
|
# seconds(4): 5s, 10s, 20s, 30s
|
21
24
|
# minutes(14): 1m, 2m, 3m, 4m, 5m, 6m, 7m, 8m, 9m, 10m, 20m, 30m, 40m, 50m
|
22
25
|
# hours(3): 1h, 2h, 3h
|
23
26
|
DELAY_QUEUES = %w(5s 10s 20s 30s 60s 120s 180s 240s 300s 360s 420s 480s 540s 600s 1200s 1800s 2400s 3000s 3600s 7200s 10800s)
|
24
|
-
|
27
|
+
|
25
28
|
class << self
|
26
29
|
def connect
|
27
30
|
ActiveJob::QueueAdapters::HutchAdapter.register_actice_job_classes if defined?(ActiveJob::QueueAdapters::HutchAdapter)
|
@@ -31,20 +34,26 @@ module Hutch
|
|
31
34
|
@core = Hutch::Schedule::Core.new(Hutch.broker)
|
32
35
|
@core.connect!
|
33
36
|
end
|
34
|
-
|
37
|
+
|
38
|
+
def disconnect
|
39
|
+
Hutch.disconnect if Hutch.connected?
|
40
|
+
@core = nil
|
41
|
+
end
|
42
|
+
|
43
|
+
|
35
44
|
def core
|
36
45
|
@core
|
37
46
|
end
|
38
|
-
|
47
|
+
|
39
48
|
def publish(*args)
|
40
49
|
core.publish(*args)
|
41
50
|
end
|
42
|
-
|
51
|
+
|
43
52
|
# fixed delay level queue's routing_key
|
44
53
|
def delay_routing_key(suffix)
|
45
54
|
"#{Hutch::Config.get(:mq_exchange)}.schedule.#{suffix}"
|
46
55
|
end
|
47
|
-
|
56
|
+
|
48
57
|
def delay_queue_name(suffix)
|
49
58
|
"#{Hutch::Config.get(:mq_exchange)}_delay_queue_#{suffix}"
|
50
59
|
end
|
data/lib/hutch/schedule/core.rb
CHANGED
@@ -8,52 +8,52 @@ module Hutch
|
|
8
8
|
class Core
|
9
9
|
attr_reader :broker, :exchange
|
10
10
|
delegate :channel, :connection, :logger, to: :broker
|
11
|
-
|
11
|
+
|
12
12
|
def initialize(broker)
|
13
13
|
raise "Broker can`t be nil" if broker.blank?
|
14
14
|
@broker = broker
|
15
15
|
end
|
16
|
-
|
16
|
+
|
17
17
|
# Becareful with the sequence of initialize
|
18
18
|
def connect!
|
19
19
|
declare_delay_exchange!
|
20
20
|
declare_publisher!
|
21
21
|
setup_delay_queues!
|
22
22
|
end
|
23
|
-
|
23
|
+
|
24
24
|
def declare_publisher!
|
25
25
|
@publisher = Hutch::Publisher.new(connection, channel, exchange)
|
26
26
|
end
|
27
|
-
|
27
|
+
|
28
28
|
# The exchange used by Hutch::Schedule
|
29
29
|
def declare_delay_exchange!
|
30
30
|
@exchange = declare_delay_exchange
|
31
31
|
end
|
32
|
-
|
32
|
+
|
33
33
|
def declare_delay_exchange(ch = channel)
|
34
|
-
exchange_name
|
34
|
+
exchange_name = "#{Hutch::Config.get(:mq_exchange)}.schedule"
|
35
35
|
exchange_options = { durable: true }.merge(Hutch::Config.get(:mq_exchange_options))
|
36
36
|
logger.info "using topic exchange(schedule) '#{exchange_name}'"
|
37
|
-
|
37
|
+
|
38
38
|
broker.send(:with_bunny_precondition_handler, 'schedule exchange') do
|
39
39
|
ch.topic(exchange_name, exchange_options)
|
40
40
|
end
|
41
41
|
end
|
42
|
-
|
42
|
+
|
43
43
|
# The queue used by Hutch::Schedule
|
44
44
|
def setup_delay_queues!
|
45
45
|
DELAY_QUEUES.map { |suffix| setup_delay_queue!(suffix) }
|
46
46
|
end
|
47
|
-
|
47
|
+
|
48
48
|
def setup_delay_queue!(suffix)
|
49
49
|
# TODO: extract the ttl to config params
|
50
50
|
props = { :'x-message-ttl' => 30.days.in_milliseconds, :'x-dead-letter-exchange' => Hutch::Config.get(:mq_exchange) }
|
51
51
|
queue = broker.queue(Hutch::Schedule.delay_queue_name(suffix), props)
|
52
|
-
|
52
|
+
|
53
53
|
# bind routing_key to schedule exchange
|
54
54
|
queue.bind(exchange, routing_key: Hutch::Schedule.delay_routing_key(suffix))
|
55
55
|
end
|
56
|
-
|
56
|
+
|
57
57
|
# Schedule`s publisher, publish the message to schedule topic exchange
|
58
58
|
def publish(*args)
|
59
59
|
@publisher.publish(*args)
|
@@ -0,0 +1,77 @@
|
|
1
|
+
require 'active_support/concern'
|
2
|
+
require 'ratelimit'
|
3
|
+
|
4
|
+
module Hutch
|
5
|
+
# If consumer need `threshold`, just include this module. (included with Hutch::Enqueue)
|
6
|
+
module Threshold
|
7
|
+
extend ActiveSupport::Concern
|
8
|
+
|
9
|
+
# Add Consumer methods
|
10
|
+
class_methods do
|
11
|
+
# 限流速度:
|
12
|
+
# context: 可选的上下文, 默认为 default
|
13
|
+
# rate: 数量
|
14
|
+
# interval: 间隔
|
15
|
+
#
|
16
|
+
# block: 采用 lambada 的方式计算, 但需要返回三个参数:
|
17
|
+
# - context: 当前约束的上下文
|
18
|
+
# - rate: 数量
|
19
|
+
# - interval: 间隔
|
20
|
+
# 例如:
|
21
|
+
# - rate: 1, interval: 1, 每秒 1 个
|
22
|
+
# - rate: 30, interval: 1, 每秒 30 个
|
23
|
+
# - rate: 30, interval: 5, 每 5 s, 30 个
|
24
|
+
def threshold(args)
|
25
|
+
@block_given = args.is_a?(Proc)
|
26
|
+
if @block_given
|
27
|
+
@threshold_block = args
|
28
|
+
else
|
29
|
+
raise "need args or block" if args.blank?
|
30
|
+
raise "args need hash type" if args.class != Hash
|
31
|
+
args.symbolize_keys!
|
32
|
+
@context = args[:context].presence || "default"
|
33
|
+
@rate = args[:rate]
|
34
|
+
@interval = args[:interval]
|
35
|
+
end
|
36
|
+
# call redis.ping let fail fast if redis is not avalible
|
37
|
+
_redis.ping
|
38
|
+
# redis: 传入设置的 redis
|
39
|
+
# bucket_interval: 记录的间隔, 越小精度越大
|
40
|
+
@rate_limiter = Ratelimit.new(self.name,
|
41
|
+
bucket_interval: Hutch::Config.get(:ratelimit_bucket_interval),
|
42
|
+
redis: _redis)
|
43
|
+
end
|
44
|
+
|
45
|
+
# is class level @rate_limiter _context exceeded?
|
46
|
+
# if class level @rate_limiter is nil alwayt return false
|
47
|
+
def ratelimit_exceeded?
|
48
|
+
return false if @rate_limiter.blank?
|
49
|
+
@rate_limiter.exceeded?(_context, threshold: _rate, interval: _interval)
|
50
|
+
end
|
51
|
+
|
52
|
+
# 增加一次调用
|
53
|
+
def ratelimit_add
|
54
|
+
return if @rate_limiter.blank?
|
55
|
+
@rate_limiter.add(_context)
|
56
|
+
end
|
57
|
+
|
58
|
+
def _context
|
59
|
+
@block_given ? @threshold_block.call[:context] : @context
|
60
|
+
end
|
61
|
+
|
62
|
+
def _rate
|
63
|
+
@block_given ? @threshold_block.call[:rate] : @rate
|
64
|
+
end
|
65
|
+
|
66
|
+
def _interval
|
67
|
+
@block_given ? @threshold_block.call[:interval] : @interval
|
68
|
+
end
|
69
|
+
|
70
|
+
# all Consumers that use threshold module shared the same redis instance
|
71
|
+
def _redis
|
72
|
+
@@redis ||= Redis.new(url: Hutch::Config.get(:ratelimit_redis_url))
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: hutch-schedule
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.7.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Wyatt pan
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-04-12 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: hutch
|
@@ -16,14 +16,14 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: '0
|
19
|
+
version: '1.0'
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: '0
|
26
|
+
version: '1.0'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: multi_json
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
@@ -38,6 +38,48 @@ dependencies:
|
|
38
38
|
- - ">="
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: concurrent-ruby
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '1.1'
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '1.1'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: ratelimit
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '1.0'
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '1.0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: activesupport
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - ">="
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '5.0'
|
76
|
+
type: :runtime
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - ">="
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '5.0'
|
41
83
|
- !ruby/object:Gem::Dependency
|
42
84
|
name: activejob
|
43
85
|
requirement: !ruby/object:Gem::Requirement
|
@@ -56,30 +98,30 @@ dependencies:
|
|
56
98
|
name: bundler
|
57
99
|
requirement: !ruby/object:Gem::Requirement
|
58
100
|
requirements:
|
59
|
-
- - "
|
101
|
+
- - ">="
|
60
102
|
- !ruby/object:Gem::Version
|
61
103
|
version: '1.14'
|
62
104
|
type: :development
|
63
105
|
prerelease: false
|
64
106
|
version_requirements: !ruby/object:Gem::Requirement
|
65
107
|
requirements:
|
66
|
-
- - "
|
108
|
+
- - ">="
|
67
109
|
- !ruby/object:Gem::Version
|
68
110
|
version: '1.14'
|
69
111
|
- !ruby/object:Gem::Dependency
|
70
112
|
name: rake
|
71
113
|
requirement: !ruby/object:Gem::Requirement
|
72
114
|
requirements:
|
73
|
-
- - "
|
115
|
+
- - ">="
|
74
116
|
- !ruby/object:Gem::Version
|
75
|
-
version:
|
117
|
+
version: 12.3.3
|
76
118
|
type: :development
|
77
119
|
prerelease: false
|
78
120
|
version_requirements: !ruby/object:Gem::Requirement
|
79
121
|
requirements:
|
80
|
-
- - "
|
122
|
+
- - ">="
|
81
123
|
- !ruby/object:Gem::Version
|
82
|
-
version:
|
124
|
+
version: 12.3.3
|
83
125
|
- !ruby/object:Gem::Dependency
|
84
126
|
name: rspec
|
85
127
|
requirement: !ruby/object:Gem::Requirement
|
@@ -130,9 +172,12 @@ files:
|
|
130
172
|
- lib/hutch-schedule.rb
|
131
173
|
- lib/hutch/enqueue.rb
|
132
174
|
- lib/hutch/error_handlers/max_retry.rb
|
175
|
+
- lib/hutch/patch/config.rb
|
176
|
+
- lib/hutch/patch/worker.rb
|
133
177
|
- lib/hutch/schedule.rb
|
134
178
|
- lib/hutch/schedule/core.rb
|
135
179
|
- lib/hutch/schedule/version.rb
|
180
|
+
- lib/hutch/threshold.rb
|
136
181
|
homepage: https://github.com/wppurking/hutch-schedule
|
137
182
|
licenses:
|
138
183
|
- MIT
|
@@ -152,8 +197,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
152
197
|
- !ruby/object:Gem::Version
|
153
198
|
version: '0'
|
154
199
|
requirements: []
|
155
|
-
|
156
|
-
rubygems_version: 2.7.6
|
200
|
+
rubygems_version: 3.1.2
|
157
201
|
signing_key:
|
158
202
|
specification_version: 4
|
159
203
|
summary: Add Schedule and Error Retry To Hutch.
|