hutch-schedule 0.6.1 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +24 -0
- data/README.md +4 -14
- data/hutch-schedule.gemspec +10 -7
- data/lib/active_job/queue_adapters/hutch_adapter.rb +14 -13
- data/lib/hutch/enqueue.rb +14 -12
- data/lib/hutch/error_handlers/max_retry.rb +7 -7
- data/lib/hutch/patch/config.rb +28 -0
- data/lib/hutch/patch/worker.rb +99 -0
- data/lib/hutch/schedule.rb +16 -7
- data/lib/hutch/schedule/core.rb +11 -11
- data/lib/hutch/schedule/version.rb +1 -1
- data/lib/hutch/threshold.rb +77 -0
- metadata +56 -12
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 67a8a0f9f0a464729740d0bb57ae9d1d10d5005927de1e746fc440ea9c15bbc0
|
4
|
+
data.tar.gz: 5b66aca0219764dfefc211cf4ee55bdd17dc13d9cb02a61e5cce39fbb03ee470
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4b2db45314d63e5ed2b2a12810146e1d09aa85194a2a575246d83fc4cd850de06f055dcff4e489bcf1889058fbd1891e0479c53a796efb8e7cfa3eb577dcd5cb
|
7
|
+
data.tar.gz: ac1bd4cd5e6469d1d4d80690c29119a5929f947669753f54504a619b69a93f07549147198fcb669f3315703ce6e18646e6a8267eead5415abee9f14c9150fab3
|
data/CHANGELOG.md
CHANGED
@@ -1,6 +1,30 @@
|
|
1
1
|
# Change Log
|
2
2
|
All notable changes to this project will be documented in this file.
|
3
3
|
|
4
|
+
## [0.7.0] - 2020-04-13
|
5
|
+
### Fixed
|
6
|
+
- Use monkey patch to support Conumser ratelimit through Hutch::Threshold
|
7
|
+
- Use monkey patch to expand Hutch::Config add more work pool relate configurations
|
8
|
+
- Upgrade Hutch to 1.0
|
9
|
+
|
10
|
+
## [0.6.1] - 2018-05-23
|
11
|
+
### Fixed
|
12
|
+
- When message requeue merge the message headers properties
|
13
|
+
|
14
|
+
## [0.6.0] - 2018-04-17
|
15
|
+
### Fixed
|
16
|
+
- Reuse Hutch::Enqueue in HutchAdapter#dynamic_consumer
|
17
|
+
|
18
|
+
## [0.5.1] - 2018-04-15
|
19
|
+
### Fixed
|
20
|
+
- Fix dynamic_consumer name/inspect/to_s method issue
|
21
|
+
- Add delay time from seconds to days
|
22
|
+
|
23
|
+
## [0.4.3] - 2017-06-06
|
24
|
+
### Fixed
|
25
|
+
- Support #dynamic_consumer for HutchAdapter to adoption ActiveJob
|
26
|
+
- Provider correct #name for dynamic_consumer
|
27
|
+
|
4
28
|
## [0.4.2] - 2017-05-23
|
5
29
|
### Fixed
|
6
30
|
- Reuse consumer when register ActiveJob queue to consumer
|
data/README.md
CHANGED
@@ -84,9 +84,7 @@ Add an `hutch.rb` to `conf/initializers`:
|
|
84
84
|
Hutch::Config.load_from_file(Rails.root.join('config', 'config.yaml'))
|
85
85
|
# replace error_handlers with Hutch::ErrorHandlers::MaxRetry
|
86
86
|
Hutch::Config.error_handlers = [Hutch::ErrorHandlers::MaxRetry.new]
|
87
|
-
# Init Hutch
|
88
|
-
Hutch.connect
|
89
|
-
# Init Hutch::Schedule
|
87
|
+
# Init Hutch and Hutch::Schedule
|
90
88
|
Hutch::Schedule.connect
|
91
89
|
```
|
92
90
|
|
@@ -128,14 +126,6 @@ The gem is available as open source under the terms of the [MIT License](http://
|
|
128
126
|
## Performance
|
129
127
|
Use the repo: https://github.com/wppurking/hutch-schedule-demo
|
130
128
|
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
#### Hutch publish message
|
135
|
-

|
136
|
-
|
137
|
-
#### ActiveJob dequeue to execute
|
138
|
-

|
139
|
-
|
140
|
-
#### Hutch dequeue to execute
|
141
|
-

|
129
|
+
# TODO
|
130
|
+
* add cron job support
|
131
|
+
* add unique job support
|
data/hutch-schedule.gemspec
CHANGED
@@ -8,25 +8,28 @@ Gem::Specification.new do |spec|
|
|
8
8
|
spec.version = Hutch::Schedule::VERSION
|
9
9
|
spec.authors = ["Wyatt pan"]
|
10
10
|
spec.email = ["wppurking@gmail.com"]
|
11
|
-
|
11
|
+
|
12
12
|
spec.summary = %q{Add Schedule and Error Retry To Hutch.}
|
13
13
|
spec.description = %q{Add Schedule and Error Retry To Hutch.}
|
14
14
|
spec.homepage = "https://github.com/wppurking/hutch-schedule"
|
15
15
|
spec.license = "MIT"
|
16
|
-
|
16
|
+
|
17
17
|
spec.files = `git ls-files -z`.split("\x0").reject do |f|
|
18
18
|
f.match(%r{^(test|spec|features)/})
|
19
19
|
end
|
20
20
|
spec.bindir = "exe"
|
21
21
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
22
22
|
spec.require_paths = ["lib"]
|
23
|
-
|
24
|
-
spec.add_runtime_dependency 'hutch', '~> 0
|
23
|
+
|
24
|
+
spec.add_runtime_dependency 'hutch', '~> 1.0'
|
25
25
|
spec.add_runtime_dependency "multi_json"
|
26
|
-
|
26
|
+
spec.add_runtime_dependency "concurrent-ruby", "~> 1.1"
|
27
|
+
spec.add_runtime_dependency "ratelimit", "~> 1.0"
|
28
|
+
spec.add_runtime_dependency "activesupport", ">= 5.0"
|
29
|
+
|
27
30
|
spec.add_development_dependency "activejob"
|
28
|
-
spec.add_development_dependency "bundler", "
|
29
|
-
spec.add_development_dependency "rake", "
|
31
|
+
spec.add_development_dependency "bundler", ">= 1.14"
|
32
|
+
spec.add_development_dependency "rake", ">= 12.3.3"
|
30
33
|
spec.add_development_dependency "rspec", "~> 3.0"
|
31
34
|
spec.add_development_dependency "timecop", "~> 0.8"
|
32
35
|
end
|
@@ -10,30 +10,30 @@ module ActiveJob
|
|
10
10
|
class HutchAdapter
|
11
11
|
# All activejob Message will routing to one RabbitMQ Queue.
|
12
12
|
# Because Hutch will one Consumer per Queue
|
13
|
-
AJ_ROUTING_KEY
|
13
|
+
AJ_ROUTING_KEY = "active_job"
|
14
14
|
@@queue_consumers = {}
|
15
|
-
|
15
|
+
|
16
16
|
def initialize
|
17
17
|
@monitor = Monitor.new
|
18
18
|
end
|
19
|
-
|
19
|
+
|
20
20
|
def enqueue(job) #:nodoc:
|
21
21
|
@monitor.synchronize do
|
22
22
|
@@queue_consumers[job.queue_name].enqueue(job.serialize)
|
23
23
|
end
|
24
24
|
end
|
25
|
-
|
25
|
+
|
26
26
|
def enqueue_at(job, timestamp) #:nodoc:
|
27
27
|
@monitor.synchronize do
|
28
28
|
@@queue_consumers[job.queue_name].enqueue_at(timestamp, job.serialize)
|
29
29
|
end
|
30
30
|
end
|
31
|
-
|
31
|
+
|
32
32
|
# Get an routing_key
|
33
33
|
def self.routing_key(job)
|
34
34
|
"#{AJ_ROUTING_KEY}.#{job.queue_name}"
|
35
35
|
end
|
36
|
-
|
36
|
+
|
37
37
|
# Register all ActiveJob Class to Hutch. (per queue per consumer)
|
38
38
|
def self.register_actice_job_classes
|
39
39
|
# TODO: 需要考虑如何将 AJ 的 Proc queue_name 动态注册到 Hutch
|
@@ -47,32 +47,33 @@ module ActiveJob
|
|
47
47
|
Hutch.register_consumer(@@queue_consumers[job.queue_name])
|
48
48
|
end
|
49
49
|
end
|
50
|
-
|
50
|
+
|
51
51
|
private
|
52
|
+
|
52
53
|
def self.dynamic_consumer(job_instance)
|
53
54
|
Class.new do
|
54
55
|
# don't include Hutch::Consumer, we should change the name of consumer to registe
|
55
56
|
extend Hutch::Consumer::ClassMethods
|
56
57
|
include Hutch::Enqueue
|
57
|
-
|
58
|
+
|
58
59
|
attr_accessor :broker, :delivery_info
|
59
|
-
|
60
|
+
|
60
61
|
queue_name job_instance.queue_name
|
61
62
|
consume HutchAdapter.routing_key(job_instance)
|
62
|
-
|
63
|
+
|
63
64
|
def process(job_data)
|
64
65
|
ActiveJob::Base.execute(job_data)
|
65
66
|
end
|
66
|
-
|
67
|
+
|
67
68
|
define_singleton_method :name do
|
68
69
|
"#{job_instance.queue_name}_dynamic_consumer".camelize
|
69
70
|
end
|
70
|
-
|
71
|
+
|
71
72
|
# inspect name
|
72
73
|
define_singleton_method :inspect do
|
73
74
|
"#{job_instance.queue_name}_dynamic_consumer".camelize
|
74
75
|
end
|
75
|
-
|
76
|
+
|
76
77
|
define_singleton_method :to_s do
|
77
78
|
"#{job_instance.queue_name}_dynamic_consumer".camelize
|
78
79
|
end
|
data/lib/hutch/enqueue.rb
CHANGED
@@ -1,37 +1,39 @@
|
|
1
1
|
require 'active_support/concern'
|
2
2
|
require 'active_support/core_ext/numeric/time'
|
3
3
|
require 'hutch/schedule'
|
4
|
+
require 'hutch/threshold'
|
4
5
|
|
5
6
|
module Hutch
|
6
7
|
# If consumer need `enqueue`, just include this module
|
7
8
|
module Enqueue
|
8
9
|
extend ActiveSupport::Concern
|
9
|
-
|
10
|
+
include Hutch::Threshold
|
11
|
+
|
10
12
|
# Add Consumer methods
|
11
13
|
class_methods do
|
12
14
|
# Publish the message to this consumer with one routing_key
|
13
15
|
def enqueue(message)
|
14
16
|
Hutch.publish(enqueue_routing_key, message)
|
15
17
|
end
|
16
|
-
|
18
|
+
|
17
19
|
# publish message at a delay times
|
18
20
|
# interval: delay interval seconds
|
19
21
|
# message: publish message
|
20
22
|
def enqueue_in(interval, message, props = {})
|
21
23
|
# TODO: 超过 3h 的延迟也会接收, 但是不会延迟那么长时间, 但给予 warn
|
22
24
|
delay_seconds = delay_seconds_level(interval)
|
23
|
-
|
25
|
+
|
24
26
|
# 设置固定的延迟, 利用 headers 中的 CC, 以及区分的 topic, 将消息重新投递进入队列
|
25
|
-
prop_headers
|
26
|
-
properties
|
27
|
+
prop_headers = props[:headers] || {}
|
28
|
+
properties = props.merge(
|
27
29
|
expiration: (delay_seconds * 1000).to_i,
|
28
|
-
headers:
|
30
|
+
headers: prop_headers.merge(CC: [enqueue_routing_key])
|
29
31
|
)
|
30
32
|
delay_routing_key = Hutch::Schedule.delay_routing_key("#{delay_seconds}s")
|
31
|
-
|
33
|
+
|
32
34
|
Hutch::Schedule.publish(delay_routing_key, message, properties)
|
33
35
|
end
|
34
|
-
|
36
|
+
|
35
37
|
# delay at exatly time point
|
36
38
|
def enqueue_at(time, message, props = {})
|
37
39
|
# compatible with with ActiveJob API
|
@@ -40,22 +42,22 @@ module Hutch
|
|
40
42
|
interval = [(time_or_timestamp - Time.now.utc.to_f), 1.second].max
|
41
43
|
enqueue_in(interval, message, props)
|
42
44
|
end
|
43
|
-
|
45
|
+
|
44
46
|
# routing_key: the purpose is to send message to hutch exchange and then routing to the correct queue,
|
45
47
|
# so can use any of them routing_key that the consumer is consuming.
|
46
48
|
def enqueue_routing_key
|
47
49
|
raise "Routing Keys is not set!" if routing_keys.size < 1
|
48
50
|
routing_keys.to_a.last
|
49
51
|
end
|
50
|
-
|
52
|
+
|
51
53
|
def attempts(times)
|
52
54
|
@max_retries = [times, 0].max
|
53
55
|
end
|
54
|
-
|
56
|
+
|
55
57
|
def max_attempts
|
56
58
|
@max_retries || 0
|
57
59
|
end
|
58
|
-
|
60
|
+
|
59
61
|
# 计算 delay 的 level
|
60
62
|
# 5s 10s 20s 30s
|
61
63
|
# 60s 120s 180s 240s 300s 360s 420s 480s 540s 600s 1200s 1800s 2400s
|
@@ -3,12 +3,12 @@ require 'active_support/core_ext/object/blank'
|
|
3
3
|
|
4
4
|
module Hutch
|
5
5
|
module ErrorHandlers
|
6
|
-
|
6
|
+
|
7
7
|
# When reach the Max Attempts, republish this message to RabbitMQ,
|
8
8
|
# And persisted the properties[:headers] to tell RabbitMQ the `x-dead.count`
|
9
9
|
class MaxRetry
|
10
10
|
include Logging
|
11
|
-
|
11
|
+
|
12
12
|
# properties.headers example:
|
13
13
|
# {
|
14
14
|
# "x-death": [
|
@@ -40,7 +40,7 @@ module Hutch
|
|
40
40
|
logger.warn("Consumer: #{consumer} is not include Hutch::Enqueue can`t use #enqueue_in`")
|
41
41
|
return false
|
42
42
|
end
|
43
|
-
|
43
|
+
|
44
44
|
prop_headers = properties[:headers] || {}
|
45
45
|
attempts = failure_count(prop_headers, consumer) + 1
|
46
46
|
if attempts <= consumer.max_attempts
|
@@ -51,13 +51,13 @@ module Hutch
|
|
51
51
|
logger.debug("failing, retry_count=#{attempts}, ex:#{ex}")
|
52
52
|
end
|
53
53
|
end
|
54
|
-
|
54
|
+
|
55
55
|
# becareful with the RabbitMQ fixed delay level, this retry_dealy seconds will fit to one fixed delay level.
|
56
56
|
# so the max delay time is limit to 3 hours(10800s, error times 11: 14643)
|
57
57
|
def retry_delay(executes)
|
58
|
-
(executes**4) + 2
|
58
|
+
(executes ** 4) + 2
|
59
59
|
end
|
60
|
-
|
60
|
+
|
61
61
|
def failure_count(headers, consumer)
|
62
62
|
if headers.nil? || headers['x-death'].nil?
|
63
63
|
0
|
@@ -66,7 +66,7 @@ module Hutch
|
|
66
66
|
# http://ruby-doc.org/stdlib-2.2.3/libdoc/set/rdoc/Set.html#method-i-intersect-3F
|
67
67
|
(x_death['routing-keys'].presence || []).to_set.intersect?(consumer.routing_keys)
|
68
68
|
end
|
69
|
-
|
69
|
+
|
70
70
|
if x_death_array.count > 0 && x_death_array.first['count']
|
71
71
|
# Newer versions of RabbitMQ return headers with a count key
|
72
72
|
x_death_array.inject(0) { |sum, x_death| sum + x_death['count'] }
|
@@ -0,0 +1,28 @@
|
|
1
|
+
module Hutch
|
2
|
+
module Config
|
3
|
+
# Hutch Schedule work pool size
|
4
|
+
number_setting :worker_pool_size, 20
|
5
|
+
|
6
|
+
# Hutch Schedule exceeded checker and poller interval seconds
|
7
|
+
# Hutch Worker Pool heartbeat interval (share interval config)
|
8
|
+
number_setting :poller_interval, 1
|
9
|
+
|
10
|
+
# Hutch Schedule poller batch size
|
11
|
+
number_setting :poller_batch_size, 100
|
12
|
+
|
13
|
+
# Ratelimit redis url
|
14
|
+
string_setting :ratelimit_redis_url, "redis://127.0.0.1:6379/0"
|
15
|
+
|
16
|
+
# Ratelimit bucket interval
|
17
|
+
number_setting :ratelimit_bucket_interval, 1
|
18
|
+
|
19
|
+
initialize(
|
20
|
+
worker_pool_size: 20,
|
21
|
+
poller_interval: 1,
|
22
|
+
poller_batch_size: 100,
|
23
|
+
# @see Redis::Client
|
24
|
+
ratelimit_redis_url: "redis://127.0.0.1:6379/0",
|
25
|
+
ratelimit_bucket_interval: 1
|
26
|
+
)
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,99 @@
|
|
1
|
+
require 'concurrent'
|
2
|
+
|
3
|
+
module Hutch
|
4
|
+
# Monkey patch worker. 因为 hutch 是借用的底层的 bunny 的 ConsumerWorkPool 来完成的并发任务处理,
|
5
|
+
# 但这个 Pool 太过于通用, 而我们需要针对 rabbitmq 传入过来的 Message 进行处理, 需要在任务执行的过程中
|
6
|
+
# 添加额外的处理信息, 所以我们不由 ConsumerWorkPool 来处理任务, 改为由 ConsumerWorkPool 执行一次任务提交,
|
7
|
+
# 由 Bunny::ConsumerWorkPool 将需要执行的 block 提交给自己的 WorkerPool 来进行最终执行.
|
8
|
+
# 因为 RabbitMQ 队列中的任务是需要手动 Ack 才会标记完成, 并且根据 Channel 会有 Prefetch, 所以结合这两个特性
|
9
|
+
# 则可以利用本地进程中的 Queue 进行缓存任务, 只要没有执行会有 Prefetch 控制当前节点缓存的总任务数, 根据 Ack 会
|
10
|
+
# 明确告知 RabbitMQ 此任务完成.
|
11
|
+
class Worker
|
12
|
+
def initialize(broker, consumers, setup_procs)
|
13
|
+
@broker = broker
|
14
|
+
self.consumers = consumers
|
15
|
+
self.setup_procs = setup_procs
|
16
|
+
|
17
|
+
@message_worker = Concurrent::FixedThreadPool.new(Hutch::Config.get(:worker_pool_size))
|
18
|
+
@timer_worker = Concurrent::TimerTask.execute(execution_interval: Hutch::Config.get(:poller_interval)) do
|
19
|
+
heartbeat_connection
|
20
|
+
retry_buffer_queue
|
21
|
+
end
|
22
|
+
@buffer_queue = ::Queue.new
|
23
|
+
@batch_size = Hutch::Config.get(:poller_batch_size)
|
24
|
+
@connected = Hutch.connected?
|
25
|
+
end
|
26
|
+
|
27
|
+
# 停止两个线程池
|
28
|
+
def shutdown
|
29
|
+
@message_worker.shutdown
|
30
|
+
@timer_worker.shutdown
|
31
|
+
end
|
32
|
+
|
33
|
+
# Bind a consumer's routing keys to its queue, and set up a subscription to
|
34
|
+
# receive messages sent to the queue.
|
35
|
+
def setup_queue(consumer)
|
36
|
+
logger.info "setting up queue: #{consumer.get_queue_name}"
|
37
|
+
|
38
|
+
queue = @broker.queue(consumer.get_queue_name, consumer.get_arguments)
|
39
|
+
@broker.bind_queue(queue, consumer.routing_keys)
|
40
|
+
|
41
|
+
queue.subscribe(consumer_tag: unique_consumer_tag, manual_ack: true) do |*args|
|
42
|
+
delivery_info, properties, payload = Hutch::Adapter.decode_message(*args)
|
43
|
+
handle_message_with_limits(consumer, delivery_info, properties, payload)
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
def handle_message_with_limits(consumer, delivery_info, properties, payload)
|
48
|
+
# 1. consumer.limit?
|
49
|
+
# 2. yes: make and ConsumerMsg to queue
|
50
|
+
# 3. no: post handle
|
51
|
+
@message_worker.post do
|
52
|
+
if consumer.ratelimit_exceeded?
|
53
|
+
@buffer_queue.push(ConsumerMsg.new(consumer, delivery_info, properties, payload))
|
54
|
+
else
|
55
|
+
# if Hutch disconnect skip do work let message timeout in rabbitmq waiting message push again
|
56
|
+
return unless @connected
|
57
|
+
consumer.ratelimit_add
|
58
|
+
handle_message(consumer, delivery_info, properties, payload)
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
# 心跳检查 Hutch 的连接
|
64
|
+
def heartbeat_connection
|
65
|
+
@connected = Hutch.connected?
|
66
|
+
end
|
67
|
+
|
68
|
+
# 每隔一段时间, 从 buffer queue 中转移任务到执行
|
69
|
+
def retry_buffer_queue
|
70
|
+
@batch_size.times do
|
71
|
+
cmsg = peak
|
72
|
+
return if cmsg.blank?
|
73
|
+
handle_message_with_limits(cmsg.consumer, cmsg.delivery_info, cmsg.properties, cmsg.payload)
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
# non-blocking pop message, if empty return nil. other error raise exception
|
78
|
+
def peak
|
79
|
+
@buffer_queue.pop(true)
|
80
|
+
rescue ThreadError => e
|
81
|
+
nil if e.to_s == "queue empty"
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
# Consumer Message wrap rabbitmq message infomation
|
86
|
+
class ConsumerMsg
|
87
|
+
attr_reader :consumer, :delivery_info, :properties, :payload
|
88
|
+
|
89
|
+
def initialize(consumer, delivery_info, properties, payload)
|
90
|
+
@consumer = consumer
|
91
|
+
@delivery_info = delivery_info
|
92
|
+
@properties = properties
|
93
|
+
@payload = payload
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
|
99
|
+
|
data/lib/hutch/schedule.rb
CHANGED
@@ -1,9 +1,12 @@
|
|
1
1
|
require 'active_support/core_ext/module/delegation'
|
2
2
|
require 'active_support/core_ext/object/blank'
|
3
3
|
require 'hutch'
|
4
|
+
require 'hutch/patch/config'
|
5
|
+
require 'hutch/patch/worker'
|
6
|
+
require 'hutch/schedule/core'
|
4
7
|
require 'hutch/enqueue'
|
8
|
+
require 'hutch/threshold'
|
5
9
|
require 'hutch/error_handlers/max_retry'
|
6
|
-
require 'hutch/schedule/core'
|
7
10
|
|
8
11
|
# If ActiveJob is requried then required the adapter
|
9
12
|
if defined?(ActiveJob)
|
@@ -15,13 +18,13 @@ module Hutch
|
|
15
18
|
# If you want use it, just do `Hutch::Schedule.connect(Hutch.broker)` to initialize it
|
16
19
|
# and then just use like Hutch to publish message `Hutch::Schedule.publish`
|
17
20
|
module Schedule
|
18
|
-
|
21
|
+
|
19
22
|
# fixed delay levels
|
20
23
|
# seconds(4): 5s, 10s, 20s, 30s
|
21
24
|
# minutes(14): 1m, 2m, 3m, 4m, 5m, 6m, 7m, 8m, 9m, 10m, 20m, 30m, 40m, 50m
|
22
25
|
# hours(3): 1h, 2h, 3h
|
23
26
|
DELAY_QUEUES = %w(5s 10s 20s 30s 60s 120s 180s 240s 300s 360s 420s 480s 540s 600s 1200s 1800s 2400s 3000s 3600s 7200s 10800s)
|
24
|
-
|
27
|
+
|
25
28
|
class << self
|
26
29
|
def connect
|
27
30
|
ActiveJob::QueueAdapters::HutchAdapter.register_actice_job_classes if defined?(ActiveJob::QueueAdapters::HutchAdapter)
|
@@ -31,20 +34,26 @@ module Hutch
|
|
31
34
|
@core = Hutch::Schedule::Core.new(Hutch.broker)
|
32
35
|
@core.connect!
|
33
36
|
end
|
34
|
-
|
37
|
+
|
38
|
+
def disconnect
|
39
|
+
Hutch.disconnect if Hutch.connected?
|
40
|
+
@core = nil
|
41
|
+
end
|
42
|
+
|
43
|
+
|
35
44
|
def core
|
36
45
|
@core
|
37
46
|
end
|
38
|
-
|
47
|
+
|
39
48
|
def publish(*args)
|
40
49
|
core.publish(*args)
|
41
50
|
end
|
42
|
-
|
51
|
+
|
43
52
|
# fixed delay level queue's routing_key
|
44
53
|
def delay_routing_key(suffix)
|
45
54
|
"#{Hutch::Config.get(:mq_exchange)}.schedule.#{suffix}"
|
46
55
|
end
|
47
|
-
|
56
|
+
|
48
57
|
def delay_queue_name(suffix)
|
49
58
|
"#{Hutch::Config.get(:mq_exchange)}_delay_queue_#{suffix}"
|
50
59
|
end
|
data/lib/hutch/schedule/core.rb
CHANGED
@@ -8,52 +8,52 @@ module Hutch
|
|
8
8
|
class Core
|
9
9
|
attr_reader :broker, :exchange
|
10
10
|
delegate :channel, :connection, :logger, to: :broker
|
11
|
-
|
11
|
+
|
12
12
|
def initialize(broker)
|
13
13
|
raise "Broker can`t be nil" if broker.blank?
|
14
14
|
@broker = broker
|
15
15
|
end
|
16
|
-
|
16
|
+
|
17
17
|
# Becareful with the sequence of initialize
|
18
18
|
def connect!
|
19
19
|
declare_delay_exchange!
|
20
20
|
declare_publisher!
|
21
21
|
setup_delay_queues!
|
22
22
|
end
|
23
|
-
|
23
|
+
|
24
24
|
def declare_publisher!
|
25
25
|
@publisher = Hutch::Publisher.new(connection, channel, exchange)
|
26
26
|
end
|
27
|
-
|
27
|
+
|
28
28
|
# The exchange used by Hutch::Schedule
|
29
29
|
def declare_delay_exchange!
|
30
30
|
@exchange = declare_delay_exchange
|
31
31
|
end
|
32
|
-
|
32
|
+
|
33
33
|
def declare_delay_exchange(ch = channel)
|
34
|
-
exchange_name
|
34
|
+
exchange_name = "#{Hutch::Config.get(:mq_exchange)}.schedule"
|
35
35
|
exchange_options = { durable: true }.merge(Hutch::Config.get(:mq_exchange_options))
|
36
36
|
logger.info "using topic exchange(schedule) '#{exchange_name}'"
|
37
|
-
|
37
|
+
|
38
38
|
broker.send(:with_bunny_precondition_handler, 'schedule exchange') do
|
39
39
|
ch.topic(exchange_name, exchange_options)
|
40
40
|
end
|
41
41
|
end
|
42
|
-
|
42
|
+
|
43
43
|
# The queue used by Hutch::Schedule
|
44
44
|
def setup_delay_queues!
|
45
45
|
DELAY_QUEUES.map { |suffix| setup_delay_queue!(suffix) }
|
46
46
|
end
|
47
|
-
|
47
|
+
|
48
48
|
def setup_delay_queue!(suffix)
|
49
49
|
# TODO: extract the ttl to config params
|
50
50
|
props = { :'x-message-ttl' => 30.days.in_milliseconds, :'x-dead-letter-exchange' => Hutch::Config.get(:mq_exchange) }
|
51
51
|
queue = broker.queue(Hutch::Schedule.delay_queue_name(suffix), props)
|
52
|
-
|
52
|
+
|
53
53
|
# bind routing_key to schedule exchange
|
54
54
|
queue.bind(exchange, routing_key: Hutch::Schedule.delay_routing_key(suffix))
|
55
55
|
end
|
56
|
-
|
56
|
+
|
57
57
|
# Schedule`s publisher, publish the message to schedule topic exchange
|
58
58
|
def publish(*args)
|
59
59
|
@publisher.publish(*args)
|
@@ -0,0 +1,77 @@
|
|
1
|
+
require 'active_support/concern'
|
2
|
+
require 'ratelimit'
|
3
|
+
|
4
|
+
module Hutch
|
5
|
+
# If consumer need `threshold`, just include this module. (included with Hutch::Enqueue)
|
6
|
+
module Threshold
|
7
|
+
extend ActiveSupport::Concern
|
8
|
+
|
9
|
+
# Add Consumer methods
|
10
|
+
class_methods do
|
11
|
+
# 限流速度:
|
12
|
+
# context: 可选的上下文, 默认为 default
|
13
|
+
# rate: 数量
|
14
|
+
# interval: 间隔
|
15
|
+
#
|
16
|
+
# block: 采用 lambada 的方式计算, 但需要返回三个参数:
|
17
|
+
# - context: 当前约束的上下文
|
18
|
+
# - rate: 数量
|
19
|
+
# - interval: 间隔
|
20
|
+
# 例如:
|
21
|
+
# - rate: 1, interval: 1, 每秒 1 个
|
22
|
+
# - rate: 30, interval: 1, 每秒 30 个
|
23
|
+
# - rate: 30, interval: 5, 每 5 s, 30 个
|
24
|
+
def threshold(args)
|
25
|
+
@block_given = args.is_a?(Proc)
|
26
|
+
if @block_given
|
27
|
+
@threshold_block = args
|
28
|
+
else
|
29
|
+
raise "need args or block" if args.blank?
|
30
|
+
raise "args need hash type" if args.class != Hash
|
31
|
+
args.symbolize_keys!
|
32
|
+
@context = args[:context].presence || "default"
|
33
|
+
@rate = args[:rate]
|
34
|
+
@interval = args[:interval]
|
35
|
+
end
|
36
|
+
# call redis.ping let fail fast if redis is not avalible
|
37
|
+
_redis.ping
|
38
|
+
# redis: 传入设置的 redis
|
39
|
+
# bucket_interval: 记录的间隔, 越小精度越大
|
40
|
+
@rate_limiter = Ratelimit.new(self.name,
|
41
|
+
bucket_interval: Hutch::Config.get(:ratelimit_bucket_interval),
|
42
|
+
redis: _redis)
|
43
|
+
end
|
44
|
+
|
45
|
+
# is class level @rate_limiter _context exceeded?
|
46
|
+
# if class level @rate_limiter is nil alwayt return false
|
47
|
+
def ratelimit_exceeded?
|
48
|
+
return false if @rate_limiter.blank?
|
49
|
+
@rate_limiter.exceeded?(_context, threshold: _rate, interval: _interval)
|
50
|
+
end
|
51
|
+
|
52
|
+
# 增加一次调用
|
53
|
+
def ratelimit_add
|
54
|
+
return if @rate_limiter.blank?
|
55
|
+
@rate_limiter.add(_context)
|
56
|
+
end
|
57
|
+
|
58
|
+
def _context
|
59
|
+
@block_given ? @threshold_block.call[:context] : @context
|
60
|
+
end
|
61
|
+
|
62
|
+
def _rate
|
63
|
+
@block_given ? @threshold_block.call[:rate] : @rate
|
64
|
+
end
|
65
|
+
|
66
|
+
def _interval
|
67
|
+
@block_given ? @threshold_block.call[:interval] : @interval
|
68
|
+
end
|
69
|
+
|
70
|
+
# all Consumers that use threshold module shared the same redis instance
|
71
|
+
def _redis
|
72
|
+
@@redis ||= Redis.new(url: Hutch::Config.get(:ratelimit_redis_url))
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: hutch-schedule
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.7.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Wyatt pan
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2020-04-12 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: hutch
|
@@ -16,14 +16,14 @@ dependencies:
|
|
16
16
|
requirements:
|
17
17
|
- - "~>"
|
18
18
|
- !ruby/object:Gem::Version
|
19
|
-
version: '0
|
19
|
+
version: '1.0'
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
24
|
- - "~>"
|
25
25
|
- !ruby/object:Gem::Version
|
26
|
-
version: '0
|
26
|
+
version: '1.0'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: multi_json
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
@@ -38,6 +38,48 @@ dependencies:
|
|
38
38
|
- - ">="
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: concurrent-ruby
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '1.1'
|
48
|
+
type: :runtime
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '1.1'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
name: ratelimit
|
57
|
+
requirement: !ruby/object:Gem::Requirement
|
58
|
+
requirements:
|
59
|
+
- - "~>"
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '1.0'
|
62
|
+
type: :runtime
|
63
|
+
prerelease: false
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - "~>"
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '1.0'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
name: activesupport
|
71
|
+
requirement: !ruby/object:Gem::Requirement
|
72
|
+
requirements:
|
73
|
+
- - ">="
|
74
|
+
- !ruby/object:Gem::Version
|
75
|
+
version: '5.0'
|
76
|
+
type: :runtime
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - ">="
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '5.0'
|
41
83
|
- !ruby/object:Gem::Dependency
|
42
84
|
name: activejob
|
43
85
|
requirement: !ruby/object:Gem::Requirement
|
@@ -56,30 +98,30 @@ dependencies:
|
|
56
98
|
name: bundler
|
57
99
|
requirement: !ruby/object:Gem::Requirement
|
58
100
|
requirements:
|
59
|
-
- - "
|
101
|
+
- - ">="
|
60
102
|
- !ruby/object:Gem::Version
|
61
103
|
version: '1.14'
|
62
104
|
type: :development
|
63
105
|
prerelease: false
|
64
106
|
version_requirements: !ruby/object:Gem::Requirement
|
65
107
|
requirements:
|
66
|
-
- - "
|
108
|
+
- - ">="
|
67
109
|
- !ruby/object:Gem::Version
|
68
110
|
version: '1.14'
|
69
111
|
- !ruby/object:Gem::Dependency
|
70
112
|
name: rake
|
71
113
|
requirement: !ruby/object:Gem::Requirement
|
72
114
|
requirements:
|
73
|
-
- - "
|
115
|
+
- - ">="
|
74
116
|
- !ruby/object:Gem::Version
|
75
|
-
version:
|
117
|
+
version: 12.3.3
|
76
118
|
type: :development
|
77
119
|
prerelease: false
|
78
120
|
version_requirements: !ruby/object:Gem::Requirement
|
79
121
|
requirements:
|
80
|
-
- - "
|
122
|
+
- - ">="
|
81
123
|
- !ruby/object:Gem::Version
|
82
|
-
version:
|
124
|
+
version: 12.3.3
|
83
125
|
- !ruby/object:Gem::Dependency
|
84
126
|
name: rspec
|
85
127
|
requirement: !ruby/object:Gem::Requirement
|
@@ -130,9 +172,12 @@ files:
|
|
130
172
|
- lib/hutch-schedule.rb
|
131
173
|
- lib/hutch/enqueue.rb
|
132
174
|
- lib/hutch/error_handlers/max_retry.rb
|
175
|
+
- lib/hutch/patch/config.rb
|
176
|
+
- lib/hutch/patch/worker.rb
|
133
177
|
- lib/hutch/schedule.rb
|
134
178
|
- lib/hutch/schedule/core.rb
|
135
179
|
- lib/hutch/schedule/version.rb
|
180
|
+
- lib/hutch/threshold.rb
|
136
181
|
homepage: https://github.com/wppurking/hutch-schedule
|
137
182
|
licenses:
|
138
183
|
- MIT
|
@@ -152,8 +197,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
152
197
|
- !ruby/object:Gem::Version
|
153
198
|
version: '0'
|
154
199
|
requirements: []
|
155
|
-
|
156
|
-
rubygems_version: 2.7.6
|
200
|
+
rubygems_version: 3.1.2
|
157
201
|
signing_key:
|
158
202
|
specification_version: 4
|
159
203
|
summary: Add Schedule and Error Retry To Hutch.
|