hutch-schedule 0.1.0 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +7 -6
- data/lib/active_job/queue_adapters/hutch_adapter.rb +23 -18
- data/lib/hutch/enqueue.rb +8 -7
- data/lib/hutch/error_handlers/max_retry.rb +44 -0
- data/lib/hutch/schedule/core.rb +9 -12
- data/lib/hutch/schedule/version.rb +1 -1
- data/lib/hutch/schedule.rb +6 -4
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: fb70893978620f96f7701d003b06812d0b1de18c
|
4
|
+
data.tar.gz: 739355b288a4be2a3e54cc116a3193ec7edddff1
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: dda680d37cf8846741fa624db5e0b3d656f2dc3e7a2f1d14a347b04185f648bf81dfaf7e57b37df6a22507815c547a837efb1fabaf5e31600d7deb03652844f0
|
7
|
+
data.tar.gz: 32afd0caa0ff7b1510c4158344b1e164576eb4971e4778b5f3568410dd7d04fdc1bfaad6822229ca43883e5ee59c11a4b6140dcd800abe27e3701dfa471fee67
|
data/README.md
CHANGED
@@ -20,7 +20,7 @@ Or install it yourself as:
|
|
20
20
|
|
21
21
|
## Usage
|
22
22
|
|
23
|
-
|
23
|
+
Use the code below to initialize the Hutch::Schedule
|
24
24
|
|
25
25
|
```ruby
|
26
26
|
Hutch::Config.setup_procs << -> {
|
@@ -28,11 +28,12 @@ Hutch::Config.setup_procs << -> {
|
|
28
28
|
}
|
29
29
|
```
|
30
30
|
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
31
|
+
They will do something below:
|
32
|
+
|
33
|
+
1. Declear an topic exchange called `<hutch>.schedule` just for routing message to schedule_queue.
|
34
|
+
2. Declear an queue named `<hutch>_schedule_queue` and with some params:
|
35
|
+
- Set `x-dead-letter-exchange: <hutch>`: let queue republish message to default <hutch> exchange.
|
36
|
+
- Set `x-message-ttl: <30.days>`: to avoid the queue is to large, because there is no consumer with this queue.
|
36
37
|
|
37
38
|
## Development
|
38
39
|
|
@@ -4,54 +4,59 @@ module ActiveJob
|
|
4
4
|
module QueueAdapters
|
5
5
|
# == Hutch adapter for Active Job
|
6
6
|
#
|
7
|
-
# 简单高效的消息服务方案, Hutch 以多线程的方式处理 RabbitMQ 中的不同 Queue 的 Message.
|
8
|
-
#
|
9
7
|
# Read more about Hutch {here}[https://github.com/gocardless/hutch].
|
10
8
|
#
|
11
9
|
# Rails.application.config.active_job.queue_adapter = :hutch
|
12
10
|
class HutchAdapter
|
13
|
-
#
|
11
|
+
# All activejob Message will routing to one RabbitMQ Queue.
|
12
|
+
# Because Hutch will one Consumer per Queue
|
14
13
|
AJ_ROUTING_KEY = "active_job"
|
15
14
|
|
16
15
|
def initialize
|
17
16
|
@monitor = Monitor.new
|
18
17
|
end
|
19
18
|
|
20
|
-
# 不适用 Consumer 的 enqueue, 无需每次重复计算
|
21
19
|
def enqueue(job) #:nodoc:
|
22
20
|
@monitor.synchronize do
|
23
|
-
#
|
24
|
-
Hutch.publish(routing_key(job), job.serialize)
|
21
|
+
# publish all job data to hutch
|
22
|
+
Hutch.publish(HutchAdapter.routing_key(job), job.serialize)
|
25
23
|
end
|
26
24
|
end
|
27
25
|
|
28
26
|
def enqueue_at(job, timestamp) #:nodoc:
|
29
27
|
interval = [(timestamp - Time.now.utc.to_i), 1.second].max
|
30
|
-
enqueue_in(interval, job.serialize, routing_key(job))
|
28
|
+
enqueue_in(interval, job.serialize, HutchAdapter.routing_key(job))
|
31
29
|
end
|
32
30
|
|
33
|
-
# 不适用 Consumer 的 enqueue, 无需每次重复计算
|
34
31
|
def enqueue_in(interval, message, routing_key)
|
35
32
|
@monitor.synchronize do
|
36
|
-
#
|
33
|
+
# must be integer
|
37
34
|
props = { expiration: interval.in_milliseconds.to_i }
|
38
35
|
Hutch::Schedule.publish(routing_key, message, props)
|
39
36
|
end
|
40
37
|
end
|
41
38
|
|
42
|
-
#
|
43
|
-
def routing_key(job)
|
39
|
+
# Get an routing_key
|
40
|
+
def self.routing_key(job)
|
44
41
|
"#{AJ_ROUTING_KEY}.#{job.queue_name}"
|
45
42
|
end
|
46
43
|
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
44
|
+
# Register all ActiveJob Class to Hutch. (per queue per consumer)
|
45
|
+
def self.register_actice_job_classes
|
46
|
+
Dir.glob(Rails.root.join('app/jobs/**/*.rb')).each { |x| require_dependency x }
|
47
|
+
ActiveJob::Base.descendants.each do |job|
|
48
|
+
Hutch.consumers << Class.new do
|
49
|
+
extend Hutch::Consumer::ClassMethods
|
50
|
+
|
51
|
+
attr_accessor :broker, :delivery_info
|
52
|
+
|
53
|
+
queue_name job.queue_name
|
54
|
+
consume HutchAdapter.routing_key(job)
|
51
55
|
|
52
|
-
|
53
|
-
|
54
|
-
|
56
|
+
define_method :process do |job_data|
|
57
|
+
ActiveJob::Base.execute(job_data)
|
58
|
+
end
|
59
|
+
end
|
55
60
|
end
|
56
61
|
end
|
57
62
|
end
|
data/lib/hutch/enqueue.rb
CHANGED
@@ -3,33 +3,34 @@ require 'active_support/core_ext/numeric/time'
|
|
3
3
|
require 'hutch/schedule'
|
4
4
|
|
5
5
|
module Hutch
|
6
|
-
#
|
6
|
+
# If consumer need `enqueue`, just include this module
|
7
7
|
module Enqueue
|
8
8
|
extend ActiveSupport::Concern
|
9
9
|
|
10
10
|
# Add Consumer methods
|
11
11
|
class_methods do
|
12
|
-
#
|
12
|
+
# Publish the message to this consumer with one routing_key
|
13
13
|
def enqueue(message)
|
14
14
|
Hutch.publish(enqueue_routing_key, message)
|
15
15
|
end
|
16
16
|
|
17
17
|
# publish message at a delay times
|
18
|
-
# interval:
|
19
|
-
# message:
|
18
|
+
# interval: delay interval
|
19
|
+
# message: publish message
|
20
20
|
def enqueue_in(interval, message)
|
21
21
|
props = { expiration: interval.in_milliseconds.to_i }
|
22
22
|
Hutch::Schedule.publish(enqueue_routing_key, message, props)
|
23
23
|
end
|
24
24
|
|
25
|
-
#
|
25
|
+
# delay at exatly time point
|
26
26
|
def enqueue_at(time, message)
|
27
|
-
#
|
27
|
+
# if time is early then now then just delay 1 second
|
28
28
|
interval = [(time.utc - Time.now.utc), 1.second].max
|
29
29
|
enqueue_in(interval, message)
|
30
30
|
end
|
31
31
|
|
32
|
-
# routing_key:
|
32
|
+
# routing_key: the purpose is to send message to hutch exchange and then routing to the correct queue,
|
33
|
+
# so can use any of them routing_key that the consumer is consuming.
|
33
34
|
def enqueue_routing_key
|
34
35
|
raise "Routing Keys is not set!" if routing_keys.size < 1
|
35
36
|
routing_keys.to_a.last
|
@@ -0,0 +1,44 @@
|
|
1
|
+
require 'hutch/logging'
|
2
|
+
|
3
|
+
module Hutch
|
4
|
+
module ErrorHandlers
|
5
|
+
class MaxRetry
|
6
|
+
include Logging
|
7
|
+
|
8
|
+
# TODO: Need to be implement.
|
9
|
+
# 1. 获取 hutch 本身记录的 x-death 中的错误次数
|
10
|
+
# 2. 从每一个 consumer 身上寻找 max_retry 的次数, 不超过则进行延迟重试
|
11
|
+
# 3. 根据错误次数计算类似 active_job 的 exponentially_longer 延迟时间
|
12
|
+
#
|
13
|
+
# properties.headers example:
|
14
|
+
# {
|
15
|
+
# "x-death": [
|
16
|
+
# {
|
17
|
+
# "count": 7,
|
18
|
+
# "exchange": "hutch.topic",
|
19
|
+
# "queue": "retry_queue",
|
20
|
+
# "reason": "expired",
|
21
|
+
# "routing-keys": [
|
22
|
+
# "plan"
|
23
|
+
# ],
|
24
|
+
# "time": "2017-05-13 23:37:15 +0800"
|
25
|
+
# },
|
26
|
+
# {
|
27
|
+
# "count": 7,
|
28
|
+
# "exchange": "hutch",
|
29
|
+
# "original-expiration": "3000",
|
30
|
+
# "queue": "plan_consumer",
|
31
|
+
# "reason": "rejected",
|
32
|
+
# "routing-keys": [
|
33
|
+
# "plan"
|
34
|
+
# ],
|
35
|
+
# "time": "2017-05-13 23:37:05 +0800"
|
36
|
+
# }
|
37
|
+
# ]
|
38
|
+
# }
|
39
|
+
def handle(properties, payload, consumer, ex)
|
40
|
+
raise "Not implement"
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
data/lib/hutch/schedule/core.rb
CHANGED
@@ -3,7 +3,6 @@ require 'active_support/core_ext/object/blank'
|
|
3
3
|
require 'active_support/core_ext/numeric'
|
4
4
|
require 'active_support/core_ext/module/delegation'
|
5
5
|
|
6
|
-
# 共享 Hutch::Broker 实例的所有东西
|
7
6
|
module Hutch
|
8
7
|
module Schedule
|
9
8
|
class Core
|
@@ -12,18 +11,17 @@ module Hutch
|
|
12
11
|
|
13
12
|
delegate :channel, :connection, :logger, to: :broker
|
14
13
|
|
15
|
-
# 初始化 schedule
|
16
14
|
def initialize(broker)
|
17
15
|
raise "Broker can`t be nil" if broker.blank?
|
18
16
|
@broker = broker
|
19
17
|
end
|
20
18
|
|
21
|
-
#
|
19
|
+
# Use the config with Hutch::Broker instance
|
22
20
|
def config
|
23
21
|
broker.instance_variable_get(:@config)
|
24
22
|
end
|
25
23
|
|
26
|
-
#
|
24
|
+
# Becareful with the sequence of initialize
|
27
25
|
def connect!
|
28
26
|
declare_exchange!
|
29
27
|
declare_publisher!
|
@@ -34,17 +32,15 @@ module Hutch
|
|
34
32
|
@publisher = Hutch::Publisher.new(connection, channel, exchange, config)
|
35
33
|
end
|
36
34
|
|
37
|
-
#
|
35
|
+
# The exchange used by Hutch::Schedule
|
38
36
|
def declare_exchange!
|
39
37
|
@exchange = declare_exchange
|
40
38
|
end
|
41
39
|
|
42
40
|
def declare_exchange(ch = channel)
|
43
41
|
exchange_name = "#{config[:mq_exchange]}.schedule"
|
44
|
-
# TODO:
|
45
|
-
exchange_options = {
|
46
|
-
durable: true,
|
47
|
-
'x-dead-letter-exchange': config[:mq_exchange] }.merge(config[:mq_exchange_options])
|
42
|
+
# TODO: Check mq_exchange_options info, ensure they won`t override x-dead-letter-exchange params
|
43
|
+
exchange_options = { durable: true }.merge(config[:mq_exchange_options])
|
48
44
|
logger.info "using topic exchange(schedule) '#{exchange_name}'"
|
49
45
|
|
50
46
|
broker.send(:with_bunny_precondition_handler, 'schedule exchange') do
|
@@ -52,18 +48,19 @@ module Hutch
|
|
52
48
|
end
|
53
49
|
end
|
54
50
|
|
55
|
-
#
|
51
|
+
# The queue used by Hutch::Schedule
|
56
52
|
def setup_queue!
|
53
|
+
# TODO: extract the ttl to config params
|
57
54
|
props = { 'x-message-ttl': 30.days.in_milliseconds, 'x-dead-letter-exchange': config[:mq_exchange] }
|
58
55
|
queue = broker.queue("#{config[:mq_exchange]}_schedule_queue", props)
|
59
56
|
|
60
|
-
# TODO:
|
57
|
+
# TODO: consider extract this routing_key params to config params
|
61
58
|
# routing all to this queue
|
62
59
|
queue.unbind(exchange, routing_key: '#')
|
63
60
|
queue.bind(exchange, routing_key: '#')
|
64
61
|
end
|
65
62
|
|
66
|
-
# Schedule
|
63
|
+
# Schedule`s publisher, publish the message to schedule topic exchange
|
67
64
|
def publish(*args)
|
68
65
|
@publisher.publish(*args)
|
69
66
|
end
|
data/lib/hutch/schedule.rb
CHANGED
@@ -4,20 +4,22 @@ require 'hutch'
|
|
4
4
|
require 'hutch/enqueue'
|
5
5
|
require 'hutch/schedule/core'
|
6
6
|
|
7
|
-
#
|
7
|
+
# If ActiveJob is requried then required the adapter
|
8
8
|
if defined?(ActiveJob)
|
9
9
|
require 'active_job/queue_adapters/hutch_adapter'
|
10
10
|
end
|
11
11
|
|
12
|
-
# gem 的核心入口文件
|
13
12
|
module Hutch
|
14
|
-
#
|
13
|
+
# Hutch::Schedule, just an addon to deal with the schedule exchange.
|
14
|
+
# If you want use it, just do `Hutch::Schedule.connect(Hutch.broker)` to initialize it
|
15
|
+
# and then just use like Hutch to publish message `Hutch::Schedule.publish`
|
15
16
|
module Schedule
|
16
17
|
|
17
|
-
def self.connect(broker)
|
18
|
+
def self.connect(broker = Hutch.broker)
|
18
19
|
return if core.present?
|
19
20
|
@core = Hutch::Schedule::Core.new(broker)
|
20
21
|
@core.connect!
|
22
|
+
ActiveJob::QueueAdapters::HutchAdapter.register_actice_job_classes if defined?(ActiveJob::QueueAdapters::HutchAdapter)
|
21
23
|
end
|
22
24
|
|
23
25
|
def self.core
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: hutch-schedule
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- wyatt pan
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-05-
|
11
|
+
date: 2017-05-19 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: hutch
|
@@ -114,6 +114,7 @@ files:
|
|
114
114
|
- lib/active_job/queue_adapters/hutch_adapter.rb
|
115
115
|
- lib/hutch-schedule.rb
|
116
116
|
- lib/hutch/enqueue.rb
|
117
|
+
- lib/hutch/error_handlers/max_retry.rb
|
117
118
|
- lib/hutch/schedule.rb
|
118
119
|
- lib/hutch/schedule/core.rb
|
119
120
|
- lib/hutch/schedule/version.rb
|