hutch-schedule 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 90c31b8f2f324f07f2e2bc3c8dcd071ce129e2bf
4
- data.tar.gz: 784b5b1690b8beccbbf76a4cec009c8539f4b15b
3
+ metadata.gz: fb70893978620f96f7701d003b06812d0b1de18c
4
+ data.tar.gz: 739355b288a4be2a3e54cc116a3193ec7edddff1
5
5
  SHA512:
6
- metadata.gz: 8feccd2adb36ca1864132c5be689162a88ab5a1072d758c4661183f29e103336be13d4001167b6dfbc2002db87eccdd2d3d2d6b8ee6fb86ee5038086b3cb4439
7
- data.tar.gz: 03bc493d5ebed1b273e34cc59ab758a91e8fd5a6d88ef9b9b7689e6162442c4f140d9825f114d5dcd9aabebd84123d13f7e701b4488e05a35a7a523c2e84783f
6
+ metadata.gz: dda680d37cf8846741fa624db5e0b3d656f2dc3e7a2f1d14a347b04185f648bf81dfaf7e57b37df6a22507815c547a837efb1fabaf5e31600d7deb03652844f0
7
+ data.tar.gz: 32afd0caa0ff7b1510c4158344b1e164576eb4971e4778b5f3568410dd7d04fdc1bfaad6822229ca43883e5ee59c11a4b6140dcd800abe27e3701dfa471fee67
data/README.md CHANGED
@@ -20,7 +20,7 @@ Or install it yourself as:
20
20
 
21
21
  ## Usage
22
22
 
23
- 为了能够初始化这个项目, 需要在合适的地方设置 Hutch setup_proc 用于初始化 Hutch::Schedule
23
+ Use the code below to initialize the Hutch::Schedule
24
24
 
25
25
  ```ruby
26
26
  Hutch::Config.setup_procs << -> {
@@ -28,11 +28,12 @@ Hutch::Config.setup_procs << -> {
28
28
  }
29
29
  ```
30
30
 
31
- 提供了如下的功能支持:
32
- 1. 在 RabbitMQ 中自动创建了一个 <hutch>.schedule 的 topic exchange 用于专门转发需要延迟的 Message
33
- 2. RabbitMQ 中自动创建了一个 <hutch>_schedule_queue queue 带有:
34
- - dlx <hutch> topic exchange
35
- - 以及避免任务一直挤压的 30 天的 ttl
31
+ They will do something below:
32
+
33
+ 1. Declear an topic exchange called `<hutch>.schedule` just for routing message to schedule_queue.
34
+ 2. Declear an queue named `<hutch>_schedule_queue` and with some params:
35
+ - Set `x-dead-letter-exchange: <hutch>`: let queue republish message to default <hutch> exchange.
36
+ - Set `x-message-ttl: <30.days>`: to avoid the queue is to large, because there is no consumer with this queue.
36
37
 
37
38
  ## Development
38
39
 
@@ -4,54 +4,59 @@ module ActiveJob
4
4
  module QueueAdapters
5
5
  # == Hutch adapter for Active Job
6
6
  #
7
- # 简单高效的消息服务方案, Hutch 以多线程的方式处理 RabbitMQ 中的不同 Queue 的 Message.
8
- #
9
7
  # Read more about Hutch {here}[https://github.com/gocardless/hutch].
10
8
  #
11
9
  # Rails.application.config.active_job.queue_adapter = :hutch
12
10
  class HutchAdapter
13
- # 需要将 activejob 使用的队列全部 routing 到一个 rabbitmq queue 中
11
+ # All activejob Message will routing to one RabbitMQ Queue.
12
+ # Because Hutch will one Consumer per Queue
14
13
  AJ_ROUTING_KEY = "active_job"
15
14
 
16
15
  def initialize
17
16
  @monitor = Monitor.new
18
17
  end
19
18
 
20
- # 不适用 Consumer 的 enqueue, 无需每次重复计算
21
19
  def enqueue(job) #:nodoc:
22
20
  @monitor.synchronize do
23
- # 将整个 job data 变为 hash 发布出去
24
- Hutch.publish(routing_key(job), job.serialize)
21
+ # publish all job data to hutch
22
+ Hutch.publish(HutchAdapter.routing_key(job), job.serialize)
25
23
  end
26
24
  end
27
25
 
28
26
  def enqueue_at(job, timestamp) #:nodoc:
29
27
  interval = [(timestamp - Time.now.utc.to_i), 1.second].max
30
- enqueue_in(interval, job.serialize, routing_key(job))
28
+ enqueue_in(interval, job.serialize, HutchAdapter.routing_key(job))
31
29
  end
32
30
 
33
- # 不适用 Consumer 的 enqueue, 无需每次重复计算
34
31
  def enqueue_in(interval, message, routing_key)
35
32
  @monitor.synchronize do
36
- # 必须是 integer
33
+ # must be integer
37
34
  props = { expiration: interval.in_milliseconds.to_i }
38
35
  Hutch::Schedule.publish(routing_key, message, props)
39
36
  end
40
37
  end
41
38
 
42
- # 计算 routing_key
43
- def routing_key(job)
39
+ # Get an routing_key
40
+ def self.routing_key(job)
44
41
  "#{AJ_ROUTING_KEY}.#{job.queue_name}"
45
42
  end
46
43
 
47
- class JobWrapper #:nodoc:
48
- include Hutch::Consumer
49
- # ActiveJob 使用的一个大队列
50
- consume "#{HutchAdapter::AJ_ROUTING_KEY}.#"
44
+ # Register all ActiveJob Class to Hutch. (per queue per consumer)
45
+ def self.register_actice_job_classes
46
+ Dir.glob(Rails.root.join('app/jobs/**/*.rb')).each { |x| require_dependency x }
47
+ ActiveJob::Base.descendants.each do |job|
48
+ Hutch.consumers << Class.new do
49
+ extend Hutch::Consumer::ClassMethods
50
+
51
+ attr_accessor :broker, :delivery_info
52
+
53
+ queue_name job.queue_name
54
+ consume HutchAdapter.routing_key(job)
51
55
 
52
- def process(job_data)
53
- # 执行 ActiveJobBase, 将整个 job 的 data 传过去, 让其进行反序列化, 组织成参数等等
54
- Base.execute job_data
56
+ define_method :process do |job_data|
57
+ ActiveJob::Base.execute(job_data)
58
+ end
59
+ end
55
60
  end
56
61
  end
57
62
  end
data/lib/hutch/enqueue.rb CHANGED
@@ -3,33 +3,34 @@ require 'active_support/core_ext/numeric/time'
3
3
  require 'hutch/schedule'
4
4
 
5
5
  module Hutch
6
- # 如果需要增加让 Consumer Enqueue 的动作, 那么则 include 这个 Module
6
+ # If consumer need `enqueue`, just include this module
7
7
  module Enqueue
8
8
  extend ActiveSupport::Concern
9
9
 
10
10
  # Add Consumer methods
11
11
  class_methods do
12
- # 正常的发布 consumer 对应 routing key 的消息
12
+ # Publish the message to this consumer with one routing_key
13
13
  def enqueue(message)
14
14
  Hutch.publish(enqueue_routing_key, message)
15
15
  end
16
16
 
17
17
  # publish message at a delay times
18
- # interval: 推迟的时间
19
- # message: 具体的消息
18
+ # interval: delay interval
19
+ # message: publish message
20
20
  def enqueue_in(interval, message)
21
21
  props = { expiration: interval.in_milliseconds.to_i }
22
22
  Hutch::Schedule.publish(enqueue_routing_key, message, props)
23
23
  end
24
24
 
25
- # 延期在某一个时间点执行
25
+ # delay at exatly time point
26
26
  def enqueue_at(time, message)
27
- # 如果 time 比当前时间还早, 那么就延迟 1s 钟执行
27
+ # if time is early then now then just delay 1 second
28
28
  interval = [(time.utc - Time.now.utc), 1.second].max
29
29
  enqueue_in(interval, message)
30
30
  end
31
31
 
32
- # routing_key: 目的为将 Message 发送给 RabbitMQ 那么使用其监听的任何一个 routing_key 都可以发送
32
+ # routing_key: the purpose is to send message to hutch exchange and then routing to the correct queue,
33
+ # so can use any of them routing_key that the consumer is consuming.
33
34
  def enqueue_routing_key
34
35
  raise "Routing Keys is not set!" if routing_keys.size < 1
35
36
  routing_keys.to_a.last
@@ -0,0 +1,44 @@
1
+ require 'hutch/logging'
2
+
3
+ module Hutch
4
+ module ErrorHandlers
5
+ class MaxRetry
6
+ include Logging
7
+
8
+ # TODO: Need to be implement.
9
+ # 1. 获取 hutch 本身记录的 x-death 中的错误次数
10
+ # 2. 从每一个 consumer 身上寻找 max_retry 的次数, 不超过则进行延迟重试
11
+ # 3. 根据错误次数计算类似 active_job 的 exponentially_longer 延迟时间
12
+ #
13
+ # properties.headers example:
14
+ # {
15
+ # "x-death": [
16
+ # {
17
+ # "count": 7,
18
+ # "exchange": "hutch.topic",
19
+ # "queue": "retry_queue",
20
+ # "reason": "expired",
21
+ # "routing-keys": [
22
+ # "plan"
23
+ # ],
24
+ # "time": "2017-05-13 23:37:15 +0800"
25
+ # },
26
+ # {
27
+ # "count": 7,
28
+ # "exchange": "hutch",
29
+ # "original-expiration": "3000",
30
+ # "queue": "plan_consumer",
31
+ # "reason": "rejected",
32
+ # "routing-keys": [
33
+ # "plan"
34
+ # ],
35
+ # "time": "2017-05-13 23:37:05 +0800"
36
+ # }
37
+ # ]
38
+ # }
39
+ def handle(properties, payload, consumer, ex)
40
+ raise "Not implement"
41
+ end
42
+ end
43
+ end
44
+ end
@@ -3,7 +3,6 @@ require 'active_support/core_ext/object/blank'
3
3
  require 'active_support/core_ext/numeric'
4
4
  require 'active_support/core_ext/module/delegation'
5
5
 
6
- # 共享 Hutch::Broker 实例的所有东西
7
6
  module Hutch
8
7
  module Schedule
9
8
  class Core
@@ -12,18 +11,17 @@ module Hutch
12
11
 
13
12
  delegate :channel, :connection, :logger, to: :broker
14
13
 
15
- # 初始化 schedule
16
14
  def initialize(broker)
17
15
  raise "Broker can`t be nil" if broker.blank?
18
16
  @broker = broker
19
17
  end
20
18
 
21
- # 获取 Hutch 上的 Config
19
+ # Use the config with Hutch::Broker instance
22
20
  def config
23
21
  broker.instance_variable_get(:@config)
24
22
  end
25
23
 
26
- # Core 的连接, 注意连接是有顺序的, 必须先将 exchange 初始化好
24
+ # Becareful with the sequence of initialize
27
25
  def connect!
28
26
  declare_exchange!
29
27
  declare_publisher!
@@ -34,17 +32,15 @@ module Hutch
34
32
  @publisher = Hutch::Publisher.new(connection, channel, exchange, config)
35
33
  end
36
34
 
37
- # 申明 schedule 使用的 ex
35
+ # The exchange used by Hutch::Schedule
38
36
  def declare_exchange!
39
37
  @exchange = declare_exchange
40
38
  end
41
39
 
42
40
  def declare_exchange(ch = channel)
43
41
  exchange_name = "#{config[:mq_exchange]}.schedule"
44
- # TODO: 检查 mq_exchange_options 中的信息, 确保不会覆盖 x-dead-letter-exchange 的参数
45
- exchange_options = {
46
- durable: true,
47
- 'x-dead-letter-exchange': config[:mq_exchange] }.merge(config[:mq_exchange_options])
42
+ # TODO: Check mq_exchange_options info, ensure they won`t override x-dead-letter-exchange params
43
+ exchange_options = { durable: true }.merge(config[:mq_exchange_options])
48
44
  logger.info "using topic exchange(schedule) '#{exchange_name}'"
49
45
 
50
46
  broker.send(:with_bunny_precondition_handler, 'schedule exchange') do
@@ -52,18 +48,19 @@ module Hutch
52
48
  end
53
49
  end
54
50
 
55
- # 申明 schedule 使用的 queue
51
+ # The queue used by Hutch::Schedule
56
52
  def setup_queue!
53
+ # TODO: extract the ttl to config params
57
54
  props = { 'x-message-ttl': 30.days.in_milliseconds, 'x-dead-letter-exchange': config[:mq_exchange] }
58
55
  queue = broker.queue("#{config[:mq_exchange]}_schedule_queue", props)
59
56
 
60
- # TODO: 可以考虑将这个抽取成为参数
57
+ # TODO: consider extract this routing_key params to config params
61
58
  # routing all to this queue
62
59
  queue.unbind(exchange, routing_key: '#')
63
60
  queue.bind(exchange, routing_key: '#')
64
61
  end
65
62
 
66
- # Schedule broker 自己的 publish 方法
63
+ # Schedule`s publisher, publish the message to schedule topic exchange
67
64
  def publish(*args)
68
65
  @publisher.publish(*args)
69
66
  end
@@ -1,5 +1,5 @@
1
1
  module Hutch
2
2
  module Schedule
3
- VERSION = "0.1.0"
3
+ VERSION = "0.2.0"
4
4
  end
5
5
  end
@@ -4,20 +4,22 @@ require 'hutch'
4
4
  require 'hutch/enqueue'
5
5
  require 'hutch/schedule/core'
6
6
 
7
- # 引入了 active_job 则引入 adapter
7
+ # If ActiveJob is requried then required the adapter
8
8
  if defined?(ActiveJob)
9
9
  require 'active_job/queue_adapters/hutch_adapter'
10
10
  end
11
11
 
12
- # gem 的核心入口文件
13
12
  module Hutch
14
- # Hutch 下的独立的 Schedule module, 负责与 schedule 相关的 publish
13
+ # Hutch::Schedule, just an addon to deal with the schedule exchange.
14
+ # If you want use it, just do `Hutch::Schedule.connect(Hutch.broker)` to initialize it
15
+ # and then just use like Hutch to publish message `Hutch::Schedule.publish`
15
16
  module Schedule
16
17
 
17
- def self.connect(broker)
18
+ def self.connect(broker = Hutch.broker)
18
19
  return if core.present?
19
20
  @core = Hutch::Schedule::Core.new(broker)
20
21
  @core.connect!
22
+ ActiveJob::QueueAdapters::HutchAdapter.register_actice_job_classes if defined?(ActiveJob::QueueAdapters::HutchAdapter)
21
23
  end
22
24
 
23
25
  def self.core
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: hutch-schedule
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.0
4
+ version: 0.2.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - wyatt pan
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2017-05-15 00:00:00.000000000 Z
11
+ date: 2017-05-19 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: hutch
@@ -114,6 +114,7 @@ files:
114
114
  - lib/active_job/queue_adapters/hutch_adapter.rb
115
115
  - lib/hutch-schedule.rb
116
116
  - lib/hutch/enqueue.rb
117
+ - lib/hutch/error_handlers/max_retry.rb
117
118
  - lib/hutch/schedule.rb
118
119
  - lib/hutch/schedule/core.rb
119
120
  - lib/hutch/schedule/version.rb