hutch-schedule 0.6.1 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: dbccf1993c14b4c6150ba184e2e04fec5f6586a3dbd7958568fdbed982a6ccb9
4
- data.tar.gz: bfbb93b461ef72e5d28812c9afedbd45a2f16525301623f548c9d0e11a6c534e
3
+ metadata.gz: 67a8a0f9f0a464729740d0bb57ae9d1d10d5005927de1e746fc440ea9c15bbc0
4
+ data.tar.gz: 5b66aca0219764dfefc211cf4ee55bdd17dc13d9cb02a61e5cce39fbb03ee470
5
5
  SHA512:
6
- metadata.gz: ad9ace36edbf1023cd899c3d1eaa0ddb2906b20349f0d82bbc8edbf4bf61d0934c099a0226924cb8c1f638b7272bfb95c718f4f1a2bfacc7b05a6abe33c0d9dc
7
- data.tar.gz: ded955ab9014308b57f395d1b0280219474d91baa18e4936b00c04bd632f825277fb3f6a822543ce9f14216360bae84921b75a1b922968485c65ca3254550383
6
+ metadata.gz: 4b2db45314d63e5ed2b2a12810146e1d09aa85194a2a575246d83fc4cd850de06f055dcff4e489bcf1889058fbd1891e0479c53a796efb8e7cfa3eb577dcd5cb
7
+ data.tar.gz: ac1bd4cd5e6469d1d4d80690c29119a5929f947669753f54504a619b69a93f07549147198fcb669f3315703ce6e18646e6a8267eead5415abee9f14c9150fab3
data/CHANGELOG.md CHANGED
@@ -1,6 +1,30 @@
1
1
  # Change Log
2
2
  All notable changes to this project will be documented in this file.
3
3
 
4
+ ## [0.7.0] - 2020-04-13
5
+ ### Fixed
6
+ - Use monkey patch to support Conumser ratelimit through Hutch::Threshold
7
+ - Use monkey patch to expand Hutch::Config add more work pool relate configurations
8
+ - Upgrade Hutch to 1.0
9
+
10
+ ## [0.6.1] - 2018-05-23
11
+ ### Fixed
12
+ - When message requeue merge the message headers properties
13
+
14
+ ## [0.6.0] - 2018-04-17
15
+ ### Fixed
16
+ - Reuse Hutch::Enqueue in HutchAdapter#dynamic_consumer
17
+
18
+ ## [0.5.1] - 2018-04-15
19
+ ### Fixed
20
+ - Fix dynamic_consumer name/inspect/to_s method issue
21
+ - Add delay time from seconds to days
22
+
23
+ ## [0.4.3] - 2017-06-06
24
+ ### Fixed
25
+ - Support #dynamic_consumer for HutchAdapter to adoption ActiveJob
26
+ - Provider correct #name for dynamic_consumer
27
+
4
28
  ## [0.4.2] - 2017-05-23
5
29
  ### Fixed
6
30
  - Reuse consumer when register ActiveJob queue to consumer
data/README.md CHANGED
@@ -84,9 +84,7 @@ Add an `hutch.rb` to `conf/initializers`:
84
84
  Hutch::Config.load_from_file(Rails.root.join('config', 'config.yaml'))
85
85
  # replace error_handlers with Hutch::ErrorHandlers::MaxRetry
86
86
  Hutch::Config.error_handlers = [Hutch::ErrorHandlers::MaxRetry.new]
87
- # Init Hutch
88
- Hutch.connect
89
- # Init Hutch::Schedule
87
+ # Init Hutch and Hutch::Schedule
90
88
  Hutch::Schedule.connect
91
89
  ```
92
90
 
@@ -128,14 +126,6 @@ The gem is available as open source under the terms of the [MIT License](http://
128
126
  ## Performance
129
127
  Use the repo: https://github.com/wppurking/hutch-schedule-demo
130
128
 
131
- #### ActiveJob enqueue
132
- ![ActiveJob enqueue](http://ofooyx8i9.bkt.clouddn.com/enqueue.jpg)
133
-
134
- #### Hutch publish message
135
- ![Hutch publish message](http://ofooyx8i9.bkt.clouddn.com/enqueue_raw.jpg)
136
-
137
- #### ActiveJob dequeue to execute
138
- ![ActiveJob dequeue to execute](http://ofooyx8i9.bkt.clouddn.com/consume.jpg)
139
-
140
- #### Hutch dequeue to execute
141
- ![Hutch dequeue to execute](http://ofooyx8i9.bkt.clouddn.com/consume2.jpg)
129
+ # TODO
130
+ * add cron job support
131
+ * add unique job support
@@ -8,25 +8,28 @@ Gem::Specification.new do |spec|
8
8
  spec.version = Hutch::Schedule::VERSION
9
9
  spec.authors = ["Wyatt pan"]
10
10
  spec.email = ["wppurking@gmail.com"]
11
-
11
+
12
12
  spec.summary = %q{Add Schedule and Error Retry To Hutch.}
13
13
  spec.description = %q{Add Schedule and Error Retry To Hutch.}
14
14
  spec.homepage = "https://github.com/wppurking/hutch-schedule"
15
15
  spec.license = "MIT"
16
-
16
+
17
17
  spec.files = `git ls-files -z`.split("\x0").reject do |f|
18
18
  f.match(%r{^(test|spec|features)/})
19
19
  end
20
20
  spec.bindir = "exe"
21
21
  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
22
22
  spec.require_paths = ["lib"]
23
-
24
- spec.add_runtime_dependency 'hutch', '~> 0.24'
23
+
24
+ spec.add_runtime_dependency 'hutch', '~> 1.0'
25
25
  spec.add_runtime_dependency "multi_json"
26
-
26
+ spec.add_runtime_dependency "concurrent-ruby", "~> 1.1"
27
+ spec.add_runtime_dependency "ratelimit", "~> 1.0"
28
+ spec.add_runtime_dependency "activesupport", ">= 5.0"
29
+
27
30
  spec.add_development_dependency "activejob"
28
- spec.add_development_dependency "bundler", "~> 1.14"
29
- spec.add_development_dependency "rake", "~> 10.0"
31
+ spec.add_development_dependency "bundler", ">= 1.14"
32
+ spec.add_development_dependency "rake", ">= 12.3.3"
30
33
  spec.add_development_dependency "rspec", "~> 3.0"
31
34
  spec.add_development_dependency "timecop", "~> 0.8"
32
35
  end
@@ -10,30 +10,30 @@ module ActiveJob
10
10
  class HutchAdapter
11
11
  # All activejob Message will routing to one RabbitMQ Queue.
12
12
  # Because Hutch will one Consumer per Queue
13
- AJ_ROUTING_KEY = "active_job"
13
+ AJ_ROUTING_KEY = "active_job"
14
14
  @@queue_consumers = {}
15
-
15
+
16
16
  def initialize
17
17
  @monitor = Monitor.new
18
18
  end
19
-
19
+
20
20
  def enqueue(job) #:nodoc:
21
21
  @monitor.synchronize do
22
22
  @@queue_consumers[job.queue_name].enqueue(job.serialize)
23
23
  end
24
24
  end
25
-
25
+
26
26
  def enqueue_at(job, timestamp) #:nodoc:
27
27
  @monitor.synchronize do
28
28
  @@queue_consumers[job.queue_name].enqueue_at(timestamp, job.serialize)
29
29
  end
30
30
  end
31
-
31
+
32
32
  # Get an routing_key
33
33
  def self.routing_key(job)
34
34
  "#{AJ_ROUTING_KEY}.#{job.queue_name}"
35
35
  end
36
-
36
+
37
37
  # Register all ActiveJob Class to Hutch. (per queue per consumer)
38
38
  def self.register_actice_job_classes
39
39
  # TODO: 需要考虑如何将 AJ 的 Proc queue_name 动态注册到 Hutch
@@ -47,32 +47,33 @@ module ActiveJob
47
47
  Hutch.register_consumer(@@queue_consumers[job.queue_name])
48
48
  end
49
49
  end
50
-
50
+
51
51
  private
52
+
52
53
  def self.dynamic_consumer(job_instance)
53
54
  Class.new do
54
55
  # don't include Hutch::Consumer, we should change the name of consumer to registe
55
56
  extend Hutch::Consumer::ClassMethods
56
57
  include Hutch::Enqueue
57
-
58
+
58
59
  attr_accessor :broker, :delivery_info
59
-
60
+
60
61
  queue_name job_instance.queue_name
61
62
  consume HutchAdapter.routing_key(job_instance)
62
-
63
+
63
64
  def process(job_data)
64
65
  ActiveJob::Base.execute(job_data)
65
66
  end
66
-
67
+
67
68
  define_singleton_method :name do
68
69
  "#{job_instance.queue_name}_dynamic_consumer".camelize
69
70
  end
70
-
71
+
71
72
  # inspect name
72
73
  define_singleton_method :inspect do
73
74
  "#{job_instance.queue_name}_dynamic_consumer".camelize
74
75
  end
75
-
76
+
76
77
  define_singleton_method :to_s do
77
78
  "#{job_instance.queue_name}_dynamic_consumer".camelize
78
79
  end
data/lib/hutch/enqueue.rb CHANGED
@@ -1,37 +1,39 @@
1
1
  require 'active_support/concern'
2
2
  require 'active_support/core_ext/numeric/time'
3
3
  require 'hutch/schedule'
4
+ require 'hutch/threshold'
4
5
 
5
6
  module Hutch
6
7
  # If consumer need `enqueue`, just include this module
7
8
  module Enqueue
8
9
  extend ActiveSupport::Concern
9
-
10
+ include Hutch::Threshold
11
+
10
12
  # Add Consumer methods
11
13
  class_methods do
12
14
  # Publish the message to this consumer with one routing_key
13
15
  def enqueue(message)
14
16
  Hutch.publish(enqueue_routing_key, message)
15
17
  end
16
-
18
+
17
19
  # publish message at a delay times
18
20
  # interval: delay interval seconds
19
21
  # message: publish message
20
22
  def enqueue_in(interval, message, props = {})
21
23
  # TODO: 超过 3h 的延迟也会接收, 但是不会延迟那么长时间, 但给予 warn
22
24
  delay_seconds = delay_seconds_level(interval)
23
-
25
+
24
26
  # 设置固定的延迟, 利用 headers 中的 CC, 以及区分的 topic, 将消息重新投递进入队列
25
- prop_headers = props[:headers] || {}
26
- properties = props.merge(
27
+ prop_headers = props[:headers] || {}
28
+ properties = props.merge(
27
29
  expiration: (delay_seconds * 1000).to_i,
28
- headers: prop_headers.merge(CC: [enqueue_routing_key])
30
+ headers: prop_headers.merge(CC: [enqueue_routing_key])
29
31
  )
30
32
  delay_routing_key = Hutch::Schedule.delay_routing_key("#{delay_seconds}s")
31
-
33
+
32
34
  Hutch::Schedule.publish(delay_routing_key, message, properties)
33
35
  end
34
-
36
+
35
37
  # delay at exatly time point
36
38
  def enqueue_at(time, message, props = {})
37
39
  # compatible with with ActiveJob API
@@ -40,22 +42,22 @@ module Hutch
40
42
  interval = [(time_or_timestamp - Time.now.utc.to_f), 1.second].max
41
43
  enqueue_in(interval, message, props)
42
44
  end
43
-
45
+
44
46
  # routing_key: the purpose is to send message to hutch exchange and then routing to the correct queue,
45
47
  # so can use any of them routing_key that the consumer is consuming.
46
48
  def enqueue_routing_key
47
49
  raise "Routing Keys is not set!" if routing_keys.size < 1
48
50
  routing_keys.to_a.last
49
51
  end
50
-
52
+
51
53
  def attempts(times)
52
54
  @max_retries = [times, 0].max
53
55
  end
54
-
56
+
55
57
  def max_attempts
56
58
  @max_retries || 0
57
59
  end
58
-
60
+
59
61
  # 计算 delay 的 level
60
62
  # 5s 10s 20s 30s
61
63
  # 60s 120s 180s 240s 300s 360s 420s 480s 540s 600s 1200s 1800s 2400s
@@ -3,12 +3,12 @@ require 'active_support/core_ext/object/blank'
3
3
 
4
4
  module Hutch
5
5
  module ErrorHandlers
6
-
6
+
7
7
  # When reach the Max Attempts, republish this message to RabbitMQ,
8
8
  # And persisted the properties[:headers] to tell RabbitMQ the `x-dead.count`
9
9
  class MaxRetry
10
10
  include Logging
11
-
11
+
12
12
  # properties.headers example:
13
13
  # {
14
14
  # "x-death": [
@@ -40,7 +40,7 @@ module Hutch
40
40
  logger.warn("Consumer: #{consumer} is not include Hutch::Enqueue can`t use #enqueue_in`")
41
41
  return false
42
42
  end
43
-
43
+
44
44
  prop_headers = properties[:headers] || {}
45
45
  attempts = failure_count(prop_headers, consumer) + 1
46
46
  if attempts <= consumer.max_attempts
@@ -51,13 +51,13 @@ module Hutch
51
51
  logger.debug("failing, retry_count=#{attempts}, ex:#{ex}")
52
52
  end
53
53
  end
54
-
54
+
55
55
  # becareful with the RabbitMQ fixed delay level, this retry_dealy seconds will fit to one fixed delay level.
56
56
  # so the max delay time is limit to 3 hours(10800s, error times 11: 14643)
57
57
  def retry_delay(executes)
58
- (executes**4) + 2
58
+ (executes ** 4) + 2
59
59
  end
60
-
60
+
61
61
  def failure_count(headers, consumer)
62
62
  if headers.nil? || headers['x-death'].nil?
63
63
  0
@@ -66,7 +66,7 @@ module Hutch
66
66
  # http://ruby-doc.org/stdlib-2.2.3/libdoc/set/rdoc/Set.html#method-i-intersect-3F
67
67
  (x_death['routing-keys'].presence || []).to_set.intersect?(consumer.routing_keys)
68
68
  end
69
-
69
+
70
70
  if x_death_array.count > 0 && x_death_array.first['count']
71
71
  # Newer versions of RabbitMQ return headers with a count key
72
72
  x_death_array.inject(0) { |sum, x_death| sum + x_death['count'] }
@@ -0,0 +1,28 @@
1
+ module Hutch
2
+ module Config
3
+ # Hutch Schedule work pool size
4
+ number_setting :worker_pool_size, 20
5
+
6
+ # Hutch Schedule exceeded checker and poller interval seconds
7
+ # Hutch Worker Pool heartbeat interval (share interval config)
8
+ number_setting :poller_interval, 1
9
+
10
+ # Hutch Schedule poller batch size
11
+ number_setting :poller_batch_size, 100
12
+
13
+ # Ratelimit redis url
14
+ string_setting :ratelimit_redis_url, "redis://127.0.0.1:6379/0"
15
+
16
+ # Ratelimit bucket interval
17
+ number_setting :ratelimit_bucket_interval, 1
18
+
19
+ initialize(
20
+ worker_pool_size: 20,
21
+ poller_interval: 1,
22
+ poller_batch_size: 100,
23
+ # @see Redis::Client
24
+ ratelimit_redis_url: "redis://127.0.0.1:6379/0",
25
+ ratelimit_bucket_interval: 1
26
+ )
27
+ end
28
+ end
@@ -0,0 +1,99 @@
1
+ require 'concurrent'
2
+
3
+ module Hutch
4
+ # Monkey patch worker. 因为 hutch 是借用的底层的 bunny 的 ConsumerWorkPool 来完成的并发任务处理,
5
+ # 但这个 Pool 太过于通用, 而我们需要针对 rabbitmq 传入过来的 Message 进行处理, 需要在任务执行的过程中
6
+ # 添加额外的处理信息, 所以我们不由 ConsumerWorkPool 来处理任务, 改为由 ConsumerWorkPool 执行一次任务提交,
7
+ # 由 Bunny::ConsumerWorkPool 将需要执行的 block 提交给自己的 WorkerPool 来进行最终执行.
8
+ # 因为 RabbitMQ 队列中的任务是需要手动 Ack 才会标记完成, 并且根据 Channel 会有 Prefetch, 所以结合这两个特性
9
+ # 则可以利用本地进程中的 Queue 进行缓存任务, 只要没有执行会有 Prefetch 控制当前节点缓存的总任务数, 根据 Ack 会
10
+ # 明确告知 RabbitMQ 此任务完成.
11
+ class Worker
12
+ def initialize(broker, consumers, setup_procs)
13
+ @broker = broker
14
+ self.consumers = consumers
15
+ self.setup_procs = setup_procs
16
+
17
+ @message_worker = Concurrent::FixedThreadPool.new(Hutch::Config.get(:worker_pool_size))
18
+ @timer_worker = Concurrent::TimerTask.execute(execution_interval: Hutch::Config.get(:poller_interval)) do
19
+ heartbeat_connection
20
+ retry_buffer_queue
21
+ end
22
+ @buffer_queue = ::Queue.new
23
+ @batch_size = Hutch::Config.get(:poller_batch_size)
24
+ @connected = Hutch.connected?
25
+ end
26
+
27
+ # 停止两个线程池
28
+ def shutdown
29
+ @message_worker.shutdown
30
+ @timer_worker.shutdown
31
+ end
32
+
33
+ # Bind a consumer's routing keys to its queue, and set up a subscription to
34
+ # receive messages sent to the queue.
35
+ def setup_queue(consumer)
36
+ logger.info "setting up queue: #{consumer.get_queue_name}"
37
+
38
+ queue = @broker.queue(consumer.get_queue_name, consumer.get_arguments)
39
+ @broker.bind_queue(queue, consumer.routing_keys)
40
+
41
+ queue.subscribe(consumer_tag: unique_consumer_tag, manual_ack: true) do |*args|
42
+ delivery_info, properties, payload = Hutch::Adapter.decode_message(*args)
43
+ handle_message_with_limits(consumer, delivery_info, properties, payload)
44
+ end
45
+ end
46
+
47
+ def handle_message_with_limits(consumer, delivery_info, properties, payload)
48
+ # 1. consumer.limit?
49
+ # 2. yes: make and ConsumerMsg to queue
50
+ # 3. no: post handle
51
+ @message_worker.post do
52
+ if consumer.ratelimit_exceeded?
53
+ @buffer_queue.push(ConsumerMsg.new(consumer, delivery_info, properties, payload))
54
+ else
55
+ # if Hutch disconnect skip do work let message timeout in rabbitmq waiting message push again
56
+ return unless @connected
57
+ consumer.ratelimit_add
58
+ handle_message(consumer, delivery_info, properties, payload)
59
+ end
60
+ end
61
+ end
62
+
63
+ # 心跳检查 Hutch 的连接
64
+ def heartbeat_connection
65
+ @connected = Hutch.connected?
66
+ end
67
+
68
+ # 每隔一段时间, 从 buffer queue 中转移任务到执行
69
+ def retry_buffer_queue
70
+ @batch_size.times do
71
+ cmsg = peak
72
+ return if cmsg.blank?
73
+ handle_message_with_limits(cmsg.consumer, cmsg.delivery_info, cmsg.properties, cmsg.payload)
74
+ end
75
+ end
76
+
77
+ # non-blocking pop message, if empty return nil. other error raise exception
78
+ def peak
79
+ @buffer_queue.pop(true)
80
+ rescue ThreadError => e
81
+ nil if e.to_s == "queue empty"
82
+ end
83
+ end
84
+
85
+ # Consumer Message wrap rabbitmq message infomation
86
+ class ConsumerMsg
87
+ attr_reader :consumer, :delivery_info, :properties, :payload
88
+
89
+ def initialize(consumer, delivery_info, properties, payload)
90
+ @consumer = consumer
91
+ @delivery_info = delivery_info
92
+ @properties = properties
93
+ @payload = payload
94
+ end
95
+ end
96
+ end
97
+
98
+
99
+
@@ -1,9 +1,12 @@
1
1
  require 'active_support/core_ext/module/delegation'
2
2
  require 'active_support/core_ext/object/blank'
3
3
  require 'hutch'
4
+ require 'hutch/patch/config'
5
+ require 'hutch/patch/worker'
6
+ require 'hutch/schedule/core'
4
7
  require 'hutch/enqueue'
8
+ require 'hutch/threshold'
5
9
  require 'hutch/error_handlers/max_retry'
6
- require 'hutch/schedule/core'
7
10
 
8
11
  # If ActiveJob is requried then required the adapter
9
12
  if defined?(ActiveJob)
@@ -15,13 +18,13 @@ module Hutch
15
18
  # If you want use it, just do `Hutch::Schedule.connect(Hutch.broker)` to initialize it
16
19
  # and then just use like Hutch to publish message `Hutch::Schedule.publish`
17
20
  module Schedule
18
-
21
+
19
22
  # fixed delay levels
20
23
  # seconds(4): 5s, 10s, 20s, 30s
21
24
  # minutes(14): 1m, 2m, 3m, 4m, 5m, 6m, 7m, 8m, 9m, 10m, 20m, 30m, 40m, 50m
22
25
  # hours(3): 1h, 2h, 3h
23
26
  DELAY_QUEUES = %w(5s 10s 20s 30s 60s 120s 180s 240s 300s 360s 420s 480s 540s 600s 1200s 1800s 2400s 3000s 3600s 7200s 10800s)
24
-
27
+
25
28
  class << self
26
29
  def connect
27
30
  ActiveJob::QueueAdapters::HutchAdapter.register_actice_job_classes if defined?(ActiveJob::QueueAdapters::HutchAdapter)
@@ -31,20 +34,26 @@ module Hutch
31
34
  @core = Hutch::Schedule::Core.new(Hutch.broker)
32
35
  @core.connect!
33
36
  end
34
-
37
+
38
+ def disconnect
39
+ Hutch.disconnect if Hutch.connected?
40
+ @core = nil
41
+ end
42
+
43
+
35
44
  def core
36
45
  @core
37
46
  end
38
-
47
+
39
48
  def publish(*args)
40
49
  core.publish(*args)
41
50
  end
42
-
51
+
43
52
  # fixed delay level queue's routing_key
44
53
  def delay_routing_key(suffix)
45
54
  "#{Hutch::Config.get(:mq_exchange)}.schedule.#{suffix}"
46
55
  end
47
-
56
+
48
57
  def delay_queue_name(suffix)
49
58
  "#{Hutch::Config.get(:mq_exchange)}_delay_queue_#{suffix}"
50
59
  end
@@ -8,52 +8,52 @@ module Hutch
8
8
  class Core
9
9
  attr_reader :broker, :exchange
10
10
  delegate :channel, :connection, :logger, to: :broker
11
-
11
+
12
12
  def initialize(broker)
13
13
  raise "Broker can`t be nil" if broker.blank?
14
14
  @broker = broker
15
15
  end
16
-
16
+
17
17
  # Becareful with the sequence of initialize
18
18
  def connect!
19
19
  declare_delay_exchange!
20
20
  declare_publisher!
21
21
  setup_delay_queues!
22
22
  end
23
-
23
+
24
24
  def declare_publisher!
25
25
  @publisher = Hutch::Publisher.new(connection, channel, exchange)
26
26
  end
27
-
27
+
28
28
  # The exchange used by Hutch::Schedule
29
29
  def declare_delay_exchange!
30
30
  @exchange = declare_delay_exchange
31
31
  end
32
-
32
+
33
33
  def declare_delay_exchange(ch = channel)
34
- exchange_name = "#{Hutch::Config.get(:mq_exchange)}.schedule"
34
+ exchange_name = "#{Hutch::Config.get(:mq_exchange)}.schedule"
35
35
  exchange_options = { durable: true }.merge(Hutch::Config.get(:mq_exchange_options))
36
36
  logger.info "using topic exchange(schedule) '#{exchange_name}'"
37
-
37
+
38
38
  broker.send(:with_bunny_precondition_handler, 'schedule exchange') do
39
39
  ch.topic(exchange_name, exchange_options)
40
40
  end
41
41
  end
42
-
42
+
43
43
  # The queue used by Hutch::Schedule
44
44
  def setup_delay_queues!
45
45
  DELAY_QUEUES.map { |suffix| setup_delay_queue!(suffix) }
46
46
  end
47
-
47
+
48
48
  def setup_delay_queue!(suffix)
49
49
  # TODO: extract the ttl to config params
50
50
  props = { :'x-message-ttl' => 30.days.in_milliseconds, :'x-dead-letter-exchange' => Hutch::Config.get(:mq_exchange) }
51
51
  queue = broker.queue(Hutch::Schedule.delay_queue_name(suffix), props)
52
-
52
+
53
53
  # bind routing_key to schedule exchange
54
54
  queue.bind(exchange, routing_key: Hutch::Schedule.delay_routing_key(suffix))
55
55
  end
56
-
56
+
57
57
  # Schedule`s publisher, publish the message to schedule topic exchange
58
58
  def publish(*args)
59
59
  @publisher.publish(*args)
@@ -1,5 +1,5 @@
1
1
  module Hutch
2
2
  module Schedule
3
- VERSION = "0.6.1"
3
+ VERSION = "0.7.0"
4
4
  end
5
5
  end
@@ -0,0 +1,77 @@
1
+ require 'active_support/concern'
2
+ require 'ratelimit'
3
+
4
+ module Hutch
5
+ # If consumer need `threshold`, just include this module. (included with Hutch::Enqueue)
6
+ module Threshold
7
+ extend ActiveSupport::Concern
8
+
9
+ # Add Consumer methods
10
+ class_methods do
11
+ # 限流速度:
12
+ # context: 可选的上下文, 默认为 default
13
+ # rate: 数量
14
+ # interval: 间隔
15
+ #
16
+ # block: 采用 lambada 的方式计算, 但需要返回三个参数:
17
+ # - context: 当前约束的上下文
18
+ # - rate: 数量
19
+ # - interval: 间隔
20
+ # 例如:
21
+ # - rate: 1, interval: 1, 每秒 1 个
22
+ # - rate: 30, interval: 1, 每秒 30 个
23
+ # - rate: 30, interval: 5, 每 5 s, 30 个
24
+ def threshold(args)
25
+ @block_given = args.is_a?(Proc)
26
+ if @block_given
27
+ @threshold_block = args
28
+ else
29
+ raise "need args or block" if args.blank?
30
+ raise "args need hash type" if args.class != Hash
31
+ args.symbolize_keys!
32
+ @context = args[:context].presence || "default"
33
+ @rate = args[:rate]
34
+ @interval = args[:interval]
35
+ end
36
+ # call redis.ping let fail fast if redis is not avalible
37
+ _redis.ping
38
+ # redis: 传入设置的 redis
39
+ # bucket_interval: 记录的间隔, 越小精度越大
40
+ @rate_limiter = Ratelimit.new(self.name,
41
+ bucket_interval: Hutch::Config.get(:ratelimit_bucket_interval),
42
+ redis: _redis)
43
+ end
44
+
45
+ # is class level @rate_limiter _context exceeded?
46
+ # if class level @rate_limiter is nil alwayt return false
47
+ def ratelimit_exceeded?
48
+ return false if @rate_limiter.blank?
49
+ @rate_limiter.exceeded?(_context, threshold: _rate, interval: _interval)
50
+ end
51
+
52
+ # 增加一次调用
53
+ def ratelimit_add
54
+ return if @rate_limiter.blank?
55
+ @rate_limiter.add(_context)
56
+ end
57
+
58
+ def _context
59
+ @block_given ? @threshold_block.call[:context] : @context
60
+ end
61
+
62
+ def _rate
63
+ @block_given ? @threshold_block.call[:rate] : @rate
64
+ end
65
+
66
+ def _interval
67
+ @block_given ? @threshold_block.call[:interval] : @interval
68
+ end
69
+
70
+ # all Consumers that use threshold module shared the same redis instance
71
+ def _redis
72
+ @@redis ||= Redis.new(url: Hutch::Config.get(:ratelimit_redis_url))
73
+ end
74
+ end
75
+ end
76
+ end
77
+
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: hutch-schedule
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.6.1
4
+ version: 0.7.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Wyatt pan
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2018-05-23 00:00:00.000000000 Z
11
+ date: 2020-04-12 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: hutch
@@ -16,14 +16,14 @@ dependencies:
16
16
  requirements:
17
17
  - - "~>"
18
18
  - !ruby/object:Gem::Version
19
- version: '0.24'
19
+ version: '1.0'
20
20
  type: :runtime
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
24
  - - "~>"
25
25
  - !ruby/object:Gem::Version
26
- version: '0.24'
26
+ version: '1.0'
27
27
  - !ruby/object:Gem::Dependency
28
28
  name: multi_json
29
29
  requirement: !ruby/object:Gem::Requirement
@@ -38,6 +38,48 @@ dependencies:
38
38
  - - ">="
39
39
  - !ruby/object:Gem::Version
40
40
  version: '0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: concurrent-ruby
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: '1.1'
48
+ type: :runtime
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: '1.1'
55
+ - !ruby/object:Gem::Dependency
56
+ name: ratelimit
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - "~>"
60
+ - !ruby/object:Gem::Version
61
+ version: '1.0'
62
+ type: :runtime
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - "~>"
67
+ - !ruby/object:Gem::Version
68
+ version: '1.0'
69
+ - !ruby/object:Gem::Dependency
70
+ name: activesupport
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - ">="
74
+ - !ruby/object:Gem::Version
75
+ version: '5.0'
76
+ type: :runtime
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - ">="
81
+ - !ruby/object:Gem::Version
82
+ version: '5.0'
41
83
  - !ruby/object:Gem::Dependency
42
84
  name: activejob
43
85
  requirement: !ruby/object:Gem::Requirement
@@ -56,30 +98,30 @@ dependencies:
56
98
  name: bundler
57
99
  requirement: !ruby/object:Gem::Requirement
58
100
  requirements:
59
- - - "~>"
101
+ - - ">="
60
102
  - !ruby/object:Gem::Version
61
103
  version: '1.14'
62
104
  type: :development
63
105
  prerelease: false
64
106
  version_requirements: !ruby/object:Gem::Requirement
65
107
  requirements:
66
- - - "~>"
108
+ - - ">="
67
109
  - !ruby/object:Gem::Version
68
110
  version: '1.14'
69
111
  - !ruby/object:Gem::Dependency
70
112
  name: rake
71
113
  requirement: !ruby/object:Gem::Requirement
72
114
  requirements:
73
- - - "~>"
115
+ - - ">="
74
116
  - !ruby/object:Gem::Version
75
- version: '10.0'
117
+ version: 12.3.3
76
118
  type: :development
77
119
  prerelease: false
78
120
  version_requirements: !ruby/object:Gem::Requirement
79
121
  requirements:
80
- - - "~>"
122
+ - - ">="
81
123
  - !ruby/object:Gem::Version
82
- version: '10.0'
124
+ version: 12.3.3
83
125
  - !ruby/object:Gem::Dependency
84
126
  name: rspec
85
127
  requirement: !ruby/object:Gem::Requirement
@@ -130,9 +172,12 @@ files:
130
172
  - lib/hutch-schedule.rb
131
173
  - lib/hutch/enqueue.rb
132
174
  - lib/hutch/error_handlers/max_retry.rb
175
+ - lib/hutch/patch/config.rb
176
+ - lib/hutch/patch/worker.rb
133
177
  - lib/hutch/schedule.rb
134
178
  - lib/hutch/schedule/core.rb
135
179
  - lib/hutch/schedule/version.rb
180
+ - lib/hutch/threshold.rb
136
181
  homepage: https://github.com/wppurking/hutch-schedule
137
182
  licenses:
138
183
  - MIT
@@ -152,8 +197,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
152
197
  - !ruby/object:Gem::Version
153
198
  version: '0'
154
199
  requirements: []
155
- rubyforge_project:
156
- rubygems_version: 2.7.6
200
+ rubygems_version: 3.1.2
157
201
  signing_key:
158
202
  specification_version: 4
159
203
  summary: Add Schedule and Error Retry To Hutch.