hutch-schedule 0.6.1 → 0.7.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: dbccf1993c14b4c6150ba184e2e04fec5f6586a3dbd7958568fdbed982a6ccb9
4
- data.tar.gz: bfbb93b461ef72e5d28812c9afedbd45a2f16525301623f548c9d0e11a6c534e
3
+ metadata.gz: 67a8a0f9f0a464729740d0bb57ae9d1d10d5005927de1e746fc440ea9c15bbc0
4
+ data.tar.gz: 5b66aca0219764dfefc211cf4ee55bdd17dc13d9cb02a61e5cce39fbb03ee470
5
5
  SHA512:
6
- metadata.gz: ad9ace36edbf1023cd899c3d1eaa0ddb2906b20349f0d82bbc8edbf4bf61d0934c099a0226924cb8c1f638b7272bfb95c718f4f1a2bfacc7b05a6abe33c0d9dc
7
- data.tar.gz: ded955ab9014308b57f395d1b0280219474d91baa18e4936b00c04bd632f825277fb3f6a822543ce9f14216360bae84921b75a1b922968485c65ca3254550383
6
+ metadata.gz: 4b2db45314d63e5ed2b2a12810146e1d09aa85194a2a575246d83fc4cd850de06f055dcff4e489bcf1889058fbd1891e0479c53a796efb8e7cfa3eb577dcd5cb
7
+ data.tar.gz: ac1bd4cd5e6469d1d4d80690c29119a5929f947669753f54504a619b69a93f07549147198fcb669f3315703ce6e18646e6a8267eead5415abee9f14c9150fab3
data/CHANGELOG.md CHANGED
@@ -1,6 +1,30 @@
1
1
  # Change Log
2
2
  All notable changes to this project will be documented in this file.
3
3
 
4
+ ## [0.7.0] - 2020-04-13
5
+ ### Fixed
6
+ - Use monkey patch to support Conumser ratelimit through Hutch::Threshold
7
+ - Use monkey patch to expand Hutch::Config add more work pool relate configurations
8
+ - Upgrade Hutch to 1.0
9
+
10
+ ## [0.6.1] - 2018-05-23
11
+ ### Fixed
12
+ - When message requeue merge the message headers properties
13
+
14
+ ## [0.6.0] - 2018-04-17
15
+ ### Fixed
16
+ - Reuse Hutch::Enqueue in HutchAdapter#dynamic_consumer
17
+
18
+ ## [0.5.1] - 2018-04-15
19
+ ### Fixed
20
+ - Fix dynamic_consumer name/inspect/to_s method issue
21
+ - Add delay time from seconds to days
22
+
23
+ ## [0.4.3] - 2017-06-06
24
+ ### Fixed
25
+ - Support #dynamic_consumer for HutchAdapter to adoption ActiveJob
26
+ - Provider correct #name for dynamic_consumer
27
+
4
28
  ## [0.4.2] - 2017-05-23
5
29
  ### Fixed
6
30
  - Reuse consumer when register ActiveJob queue to consumer
data/README.md CHANGED
@@ -84,9 +84,7 @@ Add an `hutch.rb` to `conf/initializers`:
84
84
  Hutch::Config.load_from_file(Rails.root.join('config', 'config.yaml'))
85
85
  # replace error_handlers with Hutch::ErrorHandlers::MaxRetry
86
86
  Hutch::Config.error_handlers = [Hutch::ErrorHandlers::MaxRetry.new]
87
- # Init Hutch
88
- Hutch.connect
89
- # Init Hutch::Schedule
87
+ # Init Hutch and Hutch::Schedule
90
88
  Hutch::Schedule.connect
91
89
  ```
92
90
 
@@ -128,14 +126,6 @@ The gem is available as open source under the terms of the [MIT License](http://
128
126
  ## Performance
129
127
  Use the repo: https://github.com/wppurking/hutch-schedule-demo
130
128
 
131
- #### ActiveJob enqueue
132
- ![ActiveJob enqueue](http://ofooyx8i9.bkt.clouddn.com/enqueue.jpg)
133
-
134
- #### Hutch publish message
135
- ![Hutch publish message](http://ofooyx8i9.bkt.clouddn.com/enqueue_raw.jpg)
136
-
137
- #### ActiveJob dequeue to execute
138
- ![ActiveJob dequeue to execute](http://ofooyx8i9.bkt.clouddn.com/consume.jpg)
139
-
140
- #### Hutch dequeue to execute
141
- ![Hutch dequeue to execute](http://ofooyx8i9.bkt.clouddn.com/consume2.jpg)
129
+ # TODO
130
+ * add cron job support
131
+ * add unique job support
@@ -8,25 +8,28 @@ Gem::Specification.new do |spec|
8
8
  spec.version = Hutch::Schedule::VERSION
9
9
  spec.authors = ["Wyatt pan"]
10
10
  spec.email = ["wppurking@gmail.com"]
11
-
11
+
12
12
  spec.summary = %q{Add Schedule and Error Retry To Hutch.}
13
13
  spec.description = %q{Add Schedule and Error Retry To Hutch.}
14
14
  spec.homepage = "https://github.com/wppurking/hutch-schedule"
15
15
  spec.license = "MIT"
16
-
16
+
17
17
  spec.files = `git ls-files -z`.split("\x0").reject do |f|
18
18
  f.match(%r{^(test|spec|features)/})
19
19
  end
20
20
  spec.bindir = "exe"
21
21
  spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
22
22
  spec.require_paths = ["lib"]
23
-
24
- spec.add_runtime_dependency 'hutch', '~> 0.24'
23
+
24
+ spec.add_runtime_dependency 'hutch', '~> 1.0'
25
25
  spec.add_runtime_dependency "multi_json"
26
-
26
+ spec.add_runtime_dependency "concurrent-ruby", "~> 1.1"
27
+ spec.add_runtime_dependency "ratelimit", "~> 1.0"
28
+ spec.add_runtime_dependency "activesupport", ">= 5.0"
29
+
27
30
  spec.add_development_dependency "activejob"
28
- spec.add_development_dependency "bundler", "~> 1.14"
29
- spec.add_development_dependency "rake", "~> 10.0"
31
+ spec.add_development_dependency "bundler", ">= 1.14"
32
+ spec.add_development_dependency "rake", ">= 12.3.3"
30
33
  spec.add_development_dependency "rspec", "~> 3.0"
31
34
  spec.add_development_dependency "timecop", "~> 0.8"
32
35
  end
@@ -10,30 +10,30 @@ module ActiveJob
10
10
  class HutchAdapter
11
11
  # All activejob Message will routing to one RabbitMQ Queue.
12
12
  # Because Hutch will one Consumer per Queue
13
- AJ_ROUTING_KEY = "active_job"
13
+ AJ_ROUTING_KEY = "active_job"
14
14
  @@queue_consumers = {}
15
-
15
+
16
16
  def initialize
17
17
  @monitor = Monitor.new
18
18
  end
19
-
19
+
20
20
  def enqueue(job) #:nodoc:
21
21
  @monitor.synchronize do
22
22
  @@queue_consumers[job.queue_name].enqueue(job.serialize)
23
23
  end
24
24
  end
25
-
25
+
26
26
  def enqueue_at(job, timestamp) #:nodoc:
27
27
  @monitor.synchronize do
28
28
  @@queue_consumers[job.queue_name].enqueue_at(timestamp, job.serialize)
29
29
  end
30
30
  end
31
-
31
+
32
32
  # Get an routing_key
33
33
  def self.routing_key(job)
34
34
  "#{AJ_ROUTING_KEY}.#{job.queue_name}"
35
35
  end
36
-
36
+
37
37
  # Register all ActiveJob Class to Hutch. (per queue per consumer)
38
38
  def self.register_actice_job_classes
39
39
  # TODO: 需要考虑如何将 AJ 的 Proc queue_name 动态注册到 Hutch
@@ -47,32 +47,33 @@ module ActiveJob
47
47
  Hutch.register_consumer(@@queue_consumers[job.queue_name])
48
48
  end
49
49
  end
50
-
50
+
51
51
  private
52
+
52
53
  def self.dynamic_consumer(job_instance)
53
54
  Class.new do
54
55
  # don't include Hutch::Consumer, we should change the name of consumer to registe
55
56
  extend Hutch::Consumer::ClassMethods
56
57
  include Hutch::Enqueue
57
-
58
+
58
59
  attr_accessor :broker, :delivery_info
59
-
60
+
60
61
  queue_name job_instance.queue_name
61
62
  consume HutchAdapter.routing_key(job_instance)
62
-
63
+
63
64
  def process(job_data)
64
65
  ActiveJob::Base.execute(job_data)
65
66
  end
66
-
67
+
67
68
  define_singleton_method :name do
68
69
  "#{job_instance.queue_name}_dynamic_consumer".camelize
69
70
  end
70
-
71
+
71
72
  # inspect name
72
73
  define_singleton_method :inspect do
73
74
  "#{job_instance.queue_name}_dynamic_consumer".camelize
74
75
  end
75
-
76
+
76
77
  define_singleton_method :to_s do
77
78
  "#{job_instance.queue_name}_dynamic_consumer".camelize
78
79
  end
data/lib/hutch/enqueue.rb CHANGED
@@ -1,37 +1,39 @@
1
1
  require 'active_support/concern'
2
2
  require 'active_support/core_ext/numeric/time'
3
3
  require 'hutch/schedule'
4
+ require 'hutch/threshold'
4
5
 
5
6
  module Hutch
6
7
  # If consumer need `enqueue`, just include this module
7
8
  module Enqueue
8
9
  extend ActiveSupport::Concern
9
-
10
+ include Hutch::Threshold
11
+
10
12
  # Add Consumer methods
11
13
  class_methods do
12
14
  # Publish the message to this consumer with one routing_key
13
15
  def enqueue(message)
14
16
  Hutch.publish(enqueue_routing_key, message)
15
17
  end
16
-
18
+
17
19
  # publish message at a delay times
18
20
  # interval: delay interval seconds
19
21
  # message: publish message
20
22
  def enqueue_in(interval, message, props = {})
21
23
  # TODO: 超过 3h 的延迟也会接收, 但是不会延迟那么长时间, 但给予 warn
22
24
  delay_seconds = delay_seconds_level(interval)
23
-
25
+
24
26
  # 设置固定的延迟, 利用 headers 中的 CC, 以及区分的 topic, 将消息重新投递进入队列
25
- prop_headers = props[:headers] || {}
26
- properties = props.merge(
27
+ prop_headers = props[:headers] || {}
28
+ properties = props.merge(
27
29
  expiration: (delay_seconds * 1000).to_i,
28
- headers: prop_headers.merge(CC: [enqueue_routing_key])
30
+ headers: prop_headers.merge(CC: [enqueue_routing_key])
29
31
  )
30
32
  delay_routing_key = Hutch::Schedule.delay_routing_key("#{delay_seconds}s")
31
-
33
+
32
34
  Hutch::Schedule.publish(delay_routing_key, message, properties)
33
35
  end
34
-
36
+
35
37
  # delay at exatly time point
36
38
  def enqueue_at(time, message, props = {})
37
39
  # compatible with with ActiveJob API
@@ -40,22 +42,22 @@ module Hutch
40
42
  interval = [(time_or_timestamp - Time.now.utc.to_f), 1.second].max
41
43
  enqueue_in(interval, message, props)
42
44
  end
43
-
45
+
44
46
  # routing_key: the purpose is to send message to hutch exchange and then routing to the correct queue,
45
47
  # so can use any of them routing_key that the consumer is consuming.
46
48
  def enqueue_routing_key
47
49
  raise "Routing Keys is not set!" if routing_keys.size < 1
48
50
  routing_keys.to_a.last
49
51
  end
50
-
52
+
51
53
  def attempts(times)
52
54
  @max_retries = [times, 0].max
53
55
  end
54
-
56
+
55
57
  def max_attempts
56
58
  @max_retries || 0
57
59
  end
58
-
60
+
59
61
  # 计算 delay 的 level
60
62
  # 5s 10s 20s 30s
61
63
  # 60s 120s 180s 240s 300s 360s 420s 480s 540s 600s 1200s 1800s 2400s
@@ -3,12 +3,12 @@ require 'active_support/core_ext/object/blank'
3
3
 
4
4
  module Hutch
5
5
  module ErrorHandlers
6
-
6
+
7
7
  # When reach the Max Attempts, republish this message to RabbitMQ,
8
8
  # And persisted the properties[:headers] to tell RabbitMQ the `x-dead.count`
9
9
  class MaxRetry
10
10
  include Logging
11
-
11
+
12
12
  # properties.headers example:
13
13
  # {
14
14
  # "x-death": [
@@ -40,7 +40,7 @@ module Hutch
40
40
  logger.warn("Consumer: #{consumer} is not include Hutch::Enqueue can`t use #enqueue_in`")
41
41
  return false
42
42
  end
43
-
43
+
44
44
  prop_headers = properties[:headers] || {}
45
45
  attempts = failure_count(prop_headers, consumer) + 1
46
46
  if attempts <= consumer.max_attempts
@@ -51,13 +51,13 @@ module Hutch
51
51
  logger.debug("failing, retry_count=#{attempts}, ex:#{ex}")
52
52
  end
53
53
  end
54
-
54
+
55
55
  # becareful with the RabbitMQ fixed delay level, this retry_dealy seconds will fit to one fixed delay level.
56
56
  # so the max delay time is limit to 3 hours(10800s, error times 11: 14643)
57
57
  def retry_delay(executes)
58
- (executes**4) + 2
58
+ (executes ** 4) + 2
59
59
  end
60
-
60
+
61
61
  def failure_count(headers, consumer)
62
62
  if headers.nil? || headers['x-death'].nil?
63
63
  0
@@ -66,7 +66,7 @@ module Hutch
66
66
  # http://ruby-doc.org/stdlib-2.2.3/libdoc/set/rdoc/Set.html#method-i-intersect-3F
67
67
  (x_death['routing-keys'].presence || []).to_set.intersect?(consumer.routing_keys)
68
68
  end
69
-
69
+
70
70
  if x_death_array.count > 0 && x_death_array.first['count']
71
71
  # Newer versions of RabbitMQ return headers with a count key
72
72
  x_death_array.inject(0) { |sum, x_death| sum + x_death['count'] }
@@ -0,0 +1,28 @@
1
+ module Hutch
2
+ module Config
3
+ # Hutch Schedule work pool size
4
+ number_setting :worker_pool_size, 20
5
+
6
+ # Hutch Schedule exceeded checker and poller interval seconds
7
+ # Hutch Worker Pool heartbeat interval (share interval config)
8
+ number_setting :poller_interval, 1
9
+
10
+ # Hutch Schedule poller batch size
11
+ number_setting :poller_batch_size, 100
12
+
13
+ # Ratelimit redis url
14
+ string_setting :ratelimit_redis_url, "redis://127.0.0.1:6379/0"
15
+
16
+ # Ratelimit bucket interval
17
+ number_setting :ratelimit_bucket_interval, 1
18
+
19
+ initialize(
20
+ worker_pool_size: 20,
21
+ poller_interval: 1,
22
+ poller_batch_size: 100,
23
+ # @see Redis::Client
24
+ ratelimit_redis_url: "redis://127.0.0.1:6379/0",
25
+ ratelimit_bucket_interval: 1
26
+ )
27
+ end
28
+ end
@@ -0,0 +1,99 @@
1
+ require 'concurrent'
2
+
3
+ module Hutch
4
+ # Monkey patch worker. 因为 hutch 是借用的底层的 bunny 的 ConsumerWorkPool 来完成的并发任务处理,
5
+ # 但这个 Pool 太过于通用, 而我们需要针对 rabbitmq 传入过来的 Message 进行处理, 需要在任务执行的过程中
6
+ # 添加额外的处理信息, 所以我们不由 ConsumerWorkPool 来处理任务, 改为由 ConsumerWorkPool 执行一次任务提交,
7
+ # 由 Bunny::ConsumerWorkPool 将需要执行的 block 提交给自己的 WorkerPool 来进行最终执行.
8
+ # 因为 RabbitMQ 队列中的任务是需要手动 Ack 才会标记完成, 并且根据 Channel 会有 Prefetch, 所以结合这两个特性
9
+ # 则可以利用本地进程中的 Queue 进行缓存任务, 只要没有执行会有 Prefetch 控制当前节点缓存的总任务数, 根据 Ack 会
10
+ # 明确告知 RabbitMQ 此任务完成.
11
+ class Worker
12
+ def initialize(broker, consumers, setup_procs)
13
+ @broker = broker
14
+ self.consumers = consumers
15
+ self.setup_procs = setup_procs
16
+
17
+ @message_worker = Concurrent::FixedThreadPool.new(Hutch::Config.get(:worker_pool_size))
18
+ @timer_worker = Concurrent::TimerTask.execute(execution_interval: Hutch::Config.get(:poller_interval)) do
19
+ heartbeat_connection
20
+ retry_buffer_queue
21
+ end
22
+ @buffer_queue = ::Queue.new
23
+ @batch_size = Hutch::Config.get(:poller_batch_size)
24
+ @connected = Hutch.connected?
25
+ end
26
+
27
+ # 停止两个线程池
28
+ def shutdown
29
+ @message_worker.shutdown
30
+ @timer_worker.shutdown
31
+ end
32
+
33
+ # Bind a consumer's routing keys to its queue, and set up a subscription to
34
+ # receive messages sent to the queue.
35
+ def setup_queue(consumer)
36
+ logger.info "setting up queue: #{consumer.get_queue_name}"
37
+
38
+ queue = @broker.queue(consumer.get_queue_name, consumer.get_arguments)
39
+ @broker.bind_queue(queue, consumer.routing_keys)
40
+
41
+ queue.subscribe(consumer_tag: unique_consumer_tag, manual_ack: true) do |*args|
42
+ delivery_info, properties, payload = Hutch::Adapter.decode_message(*args)
43
+ handle_message_with_limits(consumer, delivery_info, properties, payload)
44
+ end
45
+ end
46
+
47
+ def handle_message_with_limits(consumer, delivery_info, properties, payload)
48
+ # 1. consumer.limit?
49
+ # 2. yes: make and ConsumerMsg to queue
50
+ # 3. no: post handle
51
+ @message_worker.post do
52
+ if consumer.ratelimit_exceeded?
53
+ @buffer_queue.push(ConsumerMsg.new(consumer, delivery_info, properties, payload))
54
+ else
55
+ # if Hutch disconnect skip do work let message timeout in rabbitmq waiting message push again
56
+ return unless @connected
57
+ consumer.ratelimit_add
58
+ handle_message(consumer, delivery_info, properties, payload)
59
+ end
60
+ end
61
+ end
62
+
63
+ # 心跳检查 Hutch 的连接
64
+ def heartbeat_connection
65
+ @connected = Hutch.connected?
66
+ end
67
+
68
+ # 每隔一段时间, 从 buffer queue 中转移任务到执行
69
+ def retry_buffer_queue
70
+ @batch_size.times do
71
+ cmsg = peak
72
+ return if cmsg.blank?
73
+ handle_message_with_limits(cmsg.consumer, cmsg.delivery_info, cmsg.properties, cmsg.payload)
74
+ end
75
+ end
76
+
77
+ # non-blocking pop message, if empty return nil. other error raise exception
78
+ def peak
79
+ @buffer_queue.pop(true)
80
+ rescue ThreadError => e
81
+ nil if e.to_s == "queue empty"
82
+ end
83
+ end
84
+
85
+ # Consumer Message wrap rabbitmq message infomation
86
+ class ConsumerMsg
87
+ attr_reader :consumer, :delivery_info, :properties, :payload
88
+
89
+ def initialize(consumer, delivery_info, properties, payload)
90
+ @consumer = consumer
91
+ @delivery_info = delivery_info
92
+ @properties = properties
93
+ @payload = payload
94
+ end
95
+ end
96
+ end
97
+
98
+
99
+
@@ -1,9 +1,12 @@
1
1
  require 'active_support/core_ext/module/delegation'
2
2
  require 'active_support/core_ext/object/blank'
3
3
  require 'hutch'
4
+ require 'hutch/patch/config'
5
+ require 'hutch/patch/worker'
6
+ require 'hutch/schedule/core'
4
7
  require 'hutch/enqueue'
8
+ require 'hutch/threshold'
5
9
  require 'hutch/error_handlers/max_retry'
6
- require 'hutch/schedule/core'
7
10
 
8
11
  # If ActiveJob is requried then required the adapter
9
12
  if defined?(ActiveJob)
@@ -15,13 +18,13 @@ module Hutch
15
18
  # If you want use it, just do `Hutch::Schedule.connect(Hutch.broker)` to initialize it
16
19
  # and then just use like Hutch to publish message `Hutch::Schedule.publish`
17
20
  module Schedule
18
-
21
+
19
22
  # fixed delay levels
20
23
  # seconds(4): 5s, 10s, 20s, 30s
21
24
  # minutes(14): 1m, 2m, 3m, 4m, 5m, 6m, 7m, 8m, 9m, 10m, 20m, 30m, 40m, 50m
22
25
  # hours(3): 1h, 2h, 3h
23
26
  DELAY_QUEUES = %w(5s 10s 20s 30s 60s 120s 180s 240s 300s 360s 420s 480s 540s 600s 1200s 1800s 2400s 3000s 3600s 7200s 10800s)
24
-
27
+
25
28
  class << self
26
29
  def connect
27
30
  ActiveJob::QueueAdapters::HutchAdapter.register_actice_job_classes if defined?(ActiveJob::QueueAdapters::HutchAdapter)
@@ -31,20 +34,26 @@ module Hutch
31
34
  @core = Hutch::Schedule::Core.new(Hutch.broker)
32
35
  @core.connect!
33
36
  end
34
-
37
+
38
+ def disconnect
39
+ Hutch.disconnect if Hutch.connected?
40
+ @core = nil
41
+ end
42
+
43
+
35
44
  def core
36
45
  @core
37
46
  end
38
-
47
+
39
48
  def publish(*args)
40
49
  core.publish(*args)
41
50
  end
42
-
51
+
43
52
  # fixed delay level queue's routing_key
44
53
  def delay_routing_key(suffix)
45
54
  "#{Hutch::Config.get(:mq_exchange)}.schedule.#{suffix}"
46
55
  end
47
-
56
+
48
57
  def delay_queue_name(suffix)
49
58
  "#{Hutch::Config.get(:mq_exchange)}_delay_queue_#{suffix}"
50
59
  end
@@ -8,52 +8,52 @@ module Hutch
8
8
  class Core
9
9
  attr_reader :broker, :exchange
10
10
  delegate :channel, :connection, :logger, to: :broker
11
-
11
+
12
12
  def initialize(broker)
13
13
  raise "Broker can`t be nil" if broker.blank?
14
14
  @broker = broker
15
15
  end
16
-
16
+
17
17
  # Becareful with the sequence of initialize
18
18
  def connect!
19
19
  declare_delay_exchange!
20
20
  declare_publisher!
21
21
  setup_delay_queues!
22
22
  end
23
-
23
+
24
24
  def declare_publisher!
25
25
  @publisher = Hutch::Publisher.new(connection, channel, exchange)
26
26
  end
27
-
27
+
28
28
  # The exchange used by Hutch::Schedule
29
29
  def declare_delay_exchange!
30
30
  @exchange = declare_delay_exchange
31
31
  end
32
-
32
+
33
33
  def declare_delay_exchange(ch = channel)
34
- exchange_name = "#{Hutch::Config.get(:mq_exchange)}.schedule"
34
+ exchange_name = "#{Hutch::Config.get(:mq_exchange)}.schedule"
35
35
  exchange_options = { durable: true }.merge(Hutch::Config.get(:mq_exchange_options))
36
36
  logger.info "using topic exchange(schedule) '#{exchange_name}'"
37
-
37
+
38
38
  broker.send(:with_bunny_precondition_handler, 'schedule exchange') do
39
39
  ch.topic(exchange_name, exchange_options)
40
40
  end
41
41
  end
42
-
42
+
43
43
  # The queue used by Hutch::Schedule
44
44
  def setup_delay_queues!
45
45
  DELAY_QUEUES.map { |suffix| setup_delay_queue!(suffix) }
46
46
  end
47
-
47
+
48
48
  def setup_delay_queue!(suffix)
49
49
  # TODO: extract the ttl to config params
50
50
  props = { :'x-message-ttl' => 30.days.in_milliseconds, :'x-dead-letter-exchange' => Hutch::Config.get(:mq_exchange) }
51
51
  queue = broker.queue(Hutch::Schedule.delay_queue_name(suffix), props)
52
-
52
+
53
53
  # bind routing_key to schedule exchange
54
54
  queue.bind(exchange, routing_key: Hutch::Schedule.delay_routing_key(suffix))
55
55
  end
56
-
56
+
57
57
  # Schedule`s publisher, publish the message to schedule topic exchange
58
58
  def publish(*args)
59
59
  @publisher.publish(*args)
@@ -1,5 +1,5 @@
1
1
  module Hutch
2
2
  module Schedule
3
- VERSION = "0.6.1"
3
+ VERSION = "0.7.0"
4
4
  end
5
5
  end
@@ -0,0 +1,77 @@
1
+ require 'active_support/concern'
2
+ require 'ratelimit'
3
+
4
+ module Hutch
5
+ # If consumer need `threshold`, just include this module. (included with Hutch::Enqueue)
6
+ module Threshold
7
+ extend ActiveSupport::Concern
8
+
9
+ # Add Consumer methods
10
+ class_methods do
11
+ # 限流速度:
12
+ # context: 可选的上下文, 默认为 default
13
+ # rate: 数量
14
+ # interval: 间隔
15
+ #
16
+ # block: 采用 lambada 的方式计算, 但需要返回三个参数:
17
+ # - context: 当前约束的上下文
18
+ # - rate: 数量
19
+ # - interval: 间隔
20
+ # 例如:
21
+ # - rate: 1, interval: 1, 每秒 1 个
22
+ # - rate: 30, interval: 1, 每秒 30 个
23
+ # - rate: 30, interval: 5, 每 5 s, 30 个
24
+ def threshold(args)
25
+ @block_given = args.is_a?(Proc)
26
+ if @block_given
27
+ @threshold_block = args
28
+ else
29
+ raise "need args or block" if args.blank?
30
+ raise "args need hash type" if args.class != Hash
31
+ args.symbolize_keys!
32
+ @context = args[:context].presence || "default"
33
+ @rate = args[:rate]
34
+ @interval = args[:interval]
35
+ end
36
+ # call redis.ping let fail fast if redis is not avalible
37
+ _redis.ping
38
+ # redis: 传入设置的 redis
39
+ # bucket_interval: 记录的间隔, 越小精度越大
40
+ @rate_limiter = Ratelimit.new(self.name,
41
+ bucket_interval: Hutch::Config.get(:ratelimit_bucket_interval),
42
+ redis: _redis)
43
+ end
44
+
45
+ # is class level @rate_limiter _context exceeded?
46
+ # if class level @rate_limiter is nil alwayt return false
47
+ def ratelimit_exceeded?
48
+ return false if @rate_limiter.blank?
49
+ @rate_limiter.exceeded?(_context, threshold: _rate, interval: _interval)
50
+ end
51
+
52
+ # 增加一次调用
53
+ def ratelimit_add
54
+ return if @rate_limiter.blank?
55
+ @rate_limiter.add(_context)
56
+ end
57
+
58
+ def _context
59
+ @block_given ? @threshold_block.call[:context] : @context
60
+ end
61
+
62
+ def _rate
63
+ @block_given ? @threshold_block.call[:rate] : @rate
64
+ end
65
+
66
+ def _interval
67
+ @block_given ? @threshold_block.call[:interval] : @interval
68
+ end
69
+
70
+ # all Consumers that use threshold module shared the same redis instance
71
+ def _redis
72
+ @@redis ||= Redis.new(url: Hutch::Config.get(:ratelimit_redis_url))
73
+ end
74
+ end
75
+ end
76
+ end
77
+
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: hutch-schedule
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.6.1
4
+ version: 0.7.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Wyatt pan
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2018-05-23 00:00:00.000000000 Z
11
+ date: 2020-04-12 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: hutch
@@ -16,14 +16,14 @@ dependencies:
16
16
  requirements:
17
17
  - - "~>"
18
18
  - !ruby/object:Gem::Version
19
- version: '0.24'
19
+ version: '1.0'
20
20
  type: :runtime
21
21
  prerelease: false
22
22
  version_requirements: !ruby/object:Gem::Requirement
23
23
  requirements:
24
24
  - - "~>"
25
25
  - !ruby/object:Gem::Version
26
- version: '0.24'
26
+ version: '1.0'
27
27
  - !ruby/object:Gem::Dependency
28
28
  name: multi_json
29
29
  requirement: !ruby/object:Gem::Requirement
@@ -38,6 +38,48 @@ dependencies:
38
38
  - - ">="
39
39
  - !ruby/object:Gem::Version
40
40
  version: '0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: concurrent-ruby
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: '1.1'
48
+ type: :runtime
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: '1.1'
55
+ - !ruby/object:Gem::Dependency
56
+ name: ratelimit
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - "~>"
60
+ - !ruby/object:Gem::Version
61
+ version: '1.0'
62
+ type: :runtime
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - "~>"
67
+ - !ruby/object:Gem::Version
68
+ version: '1.0'
69
+ - !ruby/object:Gem::Dependency
70
+ name: activesupport
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - ">="
74
+ - !ruby/object:Gem::Version
75
+ version: '5.0'
76
+ type: :runtime
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - ">="
81
+ - !ruby/object:Gem::Version
82
+ version: '5.0'
41
83
  - !ruby/object:Gem::Dependency
42
84
  name: activejob
43
85
  requirement: !ruby/object:Gem::Requirement
@@ -56,30 +98,30 @@ dependencies:
56
98
  name: bundler
57
99
  requirement: !ruby/object:Gem::Requirement
58
100
  requirements:
59
- - - "~>"
101
+ - - ">="
60
102
  - !ruby/object:Gem::Version
61
103
  version: '1.14'
62
104
  type: :development
63
105
  prerelease: false
64
106
  version_requirements: !ruby/object:Gem::Requirement
65
107
  requirements:
66
- - - "~>"
108
+ - - ">="
67
109
  - !ruby/object:Gem::Version
68
110
  version: '1.14'
69
111
  - !ruby/object:Gem::Dependency
70
112
  name: rake
71
113
  requirement: !ruby/object:Gem::Requirement
72
114
  requirements:
73
- - - "~>"
115
+ - - ">="
74
116
  - !ruby/object:Gem::Version
75
- version: '10.0'
117
+ version: 12.3.3
76
118
  type: :development
77
119
  prerelease: false
78
120
  version_requirements: !ruby/object:Gem::Requirement
79
121
  requirements:
80
- - - "~>"
122
+ - - ">="
81
123
  - !ruby/object:Gem::Version
82
- version: '10.0'
124
+ version: 12.3.3
83
125
  - !ruby/object:Gem::Dependency
84
126
  name: rspec
85
127
  requirement: !ruby/object:Gem::Requirement
@@ -130,9 +172,12 @@ files:
130
172
  - lib/hutch-schedule.rb
131
173
  - lib/hutch/enqueue.rb
132
174
  - lib/hutch/error_handlers/max_retry.rb
175
+ - lib/hutch/patch/config.rb
176
+ - lib/hutch/patch/worker.rb
133
177
  - lib/hutch/schedule.rb
134
178
  - lib/hutch/schedule/core.rb
135
179
  - lib/hutch/schedule/version.rb
180
+ - lib/hutch/threshold.rb
136
181
  homepage: https://github.com/wppurking/hutch-schedule
137
182
  licenses:
138
183
  - MIT
@@ -152,8 +197,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
152
197
  - !ruby/object:Gem::Version
153
198
  version: '0'
154
199
  requirements: []
155
- rubyforge_project:
156
- rubygems_version: 2.7.6
200
+ rubygems_version: 3.1.2
157
201
  signing_key:
158
202
  specification_version: 4
159
203
  summary: Add Schedule and Error Retry To Hutch.