hutch-schedule 0.7.2 → 0.7.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 51507037110c8e884ac519fe99c27f8547da70f5ada9744bc034a7ec10da5ee3
-  data.tar.gz: 45f2fcb4e686f86a13bcded021c661b69227c8eca5f5b0b96764a9ac840a8372
+  metadata.gz: adda9d8410e98a3010611c8bda21d0e79c98d840340a0f27fd5e3d4e0b60b46e
+  data.tar.gz: dd678456a60980e5e96fc4d85898afb5b71f663f5286026c0777287ee819956f
 SHA512:
-  metadata.gz: 1c507eead9eef80897516c1296cf3eaa3df4674fbba1b188138ce06317a332cfe886291e596a403f5fa3573079f4b9650a0206252e5567b451bda54dadeb07e4
-  data.tar.gz: 2f38719bc7da6b3ea9a7826a5e82555336b81b46acb62622af6d33f7ab1f52679af6f77d7e048274168dcb1166a10eb5ead8fe29fe1ffa5200a4022022444e18
+  metadata.gz: 07d613f9c09fe6dbccc0424af9edc536082b58e252d9a393685f30653c2d2564d28017eac8619299c14d5a5dae18a3fd61a897532c7ee3a8dac8ce069d0c6aed
+  data.tar.gz: 2303ce0e4213d387a96b6d1377c798e85ec36514099eb46b7a52296bc92922fe6f88f958815a035c30c66738da67802961a41b478d5c65ca30c9c20e95c48083

data/CHANGELOG.md

@@ -1,6 +1,16 @@
 # Change Log
 All notable changes to this project will be documented in this file.
 
+## [0.7.3] - 2020-04-16
+### Fixed
+- add #threshold Proc to support pass enqueue msg to lambada args 
+- add check interval for flush Hutch::Worker.buffer_queue to RabbitMQ to avoid blocking for handling limited message
+
+## [0.7.1] - 2020-04-16
+### Fixed
+- add threshold default {context, rate, interval} value
+- fix monkey patch Hutch::Config.define_methods
+
 ## [0.7.0] - 2020-04-13
 ### Fixed
 - Use monkey patch to support Conumser ratelimit through Hutch::Threshold

data/README.md

@@ -62,13 +62,17 @@ poller_interval| 1 | seconds of the poller to trigger, poller the message in Buf
 poller_batch_size | 100 | the message size of every batch triggerd by the poller
 redis_url | redis://127.0.0.1:6379/0 | Redis backend url for Ratelimit and Unique Job 
 ratelimit_bucket_interval | 1 | Ratelimit use the time bucket (seconds) to store the counts, lower the more accurate
+worker_buffer_flush_interval | 6 | Monkey patch, flush the Hutch::Worker.@buffer_queue message to RabbitMQ check interval
 
 ## Hutch::Enqueue
-Let consumer to include `Hutch::Enqueue` then it has the ability of publishing message to RabbitMQ with the `consume '<routing_key>'`
+Let consumer to include `Hutch::Enqueue` then it has the ability of publishing message to RabbitMQ with the `consume '<routing_key>'`.
+
+*Only support enqueue `Hash` format message*
 
 * enqueue: just publish one message
 * enqueue_in: publish one message and delay <interval> seconds
 * enqueue_at: publish one message and auto calculate the <interval> seconds need to delay
+* enqueue_uniq(_in/at): publish uniq message with uniq_key
 
 According to the RabbitMQ [TTL Message design limits](http://www.rabbitmq.com/ttl.html#per-message-ttl-caveats) ([discus](https://github.com/rebus-org/Rebus/issues/594#issuecomment-289961537)),
 We design the fixed delay level from seconds to hours, below is the details:
@@ -111,7 +115,7 @@ end
 ```
 
 threshold lambada need get return value must be a Hash and include:
-* context: the limit context with currency threshold
+* context: the limit context with current threshold
 * rate: the rate speed of threshold
 * interval: the time range of threshold
 

data/lib/hutch/enqueue.rb

@@ -11,21 +11,22 @@ module Hutch
     
     # Add Consumer methods
     class_methods do
+      
       # Publish the message to this consumer with one routing_key
-      def enqueue(message)
-        Hutch.publish(enqueue_routing_key, message)
+      def enqueue(msg = {})
+        Hutch.publish(enqueue_routing_key, msg)
       end
       
       # enqueue unique message
-      def enqueue_uniq(uniq_key, message)
+      def enqueue_uniq(uniq_key, msg = {})
         return false unless uniq_key_check(uniq_key)
-        enqueue(message)
+        enqueue(msg)
       end
       
       # publish message at a delay times
       # interval: delay interval seconds
       # message: publish message
-      def enqueue_in(interval, message, props = {})
+      def enqueue_in(interval, message = {}, props = {})
         # TODO: 超过 3h 的延迟也会接收, 但是不会延迟那么长时间, 但给予 warn
         delay_seconds = delay_seconds_level(interval)
         
@@ -40,13 +41,13 @@ module Hutch
         Hutch::Schedule.publish(delay_routing_key, message, properties)
       end
       
-      def enqueue_uniq_in(uniq_key, interval, message, props = {})
+      def enqueue_uniq_in(uniq_key, interval, message = {}, props = {})
         return false unless uniq_key_check(uniq_key)
         enqueue_in(interval, message, props)
       end
       
       # delay at exatly time point
-      def enqueue_at(time, message, props = {})
+      def enqueue_at(time, message = {}, props = {})
         # compatible with with ActiveJob API
         time_or_timestamp = time.respond_to?(:utc) ? time.utc.to_f : time
         # if time is early then now then just delay 1 second
@@ -54,7 +55,7 @@ module Hutch
         enqueue_in(interval, message, props)
       end
       
-      def enqueue_uniq_at(uniq_key, time, message, props = {})
+      def enqueue_uniq_at(uniq_key, time, message = {}, props = {})
         return false unless uniq_key_check(uniq_key)
         enqueue_at(time, message, props)
       end

data/lib/hutch/patch/config.rb

@@ -19,6 +19,10 @@ module Hutch
     # Ratelimit redis backend reconnect attempts
     number_setting :ratelimit_redis_reconnect_attempts, 10
     
+    # Hutch::Worker buffer flush interval in seconds
+    # 这个时间长度决定了 woker.buffer_queue 中长周期等待的任务交换给 RabbitMQ 的检查周期, 不适合太过频繁
+    number_setting :worker_buffer_flush_interval, 6
+    
     initialize(
       worker_pool_size:  20,
       poller_interval:   1,
@@ -26,7 +30,10 @@ module Hutch
       # @see Redis::Client
       redis_url:                          "redis://127.0.0.1:6379/0",
       ratelimit_bucket_interval:          1,
-      ratelimit_redis_reconnect_attempts: 10
+      ratelimit_redis_reconnect_attempts: 10,
+      worker_buffer_flush_interval:       6,
+      # need an positive channel_prefetch
+      channel_prefetch: 20
     )
     define_methods
   end

data/lib/hutch/patch/worker.rb

@@ -10,30 +10,36 @@ module Hutch
   # 明确告知 RabbitMQ 此任务完成.
   class Worker
     def initialize(broker, consumers, setup_procs)
+      raise "use Hutch::Schedule must set an positive channel_prefetch" if Hutch::Config.get(:channel_prefetch) < 1
       @broker          = broker
       self.consumers   = consumers
       self.setup_procs = setup_procs
       
       @message_worker = Concurrent::FixedThreadPool.new(Hutch::Config.get(:worker_pool_size))
       @timer_worker   = Concurrent::TimerTask.execute(execution_interval: Hutch::Config.get(:poller_interval)) do
+        # all chekcer in the same thread
         heartbeat_connection
+        flush_to_retry
         retry_buffer_queue
       end
       
-      # The queue size maybe the same as channel[prefetch] and every Consumer have it's own buffer queue with the same prefetch size,
-      # when the buffer queue have the prefetch size message rabbitmq will stop push message to this consumer but it's ok.
-      # The consumer will threshold by the shared redis instace.
-      @buffer_queue = ::Queue.new
-      @batch_size   = Hutch::Config.get(:poller_batch_size)
-      @connected    = Hutch.connected?
+      # The queue size maybe the same as channel[prefetch] and every Consumer shared one buffer queue with the
+      # same prefetch size, when current consumer have unack messages reach the prefetch size rabbitmq will stop push
+      # message to this consumer.
+      # Because the buffer queue is shared by all consumers so the max queue size is [prefetch * consumer count],
+      # if prefetch is 20 and have 30 consumer the max queue size is  20 * 30 = 600.
+      @buffer_queue    = ::Queue.new
+      @batch_size      = Hutch::Config.get(:poller_batch_size)
+      @connected       = Hutch.connected?
+      @last_flush_time = Time.now.utc
     end
     
     # Stop a running worker by killing all subscriber threads.
     # Stop two thread pool
     def stop
       @timer_worker.shutdown
-      @message_worker.shutdown
       @broker.stop
+      @message_worker.shutdown
     end
     
     # Bind a consumer's routing keys to its queue, and set up a subscription to
@@ -50,33 +56,93 @@ module Hutch
       end
     end
     
-    def handle_message_with_limits(consumer, delivery_info, properties, payload)
-      # 1. consumer.limit?
-      # 2. yes: make and ConsumerMsg to queue
-      # 3. no: post handle
+    # cmsg: ConsumerMsg
+    def handle_cmsg_with_limits(cmsg)
+      # 正常的任务处理 ratelimit 的处理逻辑, 如果有限制那么就进入 buffer 缓冲
+      consumer = cmsg.consumer
       @message_worker.post do
-        if consumer.ratelimit_exceeded?
-          @buffer_queue.push(ConsumerMsg.new(consumer, delivery_info, properties, payload))
+        if consumer.ratelimit_exceeded?(cmsg.message)
+          @buffer_queue.push(cmsg)
         else
           # if Hutch disconnect skip do work let message timeout in rabbitmq waiting message push again
           return unless @connected
-          consumer.ratelimit_add
-          handle_message(consumer, delivery_info, properties, payload)
+          consumer.ratelimit_add(cmsg.message)
+          handle_cmsg(*cmsg.handle_cmsg_args)
         end
       end
     end
     
+    def handle_message_with_limits(consumer, delivery_info, properties, payload)
+      handle_cmsg_with_limits(consumer_msg(consumer, delivery_info, properties, payload))
+    end
+    
+    # change args to message reuse the code from #handle_message
+    def consumer_msg(consumer, delivery_info, properties, payload)
+      serializer = consumer.get_serializer || Hutch::Config[:serializer]
+      logger.debug {
+        spec = serializer.binary? ? "#{payload.bytesize} bytes" : "#{payload}"
+        "message(#{properties.message_id || '-'}): " +
+          "routing key: #{delivery_info.routing_key}, " +
+          "consumer: #{consumer}, " +
+          "payload: #{spec}"
+      }
+      
+      ConsumerMsg.new(consumer, Hutch::Message.new(delivery_info, properties, payload, serializer))
+    end
+    
+    def handle_cmsg(consumer, delivery_info, properties, payload, message)
+      consumer_instance = consumer.new.tap { |c| c.broker, c.delivery_info = @broker, delivery_info }
+      with_tracing(consumer_instance).handle(message)
+      @broker.ack(delivery_info.delivery_tag)
+    rescue => ex
+      acknowledge_error(delivery_info, properties, @broker, ex)
+      handle_error(properties, payload, consumer, ex)
+    end
+    
+    
     # 心跳检查 Hutch 的连接
     def heartbeat_connection
       @connected = Hutch.connected?
     end
     
-    # 每隔一段时间, 从 buffer queue 中转移任务到执行
+    # 每隔一段时间, 从 buffer queue 中转移任务到执行, interval 比较短的会立即执行掉
     def retry_buffer_queue
+      begin_size = @buffer_queue.size
+      now        = Time.now.utc
+      stat       = {}
       @batch_size.times do
         cmsg = peak
-        return if cmsg.blank?
-        handle_message_with_limits(cmsg.consumer, cmsg.delivery_info, cmsg.properties, cmsg.payload)
+        break if cmsg.blank?
+        handle_cmsg_with_limits(cmsg)
+        
+        next unless logger.level == Logger::DEBUG
+        if stat.key?(cmsg.message.body[:b])
+          stat[cmsg.message.body[:b]] += 1
+        else
+          stat[cmsg.message.body[:b]] = 1
+        end
+      end
+      logger.debug "retry_buffer_queue #{Time.now.utc - now}, size from #{begin_size} to #{@buffer_queue.size}, stat: #{stat}"
+    end
+    
+    # 对于 rate 间隔比较长的, 不适合一直存储在 buffer 中, 所以需要根据 interval 的值将长周期的 message 重新入队给 RabbitMQ 让其进行
+    # 等待, 但同时不可以让其直接 Requeue, 这样会导致频繁的与 RabbitMQ 来往交换. 需要让消息根据周期以及执行次数逐步拉长等待, 直到最终最长
+    # 时间的等待.
+    #
+    # 有下面几个要求:
+    #  - 在 retry_buffer_queue 之前调用
+    #  - 整个方法调用时间长度需要在 1s 之内
+    def flush_to_retry
+      now = Time.now.utc
+      if now - @last_flush_time >= Hutch::Config.get(:worker_buffer_flush_interval)
+        @buffer_queue.size.times do
+          cmsg = peak
+          break if cmsg.blank?
+          # 如果没有被处理, 重新放回 buffer
+          @buffer_queue.push(cmsg) unless cmsg.enqueue_in_or_not
+        end
+        @last_flush_time = now
+        logger.debug "flush_to_retry #{Time.now.utc - now}"
       end
     end
     
@@ -88,15 +154,41 @@ module Hutch
     end
   end
   
-  # Consumer Message wrap rabbitmq message infomation
+  # Consumer Message wrap Hutch::Message and Consumer
   class ConsumerMsg
-    attr_reader :consumer, :delivery_info, :properties, :payload
+    attr_reader :consumer, :message
+    
+    def logger
+      Hutch::Logging.logger
+    end
+    
+    def initialize(consumer, message)
+      @consumer = consumer
+      @message  = message
+    end
+    
+    def handle_cmsg_args
+      [consumer, message.delivery_info, message.properties, message.payload, message]
+    end
+    
+    def interval
+      @interval ||= consumer.interval(message)
+    end
     
-    def initialize(consumer, delivery_info, properties, payload)
-      @consumer      = consumer
-      @delivery_info = delivery_info
-      @properties    = properties
-      @payload       = payload
+    # if delays > 10s then let the message to rabbitmq to delay and enqueue again instead of rabbitmq reqneue
+    def enqueue_in_or_not
+      # interval 小于 5s, 的则不会传, 在自己的 buffer 中等待
+      return false if interval < Hutch::Config.get(:worker_buffer_flush_interval)
+      # 等待时间过长的消息, 交给远端的 rabbitmq 去进行等待, 不占用 buffer 空间
+      # 如果数据量特别大, 但 ratelimit 特别严格, 那么也会变为固定周期的积压, 需要增加对执行次数的记录以及延长
+      # 市场 30s 执行一次的任务, 积累了 200 个, 那么这个积压会越来越多, 直到保持到一个 RabbitMQ 与 hutch 之间的最长等待周期, 会一直空转
+      #  - 要么增加对执行次数的考虑, 拉长延长. 但最终会有一个最长的延长 10800 (3h), 这个问题最终仍然会存在
+      #  - 设置延长多长之后, 就舍弃这个任务, 因为由于 ratelimit 的存在, 但又持续的积压, 不可能处理完这个任务
+      # 这个方案没有很好的解决方法, 这是一个典型的 "生产速度大于消费速度" 的问题, 如果长时间的 生产 > 消费, 这个问题是无解的
+      Hutch.broker.ack(message.delivery_info.delivery_tag)
+      # TODO: 如果存在 x-death 的 count 需要额外考虑, 解决与 error retry 的 x-death 复用的问题
+      # 临时给一个随机的 1,2 倍率的延迟, 大概率为 1 倍,小概率为 2 倍
+      consumer.enqueue_in(interval * [rand(3), 1].max, message.body, message.properties.to_hash)
     end
   end
 end

data/lib/hutch/schedule/version.rb

@@ -1,5 +1,5 @@
 module Hutch
   module Schedule
-    VERSION = "0.7.2"
+    VERSION = "0.7.3"
   end
 end

data/lib/hutch/threshold.rb

@@ -31,6 +31,7 @@ module Hutch
       def threshold(args)
         @block_given = args.is_a?(Proc)
         if @block_given
+          raise "block only can have zero or one arguments" if args.arity > 1
           @threshold_block = args
         else
           raise "need args or block" if args.blank?
@@ -49,34 +50,48 @@ module Hutch
                                       redis:           Hutch::Schedule.redis)
       end
       
+      
       # is class level @rate_limiter _context exceeded?
       # if class level @rate_limiter is nil alwayt return false
-      def ratelimit_exceeded?
+      def ratelimit_exceeded?(message)
         return false if @rate_limiter.blank?
-        @rate_limiter.exceeded?(_context, threshold: _rate, interval: _interval)
+        args = threshold_args(message)
+        @rate_limiter.exceeded?(_context(args), threshold: _rate(args), interval: _interval(args))
       rescue Redis::BaseError
         # when redis cann't connect return exceeded limit
         true
       end
       
       # 增加一次调用
-      def ratelimit_add
+      def ratelimit_add(message)
         return if @rate_limiter.blank?
-        @rate_limiter.add(_context)
+        @rate_limiter.add(_context(threshold_args(message)))
       rescue Redis::BaseError
         nil
       end
       
-      def _context
-        @block_given ? @threshold_block.call[:context].presence || default_context : @context
+      def threshold_args(message)
+        if @block_given
+          @threshold_block.arity == 0 ? @threshold_block.call : @threshold_block.call(message.body)
+        else
+          { context: @context, rate: @rate, interval: @interval }
+        end
+      end
+      
+      def interval(message)
+        _interval(threshold_args(message))
+      end
+      
+      def _context(args)
+        args.fetch(:context, default_context)
       end
       
-      def _rate
-        @block_given ? @threshold_block.call[:rate].presence || default_rate : @rate
+      def _rate(args)
+        args.fetch(:rate, default_rate)
       end
       
-      def _interval
-        @block_given ? @threshold_block.call[:interval].presence || default_interval : @interval
+      def _interval(args)
+        args.fetch(:interval, default_interval)
       end
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: hutch-schedule
 version: !ruby/object:Gem::Version
-  version: 0.7.2
+  version: 0.7.3
 platform: ruby
 authors:
 - Wyatt pan
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-04-15 00:00:00.000000000 Z
+date: 2020-04-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: hutch