hutch-schedule 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: fb70893978620f96f7701d003b06812d0b1de18c
-  data.tar.gz: 739355b288a4be2a3e54cc116a3193ec7edddff1
+  metadata.gz: 5cdb316a95fe95813079a1b059cd8221d1eabfec
+  data.tar.gz: 00057ddbb5ea2c6c810f292edf04a680439125f4
 SHA512:
-  metadata.gz: dda680d37cf8846741fa624db5e0b3d656f2dc3e7a2f1d14a347b04185f648bf81dfaf7e57b37df6a22507815c547a837efb1fabaf5e31600d7deb03652844f0
-  data.tar.gz: 32afd0caa0ff7b1510c4158344b1e164576eb4971e4778b5f3568410dd7d04fdc1bfaad6822229ca43883e5ee59c11a4b6140dcd800abe27e3701dfa471fee67
+  metadata.gz: 693efd110acf103a7d6513bfab197460f051676a1266c33a75f12936870b44c31fdaa1b67b20cd63adde014063a9d33eb522e3de26c24dce6628a41fd2c3fa39
+  data.tar.gz: 0b4cd7093767aba2b28128ca4fed1d8579354f23082a220c769e4f7ed707a9404e9363aaedf6efa041254a156de3eef3cbfb1e98c9ec5fe328a7223f60d9c76e

data/README.md

@@ -23,9 +23,8 @@ Or install it yourself as:
 Use the code below to initialize the Hutch::Schedule
 
 ```ruby
-Hutch::Config.setup_procs  << -> {
-	Hutch::Schedule.connect(Hutch.broker)
-}
+Hutch.connect
+Hutch::Schedule.connect
 ```
 
 They will do something below:
@@ -34,6 +33,71 @@ They will do something below:
 2. Declear an queue named `<hutch>_schedule_queue` and with some params:
   - Set `x-dead-letter-exchange: <hutch>`: let queue republish message to default <hutch> exchange.
   - Set `x-message-ttl: <30.days>`: to avoid the queue is to large, because there is no consumer with this queue.
+3. If ActiveJob is loaded. it will use `ActiveJob::Base.descendants` to register all ActiveJob class to one-job-per-consumer to Hutch::Consumer 
+
+
+### Error Retry
+If you want use error retry, then:
+
+1. Add `Hutch::ErrorHandlers::MaxRetry` to `Hutch::Config.error_handlers` like below
+```ruby
+Hutch::Config.error_handlers << Hutch::ErrorHandlers::MaxRetry.new
+```
+
+2. Let `Hutch::Consumer` to include `Hutch::Enqueue` and setup `attempts`
+```ruby
+class PlanConsumer
+  include Hutch::Consumer
+  include Hutch::Enqueue
+  
+  attempts 3
+  consume 'abc.plan'
+end
+```
+
+Error retry will use ActiveJob `exponentially_longer` algorithm `(executes**4) + 2` seconds
+
+
+## Rails
+
+### Work with Hutch it`s self
+Add an `hutch.rb` to `conf/initializers`:
+```ruby
+# reuse Hutch config.yaml file
+Hutch::Config.load_from_file(Rails.root.join('config', 'config.yaml'))
+# replace error_handlers with Hutch::ErrorHandlers::MaxRetry
+Hutch::Config.error_handlers = [Hutch::ErrorHandlers::MaxRetry.new]
+# Init Hutch
+Hutch.connect
+# Init Hutch::Schedule
+Hutch::Schedule.connect
+```
+
+Then you can enqueue message in Rails console like below:
+```ruby
+PlanConsumer.enqueue(a: 1)
+# or schedule message
+PlanConsumer.enqueue_in(5.seconds, a: 1)
+```
+
+### Work with ActiveJob
+```ruby
+class EmailJob < ApplicationJob
+  queue_as :email
+  
+  retry_on StandardError, wait: :exponentially_longer
+  
+  def perform(user_id)
+    user = User.find(user_id)
+    user.send_email
+  end
+end
+
+# in rails console, you can
+EmailJob.perform_later(user.id)
+# or
+EmailJob.set(wait: 5.seconds).perform_later(user.id)
+```
 
 ## Development
 

data/hutch-schedule.gemspec

@@ -6,7 +6,7 @@ require 'hutch/schedule/version'
 Gem::Specification.new do |spec|
   spec.name    = "hutch-schedule"
   spec.version = Hutch::Schedule::VERSION
-  spec.authors = ["wyatt pan"]
+  spec.authors = ["Wyatt pan"]
   spec.email   = ["wppurking@gmail.com"]
 
   spec.summary     = %q{Add Schedule and Error Retry To Hutch.}
@@ -22,6 +22,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_runtime_dependency 'hutch', '~> 0.24'
+  spec.add_runtime_dependency "multi_json"
 
   spec.add_development_dependency "activejob"
   spec.add_development_dependency "bundler", "~> 1.14"

data/lib/active_job/queue_adapters/hutch_adapter.rb

@@ -56,6 +56,11 @@ module ActiveJob
             define_method :process do |job_data|
               ActiveJob::Base.execute(job_data)
             end
+
+            # inspect name
+            define_method :inspect do
+              "#{job.queue_name}-anonymous-consumer"
+            end
           end
         end
       end

data/lib/hutch/enqueue.rb

@@ -17,16 +17,16 @@ module Hutch
       # publish message at a delay times
       # interval: delay interval
       # message: publish message
-      def enqueue_in(interval, message)
-        props = { expiration: interval.in_milliseconds.to_i }
-        Hutch::Schedule.publish(enqueue_routing_key, message, props)
+      def enqueue_in(interval, message, props = {})
+        properties = props.merge(expiration: interval.in_milliseconds.to_i)
+        Hutch::Schedule.publish(enqueue_routing_key, message, properties)
       end
 
       # delay at exatly time point
-      def enqueue_at(time, message)
+      def enqueue_at(time, message, props = {})
         # if time is early then now then just delay 1 second
         interval = [(time.utc - Time.now.utc), 1.second].max
-        enqueue_in(interval, message)
+        enqueue_in(interval, message, props)
       end
 
       # routing_key: the purpose is to send message to hutch exchange and then routing to the correct queue,
@@ -35,6 +35,14 @@ module Hutch
         raise "Routing Keys is not set!" if routing_keys.size < 1
         routing_keys.to_a.last
       end
+
+      def attempts(times)
+        @max_retries = [times, 0].max
+      end
+
+      def max_attempts
+        @max_retries || 0
+      end
     end
   end
 end

data/lib/hutch/error_handlers/max_retry.rb

@@ -1,15 +1,14 @@
 require 'hutch/logging'
+require 'active_support/core_ext/object/blank'
 
 module Hutch
   module ErrorHandlers
+
+    # When reach the Max Attempts, republish this message to RabbitMQ,
+    # And persisted the properties[:headers] to tell RabbitMQ the `x-dead.count`
     class MaxRetry
       include Logging
 
-      # TODO: Need to be implement.
-      # 1. 获取 hutch 本身记录的 x-death 中的错误次数
-      # 2. 从每一个 consumer 身上寻找 max_retry 的次数, 不超过则进行延迟重试
-      # 3. 根据错误次数计算类似 active_job 的 exponentially_longer 延迟时间
-      #
       # properties.headers example:
       # {
       #   "x-death": [
@@ -37,7 +36,43 @@ module Hutch
       #   ]
       # }
       def handle(properties, payload, consumer, ex)
-        raise "Not implement"
+        unless consumer.ancestors.include?(Hutch::Enqueue)
+          logger.warn("Consumer: #{consumer} is not include Hutch::Enqueue can`t use #enqueue_in`")
+          return false
+        end
+
+        prop_headers = properties[:headers] || {}
+        attempts     = failure_count(prop_headers, consumer) + 1
+        if attempts <= consumer.max_attempts
+          logger.debug("retrying, count=#{attempts}, headers:#{prop_headers}")
+          # execute_times = attempts - 1
+          consumer.enqueue_in(retry_delay(attempts - 1), MultiJson.decode(payload), { headers: prop_headers })
+        else
+          logger.debug("failing, retry_count=#{attempts}, ex:#{ex}")
+        end
+      end
+
+      def retry_delay(executes)
+        (executes**4) + 2
+      end
+
+      def failure_count(headers, consumer)
+        if headers.nil? || headers['x-death'].nil?
+          0
+        else
+          x_death_array = headers['x-death'].select do |x_death|
+            # http://ruby-doc.org/stdlib-2.2.3/libdoc/set/rdoc/Set.html#method-i-intersect-3F
+            (x_death['routing-keys'].presence || []).to_set.intersect?(consumer.routing_keys)
+          end
+
+          if x_death_array.count > 0 && x_death_array.first['count']
+            # Newer versions of RabbitMQ return headers with a count key
+            x_death_array.inject(0) { |sum, x_death| sum + x_death['count'] }
+          else
+            # Older versions return a separate x-death header for each failure
+            x_death_array.count
+          end
+        end
       end
     end
   end

data/lib/hutch/schedule.rb

@@ -2,6 +2,7 @@ require 'active_support/core_ext/module/delegation'
 require 'active_support/core_ext/object/blank'
 require 'hutch'
 require 'hutch/enqueue'
+require 'hutch/error_handlers/max_retry'
 require 'hutch/schedule/core'
 
 # If ActiveJob is requried then required the adapter
@@ -16,6 +17,7 @@ module Hutch
   module Schedule
 
     def self.connect(broker = Hutch.broker)
+      raise "Please invoke Hutch.connect before Hutch::Schedule.connect, Hutch::Schedule need Hutch.broker" unless Hutch.connected?
       return if core.present?
       @core = Hutch::Schedule::Core.new(broker)
       @core.connect!

data/lib/hutch/schedule/version.rb

@@ -1,5 +1,5 @@
 module Hutch
   module Schedule
-    VERSION = "0.2.0"
+    VERSION = "0.3.0"
   end
 end

metadata

@@ -1,10 +1,10 @@
 --- !ruby/object:Gem::Specification
 name: hutch-schedule
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
-- wyatt pan
+- Wyatt pan
 autorequire: 
 bindir: exe
 cert_chain: []
@@ -24,6 +24,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.24'
+- !ruby/object:Gem::Dependency
+  name: multi_json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: activejob
   requirement: !ruby/object:Gem::Requirement