sad 1.3.0 → 1.4.0

data/Gemfile

@@ -3,6 +3,8 @@ source "http://ruby.taobao.org"
 gem 'jeweler'
 
 gem 'activesupport'
+gem 'eventmachine'
 gem 'em-hiredis'
 gem 'daemons'
-gem	"json"
+gem	"json"
+gem 'pry'

data/Gemfile.lock

@@ -4,9 +4,11 @@ GEM
     activesupport (3.2.13)
       i18n (= 0.6.1)
       multi_json (~> 1.0)
+    coderay (1.0.9)
     daemons (1.1.9)
     em-hiredis (0.2.1)
       hiredis (~> 0.4.0)
+    eventmachine (1.0.3)
     git (1.2.5)
     hiredis (0.4.5)
     i18n (0.6.1)
@@ -16,10 +18,16 @@ GEM
       rake
       rdoc
     json (1.7.7)
+    method_source (0.8.1)
     multi_json (1.7.2)
+    pry (0.9.12)
+      coderay (~> 1.0.5)
+      method_source (~> 0.8)
+      slop (~> 3.4)
     rake (10.0.4)
     rdoc (4.0.0)
       json (~> 1.4)
+    slop (3.4.4)
 
 PLATFORMS
   ruby
@@ -28,5 +36,7 @@ DEPENDENCIES
   activesupport
   daemons
   em-hiredis
+  eventmachine
   jeweler
   json
+  pry

data/README.md

@@ -41,7 +41,6 @@ EM.run{
 
 两个运行环境都要有SadJob这个测试类的定义存在
 
---------
 
 ## 在项目中使用
 
@@ -90,6 +89,40 @@ Sad::Config.namespace = 'MyBackgroundJobQueue'
 		 6181   ??  S      0:02.61 Sad-2      
 		 6522 s002  R+     0:00.00 grep Sad
 
+
+## 测试
+
+启动一个队列
+
+```sh
+bundle exec rake sad:restart COUNT=1 QUEUE=MySadJob DIR=./tmp/pids
+```
+
+执行最简单的测试
+
+```sh
+bundle exec ruby ./test/test_sad.rb
+```
+
+执行异常任务测试,异常是指Klass.perform函数执行时出现异常，如果是异步任务，没办法知道异常，因为执行完perform后，代码运行环境已经脱离sad了，需要人工自己处理.
+
+当Klass.perform出错时，会延时重试，重试三次后，如果还有错误，则放弃该任务.
+
+```sh
+bundle exec ruby ./test/test_perform_with_exception.rb
+```
+
+在异步模式的代码中，异常如果被抛给top level的运行时的话，则代码无法控制，所以有异常的时候一定要捕获到，写在类似errback的回调中，对于Sad的重试，自动重试只能保证perform执行没问题，如果在callback中出错，则需要用户手工将该任务重新enqueue来实现重试的机制。
+
+## 任务配置
+
+```ruby
+SadJob.enqueue(1,2,3,4,5){|payload|
+	# delay -> 当sad进程得到该任务时并不立即执行，而是延迟多干秒后执行
+	payload.sad_args['delay'] = 5
+}
+```
+
 ## Contributing to sad
  
 * Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet.

data/Rakefile

@@ -43,3 +43,6 @@ Rake::RDocTask.new do |rdoc|
   rdoc.rdoc_files.include('README*')
   rdoc.rdoc_files.include('lib/**/*.rb')
 end
+
+require File.join(File.dirname(__FILE__), 'lib', 'sad')
+require File.join(File.dirname(__FILE__), 'lib', 'sad', 'tasks')

data/VERSION

@@ -1 +1 @@
-1.3.0
+1.4.0

data/lib/sad/config.rb

@@ -9,6 +9,10 @@ module Sad
 				@_namespace || 'SadQueue'
 			end
 
+			def queue(q)
+				[Sad::Config.namespace, q].join ':'
+			end
+
 			def redis=(uri)
 				@_redis_url = uri
 			end
@@ -16,6 +20,22 @@ module Sad
 			def redis
 				@_redis ||= EM::Hiredis.connect(@_redis_url)
 			end
+
+			def interval=(int)
+				@_interval = int.to_i
+			end
+
+			def interval
+				@_interval ||= 3
+			end
+
+			def max_retry=(int)
+				@_max_retry = int.to_i
+			end
+
+			def max_retry
+				@_max_retry ||= 3
+			end
 		end
 	end
 end

data/lib/sad/payload.rb

@@ -1,22 +1,55 @@
+require "json"
+
 module Sad
 	class Payload
-		attr_accessor :klass, :args
+		attr_accessor :klass, :args, :sad_args
 
-		def initialize(klass, args)
+		def initialize(klass, args = [], sad_args = {})
 			@klass = klass
 			@args = args
+			@sad_args = {
+				'retry' => 0,
+				'delay' => 0
+			}.update(sad_args)
 		end
 
 		def encode
 			{
 				'klass' => @klass,
-				'args' => @args
+				'args' => @args,
+				'sad_args' => @sad_args
 			}.to_json
 		end
 
+		# 执行任务
+		# 当执行任务的perform出错时
+		# 重试1至::Sad::Config.max_retry次
+		# 每次重试时，延迟重试次数*::Sad::Config.interval的时长后，再enqueue
+		def perform
+			begin
+				@klass.constantize.send :perform, *@args
+			rescue Exception => e
+				if self.sad_args['retry'] and (self.sad_args['retry'].to_i < ::Sad::Config.max_retry)
+					self.sad_args['retry'] = self.sad_args['retry'].to_i + 1
+					self.sad_args['delay'] = ::Sad::Config.interval * self.sad_args['retry']
+					self.enqueue
+				else
+					STDERR.puts "Payload perform error for #{self.sad_args['retry']} retrys:\n#{self.inspect}"
+				end
+			end
+		end
+
+		def enqueue
+			::Sad::Config.redis.rpush(self.sad_args['queue'], self.encode)
+		end
+
 		def self.decode(json)
 			h = JSON.parse(json)
-			self.new(h['klass'], h['args'])
+			if h['sad_args'] or h['sad_args'] != ''
+				self.new(h['klass'], h['args'], h['sad_args'])
+			else
+				self.new(h['klass'], h['args'])
+			end
 		end
 	end
 end

data/lib/sad/server.rb

@@ -4,7 +4,7 @@ module Sad
 			def run(queue)
 				@_shutdown = false
 				register_signal
-				fetch([Sad::Config.namespace, queue].join ':')
+				fetch(Sad::Config.queue(queue))
 			end
 
 			def fetch(queue)
@@ -13,17 +13,34 @@ module Sad
 					if data
 						STDOUT.puts '-'*15 + data.inspect + '-'*15
 						payload = Payload.decode(data)
-						EM.defer{perform(payload.klass, payload.args)}
+						payload_call(payload)
 					end
-					fetch(queue) unless shutdown?
+					fetch_with_interval(queue)
 				}
 				request.errback{
+					STDERR.puts 'error with redis request.'
+					fetch_with_interval(queue)
+				}
+			end
+
+			def fetch_with_interval(queue)
+				EM.add_timer(::Sad::Config.interval){
 					fetch(queue) unless shutdown?
 				}
 			end
 
-			def perform(klass, args)
-				klass.constantize.send :perform, *args
+			def payload_call(payload)
+				# 如果该任务有延时执行要求，
+				# 则在定时器执行时将其延时的key删掉，
+				# 再重新入队
+				if payload.sad_args['delay'] and payload.sad_args['delay'] != '' and payload.sad_args['delay'] != 0
+					EM.add_timer(payload.sad_args['delay'].to_i){
+						payload.sad_args.delete('delay')
+						payload.enqueue
+					}
+				else
+					payload.perform
+				end
 			end
 
 			def register_signal
@@ -43,6 +60,7 @@ module Sad
 
 			def shutdown
 				@_shutdown = true
+				EM.stop
 			end
 		end
 	end

data/lib/sad/worker.rb

@@ -6,12 +6,14 @@ module Sad
 			else
 				nil
 			end
-			[Sad::Config.namespace, name].join ':'
+			Sad::Config.queue(name)
 		end
 
 		def enqueue(*args)
 			payload = ::Sad::Payload.new(self.to_s, args)
-			::Sad::Config.redis.rpush(queue_name, payload.encode)
+			payload.sad_args['queue'] = queue_name
+			yield payload if block_given?
+			payload.enqueue
 		end
 	end
 end

data/sad.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "sad"
-  s.version = "1.3.0"
+  s.version = "1.4.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["\u{5d14}\u{5ce5}"]
-  s.date = "2013-04-26"
+  s.date = "2013-04-27"
   s.description = "a simple em baseed background job worker."
   s.email = "zheng.cuizh@gmail.com"
   s.extra_rdoc_files = [
@@ -34,6 +34,7 @@ Gem::Specification.new do |s|
     "lib/tasks/sad.rake",
     "sad.gemspec",
     "test/helper.rb",
+    "test/test_perform_with_exception.rb",
     "test/test_sad.rb"
   ]
   s.homepage = "http://github.com/charlescui/sad"
@@ -48,22 +49,28 @@ Gem::Specification.new do |s|
     if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
       s.add_runtime_dependency(%q<jeweler>, [">= 0"])
       s.add_runtime_dependency(%q<activesupport>, [">= 0"])
+      s.add_runtime_dependency(%q<eventmachine>, [">= 0"])
       s.add_runtime_dependency(%q<em-hiredis>, [">= 0"])
       s.add_runtime_dependency(%q<daemons>, [">= 0"])
       s.add_runtime_dependency(%q<json>, [">= 0"])
+      s.add_runtime_dependency(%q<pry>, [">= 0"])
     else
       s.add_dependency(%q<jeweler>, [">= 0"])
       s.add_dependency(%q<activesupport>, [">= 0"])
+      s.add_dependency(%q<eventmachine>, [">= 0"])
       s.add_dependency(%q<em-hiredis>, [">= 0"])
       s.add_dependency(%q<daemons>, [">= 0"])
       s.add_dependency(%q<json>, [">= 0"])
+      s.add_dependency(%q<pry>, [">= 0"])
     end
   else
     s.add_dependency(%q<jeweler>, [">= 0"])
     s.add_dependency(%q<activesupport>, [">= 0"])
+    s.add_dependency(%q<eventmachine>, [">= 0"])
     s.add_dependency(%q<em-hiredis>, [">= 0"])
     s.add_dependency(%q<daemons>, [">= 0"])
     s.add_dependency(%q<json>, [">= 0"])
+    s.add_dependency(%q<pry>, [">= 0"])
   end
 end
 

data/test/test_perform_with_exception.rb

@@ -0,0 +1,20 @@
+$:.unshift(File.dirname __FILE__)
+require 'helper'
+
+class SadJob
+	extend ::Sad::Worker
+	
+	def self.queue
+		'MySadJob'
+	end
+
+	def self.perform(*args)
+		raise RuntimeError, 'Error for test!!!!'
+	end
+end
+
+EM.run {
+	EM::PeriodicTimer.new(3){
+		SadJob.enqueue('this is some args', {:hello => 'code'})	
+	}
+}

data/test/test_sad.rb

@@ -1,3 +1,4 @@
+$:.unshift(File.dirname __FILE__)
 require 'helper'
 
 class SadJob
@@ -13,6 +14,8 @@ class SadJob
 	end
 end
 
-EM::PeriodicTimer.new(3){
-	SadJob.enqueue('this is some args', {:hello => 'code'})	
-}
+EM.run{
+	EM::PeriodicTimer.new(3){
+		SadJob.enqueue('this is some args', {:hello => 'code'})	
+	}
+}

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: sad
 version: !ruby/object:Gem::Version
-  version: 1.3.0
+  version: 1.4.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-04-26 00:00:00.000000000 Z
+date: 2013-04-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: jeweler
@@ -43,6 +43,22 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: eventmachine
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: em-hiredis
   requirement: !ruby/object:Gem::Requirement
@@ -91,6 +107,22 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: a simple em baseed background job worker.
 email: zheng.cuizh@gmail.com
 executables: []
@@ -116,6 +148,7 @@ files:
 - lib/tasks/sad.rake
 - sad.gemspec
 - test/helper.rb
+- test/test_perform_with_exception.rb
 - test/test_sad.rb
 homepage: http://github.com/charlescui/sad
 licenses:
@@ -132,7 +165,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: 3515885531508128638
+      hash: -1386435798103601784
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements: