magent 0.6.2 → 0.7.0

Sign up to get free protection for your applications and to get access to all the features.
data/Gemfile CHANGED
@@ -1,10 +1,11 @@
1
1
  source 'http://rubygems.org'
2
2
 
3
- gem 'mongo', '~> 1.0'
3
+ gem 'mongo'
4
4
  gem 'uuidtools'
5
5
 
6
6
  gem 'em-websocket'
7
7
 
8
- gem 'sinatra', '~> 1.2.6'
9
- gem 'haml', '~> 3.1.1'
10
- gem 'launchy', '~> 0.4.0'
8
+ gem 'sinatra'
9
+ gem 'haml'
10
+ gem 'launchy'
11
+
@@ -1,34 +1,34 @@
1
1
  GEM
2
2
  remote: http://rubygems.org/
3
3
  specs:
4
- addressable (2.2.1)
5
- bson (1.1)
6
- configuration (1.2.0)
7
- em-websocket (0.1.4)
4
+ addressable (2.2.7)
5
+ bson (1.6.2)
6
+ em-websocket (0.3.6)
8
7
  addressable (>= 2.1.1)
9
8
  eventmachine (>= 0.12.9)
10
9
  eventmachine (0.12.10)
11
- haml (3.1.1)
12
- launchy (0.4.0)
13
- configuration (>= 0.0.5)
14
- rake (>= 0.8.1)
15
- mongo (1.1)
16
- bson (>= 1.0.5)
17
- rack (1.3.0)
18
- rake (0.9.0)
19
- sinatra (1.2.6)
20
- rack (~> 1.1)
21
- tilt (>= 1.2.2, < 2.0)
22
- tilt (1.3.2)
23
- uuidtools (2.1.1)
10
+ haml (3.1.4)
11
+ launchy (2.1.0)
12
+ addressable (~> 2.2.6)
13
+ mongo (1.6.2)
14
+ bson (~> 1.6.2)
15
+ rack (1.4.1)
16
+ rack-protection (1.2.0)
17
+ rack
18
+ sinatra (1.3.2)
19
+ rack (~> 1.3, >= 1.3.6)
20
+ rack-protection (~> 1.2)
21
+ tilt (~> 1.3, >= 1.3.3)
22
+ tilt (1.3.3)
23
+ uuidtools (2.1.2)
24
24
 
25
25
  PLATFORMS
26
26
  ruby
27
27
 
28
28
  DEPENDENCIES
29
29
  em-websocket
30
- haml (~> 3.1.1)
31
- launchy (~> 0.4.0)
32
- mongo (~> 1.0)
33
- sinatra (~> 1.2.6)
30
+ haml
31
+ launchy
32
+ mongo
33
+ sinatra
34
34
  uuidtools
data/Rakefile CHANGED
@@ -1,6 +1,9 @@
1
1
  require 'rubygems'
2
2
  require 'rake'
3
3
 
4
+ $:.unshift File.expand_path("../lib", __FILE__)
5
+ require 'magent'
6
+
4
7
  begin
5
8
  require 'jeweler'
6
9
  Jeweler::Tasks.new do |gem|
@@ -40,11 +43,9 @@ rescue LoadError
40
43
  end
41
44
  end
42
45
 
43
- task :test => :check_dependencies
44
-
45
46
  task :default => :test
46
47
 
47
- require 'rake/rdoctask'
48
+ require 'rdoc/task'
48
49
  Rake::RDocTask.new do |rdoc|
49
50
  version = File.exist?('VERSION') ? File.read('VERSION') : ""
50
51
 
data/VERSION CHANGED
@@ -1 +1 @@
1
- 0.6.2
1
+ 0.7.0
data/bin/magent CHANGED
@@ -98,9 +98,7 @@ class Controller
98
98
  def stop
99
99
  begin
100
100
  pid = File.read(pid_file).to_i
101
- Process.kill("TERM", pid)
102
- Process.kill(0, pid)
103
- Process.wait
101
+ Process.kill("QUIT", pid)
104
102
  rescue Errno::ECHILD, Errno::ESRCH => e
105
103
  $stdout.puts "Process #{pid} has stopped"
106
104
  rescue Errno::ENOENT => e
@@ -0,0 +1,36 @@
1
+ $:.unshift File.expand_path("../../lib", __FILE__)
2
+
3
+ require 'magent'
4
+
5
+ class Findable
6
+ attr_accessor :id
7
+ def initialize(id)
8
+ @id = id
9
+ end
10
+
11
+ def self.find(id)
12
+ Findable.new(id)
13
+ end
14
+
15
+ def inspect
16
+ "Findable<#{@id}>"
17
+ end
18
+ end
19
+
20
+ class Manager
21
+ include Magent::Async
22
+
23
+ def self.process(findable)
24
+ puts ">>> Proccessing: #{findable.inspect}"
25
+ end
26
+ end
27
+
28
+ findable = Findable.new(19)
29
+ Manager.async(:queue1).process(findable)
30
+
31
+
32
+ # open the queue1 and process the messages
33
+ channel = Magent::AsyncChannel.new(:queue1)
34
+ processor = Magent::Processor.new(channel)
35
+
36
+ processor.run!(false)
@@ -0,0 +1,25 @@
1
+ $:.unshift File.expand_path("../../lib", __FILE__)
2
+
3
+ require 'magent'
4
+
5
+ class Worker
6
+ include Magent::Async
7
+
8
+ def self.work()
9
+ if rand < 0.95
10
+ puts "failed!"
11
+ raise "try again"
12
+ else
13
+ puts "OK!"
14
+ end
15
+ end
16
+ end
17
+ Worker.async(:queue1).work
18
+
19
+
20
+ # open the queue1 and process the messages
21
+ channel = Magent::AsyncChannel.new(:queue1)
22
+ processor = Magent::Processor.new(channel)
23
+
24
+ processor.run!(true)
25
+
@@ -0,0 +1,20 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ $:.unshift File.expand_path("../../lib", __FILE__)
4
+
5
+ require 'magent'
6
+
7
+ class MyClass
8
+ include Magent::Async
9
+
10
+ def foo
11
+ puts "bar"
12
+ end
13
+ end
14
+
15
+ Magent.sync_mode = true
16
+
17
+ c = MyClass.new
18
+ c.async.foo
19
+
20
+
@@ -11,18 +11,16 @@ require 'magent/failure'
11
11
 
12
12
  require 'magent/utils'
13
13
  require 'magent/generic_channel'
14
- require 'magent/actor_channel'
15
- require 'magent/push'
16
- require 'magent/actor'
14
+
17
15
  require 'magent/processor'
18
16
 
19
17
  require 'magent/async'
20
18
  require 'magent/async_channel'
21
19
 
22
- require 'magent/railtie' if defined?(Rails) && Rails.version >= "3.0.0"
23
-
24
- if defined?(EventMachine::WebSocket)
25
- require 'magent/web_socket_server'
20
+ if defined?(Rails) && Rails.version >= "3.0.0"
21
+ require 'magent/railtie'
22
+ elsif defined?(Rake)
23
+ require 'magent/tasks'
26
24
  end
27
25
 
28
26
  module Magent
@@ -62,6 +60,14 @@ module Magent
62
60
  @@config = config
63
61
  end
64
62
 
63
+ def self.sync_mode
64
+ @@sync_mode ||= false
65
+ end
66
+
67
+ def self.sync_mode=(m)
68
+ @@sync_mode = m
69
+ end
70
+
65
71
  def self.connect(environment, options={})
66
72
  raise 'Set config before connecting. Magent.config = {...}' if config.nil? || config.empty?
67
73
 
@@ -9,9 +9,9 @@
9
9
  # end
10
10
  #
11
11
  # async call:
12
- # Processor.find(id).async(:my_queue).process.commit!(priority)
12
+ # Processor.find(id).async(:my_queue, priority).process
13
13
  # chained methods:
14
- # Processor.async(:my_queue, true).find(1).process.commit!(priority)
14
+ # Processor.async(:my_queue, priority).find(1).process
15
15
  #
16
16
 
17
17
  module Magent
@@ -26,39 +26,36 @@ module Magent
26
26
  class Proxy
27
27
  instance_methods.each { |m| undef_method m unless m =~ /(^__|^nil\?$|^send$|proxy_|^object_id$)/ }
28
28
 
29
- def initialize(queue, target, test = false)
29
+ def initialize(queue, target, priority = 3)
30
30
  @queue = queue
31
- @method_chain = []
32
31
  @target = target
33
- @test = test
32
+ @priority = priority
34
33
 
35
34
  @channel = Magent::AsyncChannel.new(@queue)
36
35
  end
37
36
 
38
- def commit!(priority = 3)
39
- @channel.push(@target, @method_chain, priority)
40
-
41
- if @test
42
- target = @target
43
- @method_chain.each do |c|
44
- target = target.send(c[0], *c[1])
45
- end
46
-
47
- target
37
+ def commit!(method_name, args)
38
+ if Magent.sync_mode
39
+ @target.send(method_name, *args)
40
+ else
41
+ @channel.push(@target, [method_name, args], @priority)
48
42
  end
43
+
44
+ @target
49
45
  end
50
46
 
51
47
  def method_missing(m, *args, &blk)
52
48
  raise ArgumentError, "ruby blocks are not supported yet" if !blk.nil?
53
- @method_chain << [m, args]
49
+
50
+ commit!(m, args)
54
51
  self
55
52
  end
56
53
  end
57
54
 
58
55
  module Methods
59
- # @question.async(:judge).on_view_question.commit!(1)
60
- def async(queue = :default, test = false)
61
- Magent::Async::Proxy.new(queue, self, test)
56
+ # @question.async(:judge).on_view_question
57
+ def async(queue = :default, priority = 3)
58
+ Magent::Async::Proxy.new(queue, self, priority)
62
59
  end
63
60
  end
64
61
  end # Async
@@ -1,38 +1,30 @@
1
1
  module Magent
2
2
  class AsyncChannel < GenericChannel
3
- def push(target, method_chain, priority)
3
+ def push(target, method, priority)
4
+ encode_args(method[1])
5
+
4
6
  if target.kind_of?(Class)
5
- enqueue([target.to_s, nil, method_chain], priority)
7
+ enqueue([target.to_s, nil, method], priority)
6
8
  elsif target.class.respond_to?(:find) && target.respond_to?(:id)
7
- enqueue([target.class.to_s, target.id, method_chain], priority)
9
+ enqueue([target.class.to_s, target.id, method], priority)
8
10
  else
9
11
  raise ArgumentError, "I don't know how to handle #{target.inspect}"
10
12
  end
11
13
  end
12
14
 
13
15
  def process!(message)
14
- klass, id, method_chain = message
16
+ klass, id, method = message
15
17
 
16
- return false if method_chain.nil?
18
+ return false if method.nil?
17
19
 
18
- target = resolve_target(klass, id)
20
+ target = async_find(klass, id)
21
+ args = resolve_args(method[1])
19
22
 
20
- method_chain.each do |c|
21
- target = target.send(c[0], *c[1])
22
- end
23
+ target = target.send(method[0], *args)
23
24
 
24
25
  true
25
26
  end
26
27
 
27
28
  private
28
- def resolve_target(klass, id)
29
- klass = Object.module_eval(klass) if klass.kind_of?(String)
30
-
31
- if id
32
- klass.find(id)
33
- else
34
- klass
35
- end
36
- end
37
29
  end # AsyncChannel
38
30
  end
@@ -1,7 +1,12 @@
1
1
  module Magent
2
2
  module Failure
3
3
  def failed(info)
4
- error_collection.save(info.merge({:_id => generate_uid, :channel => @name, :channel_class => self.class.to_s, :created_at => Time.now.utc}))
4
+ error_collection.save(info.merge({
5
+ :_id => generate_uid,
6
+ :channel => @name,
7
+ :channel_class => self.class.to_s,
8
+ :created_at => Time.now.utc
9
+ }))
5
10
  end
6
11
 
7
12
  def error_count
@@ -22,8 +27,12 @@ module Magent
22
27
  end
23
28
 
24
29
  def retry_error(error)
25
- process!(error["message"])
26
- remove_error(error["_id"])
30
+ begin
31
+ process!(error["message"])
32
+ remove_error(error["_id"])
33
+ rescue => e
34
+ $stderr.puts "Failed to retry error #{error['_id']}: #{e.message}"
35
+ end
27
36
  end
28
37
 
29
38
  def enqueue_error(error)
@@ -11,7 +11,7 @@ module Magent
11
11
  end
12
12
 
13
13
  def enqueue(message, priority = 3)
14
- collection.save({:_id => generate_uid, :message => message, :priority => priority, :created_at => Time.now.to_i})
14
+ collection.save({:_id => generate_uid, :message => message, :priority => priority, :created_at => Time.now.to_i, :retries => 0})
15
15
  end
16
16
 
17
17
  def message_count
@@ -37,9 +37,54 @@ module Magent
37
37
  @collection ||= Magent.database.collection(@name)
38
38
  end
39
39
 
40
+ def retry_current_job
41
+ return false if !@current_job
42
+
43
+ @current_job['retries'] ||= 0
44
+ if @current_job['retries'] < 20
45
+ @current_job['retries'] += 1
46
+ collection.save(@current_job)
47
+ true
48
+ else
49
+ false
50
+ end
51
+ end
52
+
40
53
  protected
41
54
  def generate_uid
42
55
  UUIDTools::UUID.random_create.hexdigest
43
56
  end
57
+
58
+ def async_find(klass, id)
59
+ klass = Object.module_eval(klass) if klass.kind_of?(String)
60
+
61
+ if id
62
+ klass.find(id)
63
+ else
64
+ klass
65
+ end
66
+ end
67
+
68
+ def encode_args(args)
69
+ args.map! do |arg|
70
+ if arg.class.respond_to?(:find) && arg.respond_to?(:id)
71
+ {'async_find' => [arg.class.to_s, arg.id]}
72
+ elsif arg.kind_of?(Class)
73
+ {'async_find' => [arg.to_s, nil]}
74
+ else
75
+ arg
76
+ end
77
+ end
78
+ end
79
+
80
+ def resolve_args(args)
81
+ args.map do |arg|
82
+ if arg.kind_of?(Hash) && arg.include?('async_find')
83
+ async_find(*arg['async_find'])
84
+ else
85
+ arg
86
+ end
87
+ end
88
+ end
44
89
  end # GenericChannel
45
90
  end
@@ -6,15 +6,9 @@ module Magent
6
6
  @identity = identity
7
7
 
8
8
  @channel.on_start(identity)
9
-
10
- # @actor.class.actions.each do |action|
11
- # if !@actor.respond_to?(action)
12
- # raise ArgumentError, "action '#{action}' is not defined"
13
- # end
14
- # end
15
9
  end
16
10
 
17
- def run!
11
+ def run!(service = true)
18
12
  processed_messages = 0
19
13
  delay = 0
20
14
 
@@ -43,19 +37,25 @@ module Magent
43
37
  end
44
38
  rescue SystemExit
45
39
  rescue Exception => e
46
- $stderr.puts "Error processing #{message.inspect} => #{e.message}"
40
+ $stderr.puts "Error processing #{message.inspect} => #{e.message}.\n#{e.backtrace.join("\n\t")}"
47
41
  @channel.on_job_failed(@identity)
48
- @channel.failed(:error => e.message, :message => message, :backtrace => e.backtrace, :date => Time.now.utc)
42
+
43
+ if !@channel.retry_current_job
44
+ $stderr.puts "Cannot retry job because it has been retried #{@channel.current_job['retries']} times"
45
+ @channel.failed(:error => e.message, :message => message, :backtrace => e.backtrace, :date => Time.now.utc)
46
+ end
49
47
  ensure
50
48
  end
51
49
 
50
+ break if !service
51
+
52
52
  sleep (delay*100.0).to_i/100.0
53
53
  end
54
54
  end
55
55
 
56
56
  def shutdown!
57
57
  @shutdown = true
58
- @channel.on_quit
58
+ @channel.on_quit(@identity)
59
59
 
60
60
  @channel.on_shutdown if @channel.respond_to?(:on_shutdown)
61
61
  $stderr.puts "Shutting down..."