thirtysixthspan-magent 0.4.1
Sign up to get free protection for your applications and to get access to all the features.
- data/History.txt +4 -0
- data/Manifest.txt +22 -0
- data/PostInstall.txt +0 -0
- data/README.rdoc +53 -0
- data/Rakefile +27 -0
- data/bin/magent +139 -0
- data/examples/comm/run.rb +22 -0
- data/examples/comm/worker.rb +30 -0
- data/examples/error/error.rb +33 -0
- data/examples/simple/bot.rb +37 -0
- data/examples/stats/stats.rb +27 -0
- data/lib/magent.rb +57 -0
- data/lib/magent/actor.rb +78 -0
- data/lib/magent/channel.rb +37 -0
- data/lib/magent/generic_channel.rb +43 -0
- data/lib/magent/processor.rb +68 -0
- data/lib/magent/push.rb +13 -0
- data/lib/magent/utils.rb +13 -0
- data/magent.gemspec +46 -0
- data/script/console +9 -0
- data/test/test_helper.rb +3 -0
- data/test/test_magent.rb +11 -0
- metadata +149 -0
data/History.txt
ADDED
data/Manifest.txt
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
History.txt
|
2
|
+
Manifest.txt
|
3
|
+
PostInstall.txt
|
4
|
+
README.rdoc
|
5
|
+
Rakefile
|
6
|
+
bin/magent
|
7
|
+
examples/comm/run.rb
|
8
|
+
examples/comm/worker.rb
|
9
|
+
examples/error/error.rb
|
10
|
+
examples/simple/bot.rb
|
11
|
+
examples/stats/stats.rb
|
12
|
+
lib/magent.rb
|
13
|
+
lib/magent/actor.rb
|
14
|
+
lib/magent/channel.rb
|
15
|
+
lib/magent/generic_channel.rb
|
16
|
+
lib/magent/processor.rb
|
17
|
+
lib/magent/push.rb
|
18
|
+
lib/magent/utils.rb
|
19
|
+
magent.gemspec
|
20
|
+
script/console
|
21
|
+
test/test_helper.rb
|
22
|
+
test/test_magent.rb
|
data/PostInstall.txt
ADDED
File without changes
|
data/README.rdoc
ADDED
@@ -0,0 +1,53 @@
|
|
1
|
+
= magent
|
2
|
+
|
3
|
+
* http://github.com/dcu/magent
|
4
|
+
|
5
|
+
== DESCRIPTION:
|
6
|
+
|
7
|
+
Simple job queue system based on mongodb
|
8
|
+
|
9
|
+
== FEATURES/PROBLEMS:
|
10
|
+
|
11
|
+
* fast
|
12
|
+
* simple
|
13
|
+
* scalable
|
14
|
+
|
15
|
+
== SYNOPSIS:
|
16
|
+
|
17
|
+
see examples/
|
18
|
+
|
19
|
+
== REQUIREMENTS:
|
20
|
+
|
21
|
+
* mongodb >= 1.4
|
22
|
+
* mongo >= 1.0.0 (gem install mongo)
|
23
|
+
|
24
|
+
== INSTALL:
|
25
|
+
|
26
|
+
* rake gem
|
27
|
+
* sudo gem install pkg/*.gem
|
28
|
+
|
29
|
+
== LICENSE:
|
30
|
+
|
31
|
+
(The MIT License)
|
32
|
+
|
33
|
+
Copyright (c) 2009 David Cuadrado
|
34
|
+
|
35
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
36
|
+
a copy of this software and associated documentation files (the
|
37
|
+
'Software'), to deal in the Software without restriction, including
|
38
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
39
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
40
|
+
permit persons to whom the Software is furnished to do so, subject to
|
41
|
+
the following conditions:
|
42
|
+
|
43
|
+
The above copyright notice and this permission notice shall be
|
44
|
+
included in all copies or substantial portions of the Software.
|
45
|
+
|
46
|
+
THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
|
47
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
48
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
49
|
+
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
50
|
+
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
51
|
+
TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
52
|
+
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
53
|
+
|
data/Rakefile
ADDED
@@ -0,0 +1,27 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
gem 'hoe', '>= 2.1.0'
|
3
|
+
require 'hoe'
|
4
|
+
require 'fileutils'
|
5
|
+
require './lib/magent'
|
6
|
+
|
7
|
+
Hoe.plugin :newgem
|
8
|
+
# Hoe.plugin :website
|
9
|
+
# Hoe.plugin :cucumberfeatures
|
10
|
+
|
11
|
+
# Generate all the Rake tasks
|
12
|
+
# Run 'rake -T' to see list of generated tasks (from gem root directory)
|
13
|
+
$hoe = Hoe.spec 'magent' do
|
14
|
+
self.developer 'David Cuadrado', 'krawek@gmail.com'
|
15
|
+
self.post_install_message = ''
|
16
|
+
self.rubyforge_name = self.name
|
17
|
+
self.extra_deps = [['mongo','>= 0.1.0'],
|
18
|
+
['uuidtools', '>= 2.0.0']]
|
19
|
+
end
|
20
|
+
|
21
|
+
require 'newgem/tasks'
|
22
|
+
Dir['tasks/**/*.rake'].each { |t| load t }
|
23
|
+
|
24
|
+
# TODO - want other tests/tasks run by default? Add them to the list
|
25
|
+
# remove_task :default
|
26
|
+
# task :default => [:spec, :features]
|
27
|
+
|
data/bin/magent
ADDED
@@ -0,0 +1,139 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
$:.unshift File.dirname(__FILE__)+"/../lib/"
|
4
|
+
require 'magent'
|
5
|
+
require 'optparse'
|
6
|
+
require 'fileutils'
|
7
|
+
|
8
|
+
def usage(option_parser, error = nil)
|
9
|
+
$stderr.puts error if error
|
10
|
+
$stderr.puts option_parser
|
11
|
+
exit 1
|
12
|
+
end
|
13
|
+
|
14
|
+
options = {}
|
15
|
+
|
16
|
+
optparser = OptionParser.new do |opts|
|
17
|
+
opts.banner = "Usage: #{$0} [options] <start|stop|restart>\n\nExample: magent -t carl1 -a /path/to/agent.rb -d -P /tmp restart\n\nOptions:"
|
18
|
+
|
19
|
+
opts.on("-a PATH", "--agent=PATH", "Path to agent") do |o|
|
20
|
+
options[:agent] = o
|
21
|
+
end
|
22
|
+
|
23
|
+
opts.on("-t ID", "--identifier=ID", "Identifier") do |o|
|
24
|
+
options[:identifier] = o
|
25
|
+
end
|
26
|
+
|
27
|
+
opts.on("-d", "--daemonize", "Run agent as a daemon") do |o|
|
28
|
+
options[:daemonize] = true
|
29
|
+
end
|
30
|
+
|
31
|
+
opts.on("-l", "--log-path=PATH", "Log path") do |o|
|
32
|
+
options[:log_path] = o
|
33
|
+
end
|
34
|
+
|
35
|
+
opts.on("-P", "--piddir=PATH", "PID dir to use (if daemonized)", "Default: #{options[:piddir]}") do |o|
|
36
|
+
options[:piddir] = o
|
37
|
+
end
|
38
|
+
|
39
|
+
opts.on_tail("-h", "--help", "Show this help message.") do
|
40
|
+
puts opts
|
41
|
+
exit
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
args = []
|
46
|
+
begin
|
47
|
+
args = optparser.parse!
|
48
|
+
rescue => e
|
49
|
+
$stderr.puts e
|
50
|
+
$stderr.puts optparser
|
51
|
+
exit 0
|
52
|
+
end
|
53
|
+
|
54
|
+
if !options[:agent]
|
55
|
+
usage(optparser, "Error: --agent is required")
|
56
|
+
end
|
57
|
+
|
58
|
+
load options[:agent]
|
59
|
+
|
60
|
+
if Magent.current_actor.nil?
|
61
|
+
usage(optparser, "Use Magent.register(YourActor.new) to register an actor")
|
62
|
+
end
|
63
|
+
|
64
|
+
class Controller
|
65
|
+
def initialize(actor, opts)
|
66
|
+
@options = opts
|
67
|
+
@actor = actor
|
68
|
+
|
69
|
+
@options[:log_path] ||= Dir.getwd
|
70
|
+
|
71
|
+
@identity = @options[:identifier] || Magent::Utils.underscore(actor.class.to_s)
|
72
|
+
@identity << "-#{Socket.gethostname.split('.')[0]}"
|
73
|
+
end
|
74
|
+
|
75
|
+
def start
|
76
|
+
if @options[:daemonize] && @options[:piddir]
|
77
|
+
run_as_daemon
|
78
|
+
else
|
79
|
+
Magent::Processor.new(@actor).run!
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
def stop
|
84
|
+
begin
|
85
|
+
pid = File.read(pid_file).to_i
|
86
|
+
Process.kill("TERM", pid)
|
87
|
+
Process.kill(0, pid)
|
88
|
+
Process.wait
|
89
|
+
rescue Errno::ECHILD, Errno::ESRCH => e
|
90
|
+
$stdout.puts "Process #{pid} has stopped"
|
91
|
+
rescue Errno::ENOENT => e
|
92
|
+
$stdout.puts "Warning: #{e}"
|
93
|
+
ensure
|
94
|
+
File.unlink(pid_file) if File.exist?(pid_file)
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
def restart
|
99
|
+
begin
|
100
|
+
stop
|
101
|
+
rescue => e
|
102
|
+
$stderr.puts "Warning: #{e}"
|
103
|
+
end
|
104
|
+
start
|
105
|
+
end
|
106
|
+
|
107
|
+
private
|
108
|
+
def run_as_daemon
|
109
|
+
daemonize
|
110
|
+
|
111
|
+
FileUtils.mkpath(@options[:piddir])
|
112
|
+
if @options[:piddir]
|
113
|
+
File.open(pid_file, "w") do |f|
|
114
|
+
f.write(Process.pid)
|
115
|
+
end
|
116
|
+
end
|
117
|
+
|
118
|
+
Magent::Processor.new(@actor).run!
|
119
|
+
end
|
120
|
+
|
121
|
+
def pid_file
|
122
|
+
@pid_file ||= File.join(@options[:piddir], "magent.#{@identity}.pid")
|
123
|
+
end
|
124
|
+
|
125
|
+
def daemonize
|
126
|
+
exit if fork
|
127
|
+
Process.setsid
|
128
|
+
exit if fork
|
129
|
+
|
130
|
+
STDIN.reopen "/dev/null"
|
131
|
+
STDOUT.reopen "#{@options[:log_path]}/magent.#{@identity}.out", "a"
|
132
|
+
STDERR.reopen "#{@options[:log_path]}/magent.#{@identity}.err", "a"
|
133
|
+
|
134
|
+
STDERR.sync = true
|
135
|
+
STDOUT.sync = true
|
136
|
+
end
|
137
|
+
end
|
138
|
+
|
139
|
+
Controller.new(Magent.current_actor, options).send(args.shift)
|
@@ -0,0 +1,22 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
$:.unshift File.dirname(__FILE__)+"/../../lib/"
|
4
|
+
require 'magent'
|
5
|
+
|
6
|
+
id = "#{rand(16)}#{rand(16)}#{rand(16)}#{rand(16)}"
|
7
|
+
|
8
|
+
values = (1..5).to_a.map { rand(10) }
|
9
|
+
puts values.join(" + ")
|
10
|
+
Magent.push("workers", :sum, id, *values)
|
11
|
+
|
12
|
+
channel = Magent::GenericChannel.new("+#{id}")
|
13
|
+
|
14
|
+
loop do
|
15
|
+
v = channel.dequeue;
|
16
|
+
if v
|
17
|
+
$stdout.puts v.inspect
|
18
|
+
break
|
19
|
+
end
|
20
|
+
sleep 0.1
|
21
|
+
end
|
22
|
+
|
@@ -0,0 +1,30 @@
|
|
1
|
+
$:.unshift File.dirname(__FILE__)+"/../../lib/"
|
2
|
+
require 'magent'
|
3
|
+
|
4
|
+
# Use: magent /path/to/this/file
|
5
|
+
|
6
|
+
class Worker
|
7
|
+
include Magent::Actor
|
8
|
+
channel_name "workers"
|
9
|
+
expose :sum
|
10
|
+
|
11
|
+
def sum(payload)
|
12
|
+
id, *args = payload
|
13
|
+
|
14
|
+
s = args.inject(0) { |v, a| a += v }
|
15
|
+
send_to_client(id, {:method => :sum, :result => s})
|
16
|
+
end
|
17
|
+
|
18
|
+
private
|
19
|
+
def send_to_client(id, message)
|
20
|
+
c = Magent::GenericChannel.new("+#{id}")
|
21
|
+
c.enqueue(message)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
Magent.register(Worker.new)
|
26
|
+
|
27
|
+
if $0 == __FILE__
|
28
|
+
Magent::Processor.new(Magent.current_actor).run!
|
29
|
+
end
|
30
|
+
|
@@ -0,0 +1,33 @@
|
|
1
|
+
$:.unshift File.dirname(__FILE__)+"/../../lib/"
|
2
|
+
require 'magent'
|
3
|
+
|
4
|
+
Magent.push("errors", :fail, "this is a fail")
|
5
|
+
|
6
|
+
class Error
|
7
|
+
include Magent::Actor
|
8
|
+
|
9
|
+
channel_name "errors"
|
10
|
+
expose :fail
|
11
|
+
|
12
|
+
def fail(payload)
|
13
|
+
@count ||= 0
|
14
|
+
errors = self.class.channel.errors
|
15
|
+
|
16
|
+
errors.each do |error|
|
17
|
+
@count += 1
|
18
|
+
$stderr.puts "Retrying: #{error["method"]}(#{error["payload"].inspect})"
|
19
|
+
self.class.channel.retry_error(error)
|
20
|
+
end
|
21
|
+
|
22
|
+
if @count == 0
|
23
|
+
raise payload.inspect
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
Magent.register(Error.new)
|
29
|
+
|
30
|
+
if $0 == __FILE__
|
31
|
+
Magent::Processor.new(Magent.current_actor).run!
|
32
|
+
end
|
33
|
+
|
@@ -0,0 +1,37 @@
|
|
1
|
+
$:.unshift File.dirname(__FILE__)+"/../../lib/"
|
2
|
+
require 'magent'
|
3
|
+
|
4
|
+
# Use: magent /path/to/this/file
|
5
|
+
|
6
|
+
Magent.push("bots", :echo, "hello, world")
|
7
|
+
Magent.push("bots", :do_task, "File", :exist?, "/etc/passwd")
|
8
|
+
Magent.push("bots", :echo, "Press ctrl+c to close")
|
9
|
+
Magent.push("bots", :do_not_exist, "you should not see this message")
|
10
|
+
|
11
|
+
class Bot
|
12
|
+
include Magent::Actor
|
13
|
+
channel_name "bots"
|
14
|
+
expose :echo, :do_task
|
15
|
+
|
16
|
+
def echo(payload)
|
17
|
+
$stderr.puts payload.inspect
|
18
|
+
end
|
19
|
+
|
20
|
+
def do_task(payload)
|
21
|
+
klass, *args = payload
|
22
|
+
|
23
|
+
result = Object.module_eval(klass).send(*args)
|
24
|
+
$stderr.puts "RESULT: #{result}"
|
25
|
+
end
|
26
|
+
|
27
|
+
at_least_every 15 do
|
28
|
+
puts "Hi there!, you'll see this message again in ~15 seconds"
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
Magent.register(Bot.new)
|
33
|
+
|
34
|
+
if $0 == __FILE__
|
35
|
+
Magent::Processor.new(Magent.current_actor).run!
|
36
|
+
end
|
37
|
+
|
@@ -0,0 +1,27 @@
|
|
1
|
+
$:.unshift File.dirname(__FILE__)+"/../../lib/"
|
2
|
+
require 'magent'
|
3
|
+
|
4
|
+
Magent.push("stats", :calc)
|
5
|
+
Magent.push("stats", :calc)
|
6
|
+
Magent.push("stats", :calc)
|
7
|
+
Magent.push("stats", :calc)
|
8
|
+
|
9
|
+
class Stats
|
10
|
+
include Magent::Actor
|
11
|
+
|
12
|
+
channel_name "stats"
|
13
|
+
expose :calc
|
14
|
+
|
15
|
+
def calc(payload)
|
16
|
+
$stderr.puts "messages in queue: #{self.class.channel.queue_count}"
|
17
|
+
$stderr.puts "total messages count: #{self.class.channel.message_count}"
|
18
|
+
$stderr.puts "total errors: #{self.class.channel.error_count}"
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
Magent.register(Stats.new)
|
23
|
+
|
24
|
+
if $0 == __FILE__
|
25
|
+
Magent::Processor.new(Magent.current_actor).run!
|
26
|
+
end
|
27
|
+
|
data/lib/magent.rb
ADDED
@@ -0,0 +1,57 @@
|
|
1
|
+
$:.unshift(File.dirname(__FILE__)) unless
|
2
|
+
$:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
|
3
|
+
|
4
|
+
require 'mongo'
|
5
|
+
require 'set'
|
6
|
+
require 'uuidtools'
|
7
|
+
|
8
|
+
require 'magent/utils'
|
9
|
+
require 'magent/generic_channel'
|
10
|
+
require 'magent/channel'
|
11
|
+
require 'magent/push'
|
12
|
+
require 'magent/actor'
|
13
|
+
require 'magent/processor'
|
14
|
+
|
15
|
+
module Magent
|
16
|
+
VERSION = '0.4.1'
|
17
|
+
|
18
|
+
@@db_name = 'magent'
|
19
|
+
@@host = 'localhost'
|
20
|
+
@@port = '27017'
|
21
|
+
@@username = ''
|
22
|
+
@@password = ''
|
23
|
+
|
24
|
+
def self.host(host,port)
|
25
|
+
@@host = host
|
26
|
+
@@port = port
|
27
|
+
end
|
28
|
+
|
29
|
+
def self.auth(username,password)
|
30
|
+
@@username = username
|
31
|
+
@@password = password
|
32
|
+
end
|
33
|
+
|
34
|
+
def self.db_name(db_name)
|
35
|
+
@@db_name = db_name
|
36
|
+
end
|
37
|
+
|
38
|
+
def self.connection
|
39
|
+
return @@connection if defined? @@connection
|
40
|
+
@@connection = Mongo::Connection.new(@@host, @@port, :auto_reconnect => true)
|
41
|
+
@@connection.add_auth(@@db_name, @@username, @@password) if @@username!='' && @@password!=''
|
42
|
+
return @@connection
|
43
|
+
end
|
44
|
+
|
45
|
+
def self.connection=(new_connection)
|
46
|
+
@@connection = new_connection
|
47
|
+
end
|
48
|
+
|
49
|
+
def self.database=(name)
|
50
|
+
@@database = Magent.connection.db(name)
|
51
|
+
end
|
52
|
+
|
53
|
+
def self.database
|
54
|
+
@@database ||= Magent.connection.db(@@db_name)
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
data/lib/magent/actor.rb
ADDED
@@ -0,0 +1,78 @@
|
|
1
|
+
module Magent
|
2
|
+
module Actor
|
3
|
+
def self.included(klass)
|
4
|
+
klass.class_eval do
|
5
|
+
extend Actor::ClassMethods
|
6
|
+
include Actor::InstanceMethods
|
7
|
+
end
|
8
|
+
end
|
9
|
+
|
10
|
+
module ClassMethods
|
11
|
+
def expose(*methods)
|
12
|
+
methods.each do |m|
|
13
|
+
actions << m.to_s
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
def channel_name(name = nil)
|
18
|
+
@channel_name ||= (name || Magent::Utils.underscore(self.name)).to_s
|
19
|
+
end
|
20
|
+
|
21
|
+
def actions
|
22
|
+
@actions ||= Set.new
|
23
|
+
end
|
24
|
+
|
25
|
+
def can_handle?(action)
|
26
|
+
actions.include?(action.to_s)
|
27
|
+
end
|
28
|
+
|
29
|
+
def channel
|
30
|
+
@channel ||= begin
|
31
|
+
Channel.new(self.channel_name)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def tasks
|
36
|
+
@tasks ||= []
|
37
|
+
end
|
38
|
+
|
39
|
+
def at_least_every(seconds, &block)
|
40
|
+
tasks << {:every => seconds, :last_time => Time.now, :block => block}
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
module InstanceMethods
|
45
|
+
def _run_tasks
|
46
|
+
tasks = self.class.tasks
|
47
|
+
|
48
|
+
return false if tasks.empty?
|
49
|
+
performed = false
|
50
|
+
|
51
|
+
tasks.each do |task|
|
52
|
+
delta = Time.now - task[:last_time]
|
53
|
+
|
54
|
+
if delta >= task[:every]
|
55
|
+
task[:last_time] = Time.now
|
56
|
+
begin
|
57
|
+
instance_eval(&task[:block])
|
58
|
+
rescue Exception => e
|
59
|
+
$stderr.puts "Failed periodical task: #{e.message}"
|
60
|
+
$stderr.puts e.backtrace.join("\n\t")
|
61
|
+
end
|
62
|
+
performed = true
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
performed
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end # Actor
|
70
|
+
|
71
|
+
def self.register(actor)
|
72
|
+
@current_actor = actor
|
73
|
+
end
|
74
|
+
|
75
|
+
def self.current_actor
|
76
|
+
@current_actor
|
77
|
+
end
|
78
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
module Magent
|
2
|
+
class Channel < GenericChannel
|
3
|
+
def enqueue(message, args)
|
4
|
+
super([message, args])
|
5
|
+
end
|
6
|
+
|
7
|
+
def failed(info)
|
8
|
+
error_collection.save(info.merge({:_id => generate_uid, :channel => @name, :created_at => Time.now.utc}))
|
9
|
+
end
|
10
|
+
|
11
|
+
def error_count
|
12
|
+
error_collection.count()
|
13
|
+
end
|
14
|
+
|
15
|
+
def errors(conds = {})
|
16
|
+
page = conds.delete(:page) || 1
|
17
|
+
per_page = conds.delete(:per_page) || 10
|
18
|
+
|
19
|
+
error_collection.find({}, {:skip => (page-1)*per_page,
|
20
|
+
:limit => per_page,
|
21
|
+
:sort => [["created_at", -1]]})
|
22
|
+
end
|
23
|
+
|
24
|
+
def remove_error(error_id)
|
25
|
+
self.error_collection.remove(:_id => error_id)
|
26
|
+
end
|
27
|
+
|
28
|
+
def retry_error(error)
|
29
|
+
remove_error(error["_id"])
|
30
|
+
enqueue(error["method"], error["payload"])
|
31
|
+
end
|
32
|
+
|
33
|
+
def error_collection
|
34
|
+
@error_collection ||= Magent.database.collection("#{@name}-errors")
|
35
|
+
end
|
36
|
+
end # Channel
|
37
|
+
end
|
@@ -0,0 +1,43 @@
|
|
1
|
+
module Magent
|
2
|
+
class GenericChannel
|
3
|
+
attr_reader :name
|
4
|
+
|
5
|
+
def initialize(name)
|
6
|
+
@name = name
|
7
|
+
end
|
8
|
+
|
9
|
+
def enqueue(message)
|
10
|
+
collection.save({:_id => generate_uid, :message => message, :priority => 3, :created_at => Time.now.to_i})
|
11
|
+
end
|
12
|
+
|
13
|
+
def message_count
|
14
|
+
collection.count # TODO: number of processed messages (create a collection for stats)
|
15
|
+
end
|
16
|
+
|
17
|
+
def queue_count
|
18
|
+
collection.count
|
19
|
+
end
|
20
|
+
|
21
|
+
def dequeue
|
22
|
+
if m = self.next_message
|
23
|
+
m["message"]
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def next_message
|
28
|
+
Magent.database.command(BSON::OrderedHash[:findandmodify, @name,
|
29
|
+
:sort, [{:priority => -1}, {:created_at => 1}],
|
30
|
+
:remove, true
|
31
|
+
])["value"]
|
32
|
+
end
|
33
|
+
|
34
|
+
def collection
|
35
|
+
@collection ||= Magent.database.collection(@name)
|
36
|
+
end
|
37
|
+
|
38
|
+
protected
|
39
|
+
def generate_uid
|
40
|
+
UUIDTools::UUID.random_create.hexdigest
|
41
|
+
end
|
42
|
+
end # GenericChannel
|
43
|
+
end
|
@@ -0,0 +1,68 @@
|
|
1
|
+
module Magent
|
2
|
+
class Processor
|
3
|
+
attr_reader :actor
|
4
|
+
|
5
|
+
def initialize(actor)
|
6
|
+
@actor = actor
|
7
|
+
@shutdown = false
|
8
|
+
|
9
|
+
@actor.class.actions.each do |action|
|
10
|
+
if !@actor.respond_to?(action)
|
11
|
+
raise ArgumentError, "action '#{action}' is not defined"
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
def run!
|
17
|
+
processed_messages = 0
|
18
|
+
delay = 0
|
19
|
+
|
20
|
+
trap('TERM') { shutdown!; exit 0 }
|
21
|
+
trap('SIGINT') { shutdown!; exit 0 }
|
22
|
+
|
23
|
+
loop do
|
24
|
+
break if @shutdown
|
25
|
+
|
26
|
+
delay = 0 if @actor._run_tasks
|
27
|
+
|
28
|
+
@method, @payload = @actor.class.channel.dequeue
|
29
|
+
|
30
|
+
if @method.nil?
|
31
|
+
delay += 0.1 if delay <= 5
|
32
|
+
else
|
33
|
+
delay = 0
|
34
|
+
$stderr.puts "#{@actor.class}##{@method}(#{@payload.inspect})"
|
35
|
+
begin
|
36
|
+
if @actor.class.can_handle?(@method)
|
37
|
+
processed_messages += 1
|
38
|
+
@actor.send(@method, @payload)
|
39
|
+
|
40
|
+
if processed_messages > 20
|
41
|
+
processed_messages = 0
|
42
|
+
GC.start
|
43
|
+
end
|
44
|
+
else
|
45
|
+
$stderr.puts "Unknown action: #{@method} (payload=#{@payload.inspect})"
|
46
|
+
end
|
47
|
+
rescue SystemExit
|
48
|
+
rescue Exception => e
|
49
|
+
$stderr.puts "Error while executing #{@method.inspect} #{@payload.inspect}"
|
50
|
+
$stderr.puts "#{e.to_s}\n#{e.backtrace.join("\t\n")}"
|
51
|
+
@actor.class.channel.failed(:message => e.message, :method => @method, :payload => @payload, :backtrace => e.backtrace, :date => Time.now.utc)
|
52
|
+
ensure
|
53
|
+
@method, @payload = nil
|
54
|
+
end
|
55
|
+
end
|
56
|
+
sleep delay
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
def shutdown!
|
61
|
+
@shutdown = true
|
62
|
+
$stderr.puts "Shutting down..."
|
63
|
+
if @method
|
64
|
+
@actor.class.channel.enqueue(@method, @payload)
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end #Processor
|
68
|
+
end
|
data/lib/magent/push.rb
ADDED
data/lib/magent/utils.rb
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
module Magent
|
2
|
+
module Utils
|
3
|
+
def self.underscore(word)
|
4
|
+
word.to_s.gsub(/::/, '.').
|
5
|
+
gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
|
6
|
+
gsub(/([a-z\d])([A-Z])/,'\1_\2').tr("-", "_").downcase
|
7
|
+
end
|
8
|
+
|
9
|
+
def self.camelize(word)
|
10
|
+
word.to_s.gsub(/\.(.?)/) { "::#{$1.upcase}" }.gsub(/(?:^|_)(.)/) { $1.upcase }
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
data/magent.gemspec
ADDED
@@ -0,0 +1,46 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
|
3
|
+
Gem::Specification.new do |s|
|
4
|
+
s.name = %q{thirtysixthspan-magent}
|
5
|
+
s.version = "0.4.1"
|
6
|
+
|
7
|
+
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
8
|
+
s.authors = ["David Cuadrado"]
|
9
|
+
s.date = %q{2010-05-10}
|
10
|
+
s.default_executable = %q{magent}
|
11
|
+
s.description = %q{Simple job queue system based on mongodb}
|
12
|
+
s.email = ["krawek@gmail.com"]
|
13
|
+
s.executables = ["magent"]
|
14
|
+
s.extra_rdoc_files = ["History.txt", "Manifest.txt", "PostInstall.txt"]
|
15
|
+
s.files = ["History.txt", "Manifest.txt", "PostInstall.txt", "README.rdoc", "Rakefile", "bin/magent", "examples/comm/run.rb", "examples/comm/worker.rb", "examples/error/error.rb", "examples/simple/bot.rb", "examples/stats/stats.rb", "lib/magent.rb", "lib/magent/actor.rb", "lib/magent/channel.rb", "lib/magent/generic_channel.rb", "lib/magent/processor.rb", "lib/magent/push.rb", "lib/magent/utils.rb", "magent.gemspec", "script/console", "test/test_helper.rb", "test/test_magent.rb"]
|
16
|
+
s.homepage = %q{http://github.com/dcu/magent}
|
17
|
+
s.post_install_message = %q{}
|
18
|
+
s.rdoc_options = ["--main", "README.rdoc"]
|
19
|
+
s.require_paths = ["lib"]
|
20
|
+
s.rubyforge_project = %q{magent}
|
21
|
+
s.rubygems_version = %q{1.3.6}
|
22
|
+
s.summary = %q{Simple job queue system based on mongodb}
|
23
|
+
s.test_files = ["test/test_helper.rb", "test/test_magent.rb"]
|
24
|
+
|
25
|
+
if s.respond_to? :specification_version then
|
26
|
+
current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
|
27
|
+
s.specification_version = 3
|
28
|
+
|
29
|
+
if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
|
30
|
+
s.add_runtime_dependency(%q<mongo>, [">= 0.1.0"])
|
31
|
+
s.add_runtime_dependency(%q<uuidtools>, [">= 2.0.0"])
|
32
|
+
s.add_development_dependency(%q<rubyforge>, [">= 2.0.4"])
|
33
|
+
s.add_development_dependency(%q<hoe>, [">= 2.6.0"])
|
34
|
+
else
|
35
|
+
s.add_dependency(%q<mongo>, [">= 0.1.0"])
|
36
|
+
s.add_dependency(%q<uuidtools>, [">= 2.0.0"])
|
37
|
+
s.add_dependency(%q<rubyforge>, [">= 2.0.4"])
|
38
|
+
s.add_dependency(%q<hoe>, [">= 2.6.0"])
|
39
|
+
end
|
40
|
+
else
|
41
|
+
s.add_dependency(%q<mongo>, [">= 0.1.0"])
|
42
|
+
s.add_dependency(%q<uuidtools>, [">= 2.0.0"])
|
43
|
+
s.add_dependency(%q<rubyforge>, [">= 2.0.4"])
|
44
|
+
s.add_dependency(%q<hoe>, [">= 2.6.0"])
|
45
|
+
end
|
46
|
+
end
|
data/script/console
ADDED
@@ -0,0 +1,9 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
# File: script/console
|
3
|
+
irb = RUBY_PLATFORM =~ /(:?mswin|mingw)/ ? 'irb.bat' : 'irb'
|
4
|
+
|
5
|
+
libs = " -r irb/completion"
|
6
|
+
#libs << " -r #{File.dirname(__FILE__) + '/../lib/console_lib/init.rb'}"
|
7
|
+
libs << " -r #{File.dirname(__FILE__) + '/../lib/magent.rb'}"
|
8
|
+
puts "Loading magent gem"
|
9
|
+
exec "#{irb} #{libs} --simple-prompt"
|
data/test/test_helper.rb
ADDED
data/test/test_magent.rb
ADDED
metadata
ADDED
@@ -0,0 +1,149 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: thirtysixthspan-magent
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
prerelease: false
|
5
|
+
segments:
|
6
|
+
- 0
|
7
|
+
- 4
|
8
|
+
- 1
|
9
|
+
version: 0.4.1
|
10
|
+
platform: ruby
|
11
|
+
authors:
|
12
|
+
- David Cuadrado
|
13
|
+
autorequire:
|
14
|
+
bindir: bin
|
15
|
+
cert_chain: []
|
16
|
+
|
17
|
+
date: 2010-05-10 00:00:00 -05:00
|
18
|
+
default_executable: magent
|
19
|
+
dependencies:
|
20
|
+
- !ruby/object:Gem::Dependency
|
21
|
+
name: mongo
|
22
|
+
prerelease: false
|
23
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
24
|
+
none: false
|
25
|
+
requirements:
|
26
|
+
- - ">="
|
27
|
+
- !ruby/object:Gem::Version
|
28
|
+
segments:
|
29
|
+
- 0
|
30
|
+
- 1
|
31
|
+
- 0
|
32
|
+
version: 0.1.0
|
33
|
+
type: :runtime
|
34
|
+
version_requirements: *id001
|
35
|
+
- !ruby/object:Gem::Dependency
|
36
|
+
name: uuidtools
|
37
|
+
prerelease: false
|
38
|
+
requirement: &id002 !ruby/object:Gem::Requirement
|
39
|
+
none: false
|
40
|
+
requirements:
|
41
|
+
- - ">="
|
42
|
+
- !ruby/object:Gem::Version
|
43
|
+
segments:
|
44
|
+
- 2
|
45
|
+
- 0
|
46
|
+
- 0
|
47
|
+
version: 2.0.0
|
48
|
+
type: :runtime
|
49
|
+
version_requirements: *id002
|
50
|
+
- !ruby/object:Gem::Dependency
|
51
|
+
name: rubyforge
|
52
|
+
prerelease: false
|
53
|
+
requirement: &id003 !ruby/object:Gem::Requirement
|
54
|
+
none: false
|
55
|
+
requirements:
|
56
|
+
- - ">="
|
57
|
+
- !ruby/object:Gem::Version
|
58
|
+
segments:
|
59
|
+
- 2
|
60
|
+
- 0
|
61
|
+
- 4
|
62
|
+
version: 2.0.4
|
63
|
+
type: :development
|
64
|
+
version_requirements: *id003
|
65
|
+
- !ruby/object:Gem::Dependency
|
66
|
+
name: hoe
|
67
|
+
prerelease: false
|
68
|
+
requirement: &id004 !ruby/object:Gem::Requirement
|
69
|
+
none: false
|
70
|
+
requirements:
|
71
|
+
- - ">="
|
72
|
+
- !ruby/object:Gem::Version
|
73
|
+
segments:
|
74
|
+
- 2
|
75
|
+
- 6
|
76
|
+
- 0
|
77
|
+
version: 2.6.0
|
78
|
+
type: :development
|
79
|
+
version_requirements: *id004
|
80
|
+
description: Simple job queue system based on mongodb
|
81
|
+
email:
|
82
|
+
- krawek@gmail.com
|
83
|
+
executables:
|
84
|
+
- magent
|
85
|
+
extensions: []
|
86
|
+
|
87
|
+
extra_rdoc_files:
|
88
|
+
- History.txt
|
89
|
+
- Manifest.txt
|
90
|
+
- PostInstall.txt
|
91
|
+
files:
|
92
|
+
- History.txt
|
93
|
+
- Manifest.txt
|
94
|
+
- PostInstall.txt
|
95
|
+
- README.rdoc
|
96
|
+
- Rakefile
|
97
|
+
- bin/magent
|
98
|
+
- examples/comm/run.rb
|
99
|
+
- examples/comm/worker.rb
|
100
|
+
- examples/error/error.rb
|
101
|
+
- examples/simple/bot.rb
|
102
|
+
- examples/stats/stats.rb
|
103
|
+
- lib/magent.rb
|
104
|
+
- lib/magent/actor.rb
|
105
|
+
- lib/magent/channel.rb
|
106
|
+
- lib/magent/generic_channel.rb
|
107
|
+
- lib/magent/processor.rb
|
108
|
+
- lib/magent/push.rb
|
109
|
+
- lib/magent/utils.rb
|
110
|
+
- magent.gemspec
|
111
|
+
- script/console
|
112
|
+
- test/test_helper.rb
|
113
|
+
- test/test_magent.rb
|
114
|
+
has_rdoc: true
|
115
|
+
homepage: http://github.com/dcu/magent
|
116
|
+
licenses: []
|
117
|
+
|
118
|
+
post_install_message: ""
|
119
|
+
rdoc_options:
|
120
|
+
- --main
|
121
|
+
- README.rdoc
|
122
|
+
require_paths:
|
123
|
+
- lib
|
124
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
125
|
+
none: false
|
126
|
+
requirements:
|
127
|
+
- - ">="
|
128
|
+
- !ruby/object:Gem::Version
|
129
|
+
segments:
|
130
|
+
- 0
|
131
|
+
version: "0"
|
132
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
133
|
+
none: false
|
134
|
+
requirements:
|
135
|
+
- - ">="
|
136
|
+
- !ruby/object:Gem::Version
|
137
|
+
segments:
|
138
|
+
- 0
|
139
|
+
version: "0"
|
140
|
+
requirements: []
|
141
|
+
|
142
|
+
rubyforge_project: magent
|
143
|
+
rubygems_version: 1.3.7
|
144
|
+
signing_key:
|
145
|
+
specification_version: 3
|
146
|
+
summary: Simple job queue system based on mongodb
|
147
|
+
test_files:
|
148
|
+
- test/test_helper.rb
|
149
|
+
- test/test_magent.rb
|