thirtysixthspan-magent 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/History.txt +4 -0
- data/Manifest.txt +22 -0
- data/PostInstall.txt +0 -0
- data/README.rdoc +53 -0
- data/Rakefile +27 -0
- data/bin/magent +139 -0
- data/examples/comm/run.rb +22 -0
- data/examples/comm/worker.rb +30 -0
- data/examples/error/error.rb +33 -0
- data/examples/simple/bot.rb +37 -0
- data/examples/stats/stats.rb +27 -0
- data/lib/magent.rb +57 -0
- data/lib/magent/actor.rb +78 -0
- data/lib/magent/channel.rb +37 -0
- data/lib/magent/generic_channel.rb +43 -0
- data/lib/magent/processor.rb +68 -0
- data/lib/magent/push.rb +13 -0
- data/lib/magent/utils.rb +13 -0
- data/magent.gemspec +46 -0
- data/script/console +9 -0
- data/test/test_helper.rb +3 -0
- data/test/test_magent.rb +11 -0
- metadata +149 -0
data/History.txt
ADDED
data/Manifest.txt
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
History.txt
|
2
|
+
Manifest.txt
|
3
|
+
PostInstall.txt
|
4
|
+
README.rdoc
|
5
|
+
Rakefile
|
6
|
+
bin/magent
|
7
|
+
examples/comm/run.rb
|
8
|
+
examples/comm/worker.rb
|
9
|
+
examples/error/error.rb
|
10
|
+
examples/simple/bot.rb
|
11
|
+
examples/stats/stats.rb
|
12
|
+
lib/magent.rb
|
13
|
+
lib/magent/actor.rb
|
14
|
+
lib/magent/channel.rb
|
15
|
+
lib/magent/generic_channel.rb
|
16
|
+
lib/magent/processor.rb
|
17
|
+
lib/magent/push.rb
|
18
|
+
lib/magent/utils.rb
|
19
|
+
magent.gemspec
|
20
|
+
script/console
|
21
|
+
test/test_helper.rb
|
22
|
+
test/test_magent.rb
|
data/PostInstall.txt
ADDED
File without changes
|
data/README.rdoc
ADDED
@@ -0,0 +1,53 @@
|
|
1
|
+
= magent
|
2
|
+
|
3
|
+
* http://github.com/dcu/magent
|
4
|
+
|
5
|
+
== DESCRIPTION:
|
6
|
+
|
7
|
+
Simple job queue system based on mongodb
|
8
|
+
|
9
|
+
== FEATURES/PROBLEMS:
|
10
|
+
|
11
|
+
* fast
|
12
|
+
* simple
|
13
|
+
* scalable
|
14
|
+
|
15
|
+
== SYNOPSIS:
|
16
|
+
|
17
|
+
see examples/
|
18
|
+
|
19
|
+
== REQUIREMENTS:
|
20
|
+
|
21
|
+
* mongodb >= 1.4
|
22
|
+
* mongo >= 1.0.0 (gem install mongo)
|
23
|
+
|
24
|
+
== INSTALL:
|
25
|
+
|
26
|
+
* rake gem
|
27
|
+
* sudo gem install pkg/*.gem
|
28
|
+
|
29
|
+
== LICENSE:
|
30
|
+
|
31
|
+
(The MIT License)
|
32
|
+
|
33
|
+
Copyright (c) 2009 David Cuadrado
|
34
|
+
|
35
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
36
|
+
a copy of this software and associated documentation files (the
|
37
|
+
'Software'), to deal in the Software without restriction, including
|
38
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
39
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
40
|
+
permit persons to whom the Software is furnished to do so, subject to
|
41
|
+
the following conditions:
|
42
|
+
|
43
|
+
The above copyright notice and this permission notice shall be
|
44
|
+
included in all copies or substantial portions of the Software.
|
45
|
+
|
46
|
+
THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
|
47
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
48
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
49
|
+
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
50
|
+
CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
51
|
+
TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
52
|
+
SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
53
|
+
|
data/Rakefile
ADDED
@@ -0,0 +1,27 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
gem 'hoe', '>= 2.1.0'
|
3
|
+
require 'hoe'
|
4
|
+
require 'fileutils'
|
5
|
+
require './lib/magent'
|
6
|
+
|
7
|
+
Hoe.plugin :newgem
|
8
|
+
# Hoe.plugin :website
|
9
|
+
# Hoe.plugin :cucumberfeatures
|
10
|
+
|
11
|
+
# Generate all the Rake tasks
|
12
|
+
# Run 'rake -T' to see list of generated tasks (from gem root directory)
|
13
|
+
$hoe = Hoe.spec 'magent' do
|
14
|
+
self.developer 'David Cuadrado', 'krawek@gmail.com'
|
15
|
+
self.post_install_message = ''
|
16
|
+
self.rubyforge_name = self.name
|
17
|
+
self.extra_deps = [['mongo','>= 0.1.0'],
|
18
|
+
['uuidtools', '>= 2.0.0']]
|
19
|
+
end
|
20
|
+
|
21
|
+
require 'newgem/tasks'
|
22
|
+
Dir['tasks/**/*.rake'].each { |t| load t }
|
23
|
+
|
24
|
+
# TODO - want other tests/tasks run by default? Add them to the list
|
25
|
+
# remove_task :default
|
26
|
+
# task :default => [:spec, :features]
|
27
|
+
|
data/bin/magent
ADDED
@@ -0,0 +1,139 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
$:.unshift File.dirname(__FILE__)+"/../lib/"
|
4
|
+
require 'magent'
|
5
|
+
require 'optparse'
|
6
|
+
require 'fileutils'
|
7
|
+
|
8
|
+
def usage(option_parser, error = nil)
|
9
|
+
$stderr.puts error if error
|
10
|
+
$stderr.puts option_parser
|
11
|
+
exit 1
|
12
|
+
end
|
13
|
+
|
14
|
+
options = {}
|
15
|
+
|
16
|
+
optparser = OptionParser.new do |opts|
|
17
|
+
opts.banner = "Usage: #{$0} [options] <start|stop|restart>\n\nExample: magent -t carl1 -a /path/to/agent.rb -d -P /tmp restart\n\nOptions:"
|
18
|
+
|
19
|
+
opts.on("-a PATH", "--agent=PATH", "Path to agent") do |o|
|
20
|
+
options[:agent] = o
|
21
|
+
end
|
22
|
+
|
23
|
+
opts.on("-t ID", "--identifier=ID", "Identifier") do |o|
|
24
|
+
options[:identifier] = o
|
25
|
+
end
|
26
|
+
|
27
|
+
opts.on("-d", "--daemonize", "Run agent as a daemon") do |o|
|
28
|
+
options[:daemonize] = true
|
29
|
+
end
|
30
|
+
|
31
|
+
opts.on("-l", "--log-path=PATH", "Log path") do |o|
|
32
|
+
options[:log_path] = o
|
33
|
+
end
|
34
|
+
|
35
|
+
opts.on("-P", "--piddir=PATH", "PID dir to use (if daemonized)", "Default: #{options[:piddir]}") do |o|
|
36
|
+
options[:piddir] = o
|
37
|
+
end
|
38
|
+
|
39
|
+
opts.on_tail("-h", "--help", "Show this help message.") do
|
40
|
+
puts opts
|
41
|
+
exit
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
args = []
|
46
|
+
begin
|
47
|
+
args = optparser.parse!
|
48
|
+
rescue => e
|
49
|
+
$stderr.puts e
|
50
|
+
$stderr.puts optparser
|
51
|
+
exit 0
|
52
|
+
end
|
53
|
+
|
54
|
+
if !options[:agent]
|
55
|
+
usage(optparser, "Error: --agent is required")
|
56
|
+
end
|
57
|
+
|
58
|
+
load options[:agent]
|
59
|
+
|
60
|
+
if Magent.current_actor.nil?
|
61
|
+
usage(optparser, "Use Magent.register(YourActor.new) to register an actor")
|
62
|
+
end
|
63
|
+
|
64
|
+
class Controller
|
65
|
+
def initialize(actor, opts)
|
66
|
+
@options = opts
|
67
|
+
@actor = actor
|
68
|
+
|
69
|
+
@options[:log_path] ||= Dir.getwd
|
70
|
+
|
71
|
+
@identity = @options[:identifier] || Magent::Utils.underscore(actor.class.to_s)
|
72
|
+
@identity << "-#{Socket.gethostname.split('.')[0]}"
|
73
|
+
end
|
74
|
+
|
75
|
+
def start
|
76
|
+
if @options[:daemonize] && @options[:piddir]
|
77
|
+
run_as_daemon
|
78
|
+
else
|
79
|
+
Magent::Processor.new(@actor).run!
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
def stop
|
84
|
+
begin
|
85
|
+
pid = File.read(pid_file).to_i
|
86
|
+
Process.kill("TERM", pid)
|
87
|
+
Process.kill(0, pid)
|
88
|
+
Process.wait
|
89
|
+
rescue Errno::ECHILD, Errno::ESRCH => e
|
90
|
+
$stdout.puts "Process #{pid} has stopped"
|
91
|
+
rescue Errno::ENOENT => e
|
92
|
+
$stdout.puts "Warning: #{e}"
|
93
|
+
ensure
|
94
|
+
File.unlink(pid_file) if File.exist?(pid_file)
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
def restart
|
99
|
+
begin
|
100
|
+
stop
|
101
|
+
rescue => e
|
102
|
+
$stderr.puts "Warning: #{e}"
|
103
|
+
end
|
104
|
+
start
|
105
|
+
end
|
106
|
+
|
107
|
+
private
|
108
|
+
def run_as_daemon
|
109
|
+
daemonize
|
110
|
+
|
111
|
+
FileUtils.mkpath(@options[:piddir])
|
112
|
+
if @options[:piddir]
|
113
|
+
File.open(pid_file, "w") do |f|
|
114
|
+
f.write(Process.pid)
|
115
|
+
end
|
116
|
+
end
|
117
|
+
|
118
|
+
Magent::Processor.new(@actor).run!
|
119
|
+
end
|
120
|
+
|
121
|
+
def pid_file
|
122
|
+
@pid_file ||= File.join(@options[:piddir], "magent.#{@identity}.pid")
|
123
|
+
end
|
124
|
+
|
125
|
+
def daemonize
|
126
|
+
exit if fork
|
127
|
+
Process.setsid
|
128
|
+
exit if fork
|
129
|
+
|
130
|
+
STDIN.reopen "/dev/null"
|
131
|
+
STDOUT.reopen "#{@options[:log_path]}/magent.#{@identity}.out", "a"
|
132
|
+
STDERR.reopen "#{@options[:log_path]}/magent.#{@identity}.err", "a"
|
133
|
+
|
134
|
+
STDERR.sync = true
|
135
|
+
STDOUT.sync = true
|
136
|
+
end
|
137
|
+
end
|
138
|
+
|
139
|
+
Controller.new(Magent.current_actor, options).send(args.shift)
|
@@ -0,0 +1,22 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
$:.unshift File.dirname(__FILE__)+"/../../lib/"
|
4
|
+
require 'magent'
|
5
|
+
|
6
|
+
id = "#{rand(16)}#{rand(16)}#{rand(16)}#{rand(16)}"
|
7
|
+
|
8
|
+
values = (1..5).to_a.map { rand(10) }
|
9
|
+
puts values.join(" + ")
|
10
|
+
Magent.push("workers", :sum, id, *values)
|
11
|
+
|
12
|
+
channel = Magent::GenericChannel.new("+#{id}")
|
13
|
+
|
14
|
+
loop do
|
15
|
+
v = channel.dequeue;
|
16
|
+
if v
|
17
|
+
$stdout.puts v.inspect
|
18
|
+
break
|
19
|
+
end
|
20
|
+
sleep 0.1
|
21
|
+
end
|
22
|
+
|
@@ -0,0 +1,30 @@
|
|
1
|
+
$:.unshift File.dirname(__FILE__)+"/../../lib/"
|
2
|
+
require 'magent'
|
3
|
+
|
4
|
+
# Use: magent /path/to/this/file
|
5
|
+
|
6
|
+
class Worker
|
7
|
+
include Magent::Actor
|
8
|
+
channel_name "workers"
|
9
|
+
expose :sum
|
10
|
+
|
11
|
+
def sum(payload)
|
12
|
+
id, *args = payload
|
13
|
+
|
14
|
+
s = args.inject(0) { |v, a| a += v }
|
15
|
+
send_to_client(id, {:method => :sum, :result => s})
|
16
|
+
end
|
17
|
+
|
18
|
+
private
|
19
|
+
def send_to_client(id, message)
|
20
|
+
c = Magent::GenericChannel.new("+#{id}")
|
21
|
+
c.enqueue(message)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
Magent.register(Worker.new)
|
26
|
+
|
27
|
+
if $0 == __FILE__
|
28
|
+
Magent::Processor.new(Magent.current_actor).run!
|
29
|
+
end
|
30
|
+
|
@@ -0,0 +1,33 @@
|
|
1
|
+
$:.unshift File.dirname(__FILE__)+"/../../lib/"
|
2
|
+
require 'magent'
|
3
|
+
|
4
|
+
Magent.push("errors", :fail, "this is a fail")
|
5
|
+
|
6
|
+
class Error
|
7
|
+
include Magent::Actor
|
8
|
+
|
9
|
+
channel_name "errors"
|
10
|
+
expose :fail
|
11
|
+
|
12
|
+
def fail(payload)
|
13
|
+
@count ||= 0
|
14
|
+
errors = self.class.channel.errors
|
15
|
+
|
16
|
+
errors.each do |error|
|
17
|
+
@count += 1
|
18
|
+
$stderr.puts "Retrying: #{error["method"]}(#{error["payload"].inspect})"
|
19
|
+
self.class.channel.retry_error(error)
|
20
|
+
end
|
21
|
+
|
22
|
+
if @count == 0
|
23
|
+
raise payload.inspect
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
Magent.register(Error.new)
|
29
|
+
|
30
|
+
if $0 == __FILE__
|
31
|
+
Magent::Processor.new(Magent.current_actor).run!
|
32
|
+
end
|
33
|
+
|
@@ -0,0 +1,37 @@
|
|
1
|
+
$:.unshift File.dirname(__FILE__)+"/../../lib/"
|
2
|
+
require 'magent'
|
3
|
+
|
4
|
+
# Use: magent /path/to/this/file
|
5
|
+
|
6
|
+
Magent.push("bots", :echo, "hello, world")
|
7
|
+
Magent.push("bots", :do_task, "File", :exist?, "/etc/passwd")
|
8
|
+
Magent.push("bots", :echo, "Press ctrl+c to close")
|
9
|
+
Magent.push("bots", :do_not_exist, "you should not see this message")
|
10
|
+
|
11
|
+
class Bot
|
12
|
+
include Magent::Actor
|
13
|
+
channel_name "bots"
|
14
|
+
expose :echo, :do_task
|
15
|
+
|
16
|
+
def echo(payload)
|
17
|
+
$stderr.puts payload.inspect
|
18
|
+
end
|
19
|
+
|
20
|
+
def do_task(payload)
|
21
|
+
klass, *args = payload
|
22
|
+
|
23
|
+
result = Object.module_eval(klass).send(*args)
|
24
|
+
$stderr.puts "RESULT: #{result}"
|
25
|
+
end
|
26
|
+
|
27
|
+
at_least_every 15 do
|
28
|
+
puts "Hi there!, you'll see this message again in ~15 seconds"
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
Magent.register(Bot.new)
|
33
|
+
|
34
|
+
if $0 == __FILE__
|
35
|
+
Magent::Processor.new(Magent.current_actor).run!
|
36
|
+
end
|
37
|
+
|
@@ -0,0 +1,27 @@
|
|
1
|
+
$:.unshift File.dirname(__FILE__)+"/../../lib/"
|
2
|
+
require 'magent'
|
3
|
+
|
4
|
+
Magent.push("stats", :calc)
|
5
|
+
Magent.push("stats", :calc)
|
6
|
+
Magent.push("stats", :calc)
|
7
|
+
Magent.push("stats", :calc)
|
8
|
+
|
9
|
+
class Stats
|
10
|
+
include Magent::Actor
|
11
|
+
|
12
|
+
channel_name "stats"
|
13
|
+
expose :calc
|
14
|
+
|
15
|
+
def calc(payload)
|
16
|
+
$stderr.puts "messages in queue: #{self.class.channel.queue_count}"
|
17
|
+
$stderr.puts "total messages count: #{self.class.channel.message_count}"
|
18
|
+
$stderr.puts "total errors: #{self.class.channel.error_count}"
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
Magent.register(Stats.new)
|
23
|
+
|
24
|
+
if $0 == __FILE__
|
25
|
+
Magent::Processor.new(Magent.current_actor).run!
|
26
|
+
end
|
27
|
+
|
data/lib/magent.rb
ADDED
@@ -0,0 +1,57 @@
|
|
1
|
+
$:.unshift(File.dirname(__FILE__)) unless
|
2
|
+
$:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
|
3
|
+
|
4
|
+
require 'mongo'
|
5
|
+
require 'set'
|
6
|
+
require 'uuidtools'
|
7
|
+
|
8
|
+
require 'magent/utils'
|
9
|
+
require 'magent/generic_channel'
|
10
|
+
require 'magent/channel'
|
11
|
+
require 'magent/push'
|
12
|
+
require 'magent/actor'
|
13
|
+
require 'magent/processor'
|
14
|
+
|
15
|
+
module Magent
|
16
|
+
VERSION = '0.4.1'
|
17
|
+
|
18
|
+
@@db_name = 'magent'
|
19
|
+
@@host = 'localhost'
|
20
|
+
@@port = '27017'
|
21
|
+
@@username = ''
|
22
|
+
@@password = ''
|
23
|
+
|
24
|
+
def self.host(host,port)
|
25
|
+
@@host = host
|
26
|
+
@@port = port
|
27
|
+
end
|
28
|
+
|
29
|
+
def self.auth(username,password)
|
30
|
+
@@username = username
|
31
|
+
@@password = password
|
32
|
+
end
|
33
|
+
|
34
|
+
def self.db_name(db_name)
|
35
|
+
@@db_name = db_name
|
36
|
+
end
|
37
|
+
|
38
|
+
def self.connection
|
39
|
+
return @@connection if defined? @@connection
|
40
|
+
@@connection = Mongo::Connection.new(@@host, @@port, :auto_reconnect => true)
|
41
|
+
@@connection.add_auth(@@db_name, @@username, @@password) if @@username!='' && @@password!=''
|
42
|
+
return @@connection
|
43
|
+
end
|
44
|
+
|
45
|
+
def self.connection=(new_connection)
|
46
|
+
@@connection = new_connection
|
47
|
+
end
|
48
|
+
|
49
|
+
def self.database=(name)
|
50
|
+
@@database = Magent.connection.db(name)
|
51
|
+
end
|
52
|
+
|
53
|
+
def self.database
|
54
|
+
@@database ||= Magent.connection.db(@@db_name)
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
data/lib/magent/actor.rb
ADDED
@@ -0,0 +1,78 @@
|
|
1
|
+
module Magent
|
2
|
+
module Actor
|
3
|
+
def self.included(klass)
|
4
|
+
klass.class_eval do
|
5
|
+
extend Actor::ClassMethods
|
6
|
+
include Actor::InstanceMethods
|
7
|
+
end
|
8
|
+
end
|
9
|
+
|
10
|
+
module ClassMethods
|
11
|
+
def expose(*methods)
|
12
|
+
methods.each do |m|
|
13
|
+
actions << m.to_s
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
def channel_name(name = nil)
|
18
|
+
@channel_name ||= (name || Magent::Utils.underscore(self.name)).to_s
|
19
|
+
end
|
20
|
+
|
21
|
+
def actions
|
22
|
+
@actions ||= Set.new
|
23
|
+
end
|
24
|
+
|
25
|
+
def can_handle?(action)
|
26
|
+
actions.include?(action.to_s)
|
27
|
+
end
|
28
|
+
|
29
|
+
def channel
|
30
|
+
@channel ||= begin
|
31
|
+
Channel.new(self.channel_name)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def tasks
|
36
|
+
@tasks ||= []
|
37
|
+
end
|
38
|
+
|
39
|
+
def at_least_every(seconds, &block)
|
40
|
+
tasks << {:every => seconds, :last_time => Time.now, :block => block}
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
module InstanceMethods
|
45
|
+
def _run_tasks
|
46
|
+
tasks = self.class.tasks
|
47
|
+
|
48
|
+
return false if tasks.empty?
|
49
|
+
performed = false
|
50
|
+
|
51
|
+
tasks.each do |task|
|
52
|
+
delta = Time.now - task[:last_time]
|
53
|
+
|
54
|
+
if delta >= task[:every]
|
55
|
+
task[:last_time] = Time.now
|
56
|
+
begin
|
57
|
+
instance_eval(&task[:block])
|
58
|
+
rescue Exception => e
|
59
|
+
$stderr.puts "Failed periodical task: #{e.message}"
|
60
|
+
$stderr.puts e.backtrace.join("\n\t")
|
61
|
+
end
|
62
|
+
performed = true
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
performed
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end # Actor
|
70
|
+
|
71
|
+
def self.register(actor)
|
72
|
+
@current_actor = actor
|
73
|
+
end
|
74
|
+
|
75
|
+
def self.current_actor
|
76
|
+
@current_actor
|
77
|
+
end
|
78
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
module Magent
|
2
|
+
class Channel < GenericChannel
|
3
|
+
def enqueue(message, args)
|
4
|
+
super([message, args])
|
5
|
+
end
|
6
|
+
|
7
|
+
def failed(info)
|
8
|
+
error_collection.save(info.merge({:_id => generate_uid, :channel => @name, :created_at => Time.now.utc}))
|
9
|
+
end
|
10
|
+
|
11
|
+
def error_count
|
12
|
+
error_collection.count()
|
13
|
+
end
|
14
|
+
|
15
|
+
def errors(conds = {})
|
16
|
+
page = conds.delete(:page) || 1
|
17
|
+
per_page = conds.delete(:per_page) || 10
|
18
|
+
|
19
|
+
error_collection.find({}, {:skip => (page-1)*per_page,
|
20
|
+
:limit => per_page,
|
21
|
+
:sort => [["created_at", -1]]})
|
22
|
+
end
|
23
|
+
|
24
|
+
def remove_error(error_id)
|
25
|
+
self.error_collection.remove(:_id => error_id)
|
26
|
+
end
|
27
|
+
|
28
|
+
def retry_error(error)
|
29
|
+
remove_error(error["_id"])
|
30
|
+
enqueue(error["method"], error["payload"])
|
31
|
+
end
|
32
|
+
|
33
|
+
def error_collection
|
34
|
+
@error_collection ||= Magent.database.collection("#{@name}-errors")
|
35
|
+
end
|
36
|
+
end # Channel
|
37
|
+
end
|
@@ -0,0 +1,43 @@
|
|
1
|
+
module Magent
|
2
|
+
class GenericChannel
|
3
|
+
attr_reader :name
|
4
|
+
|
5
|
+
def initialize(name)
|
6
|
+
@name = name
|
7
|
+
end
|
8
|
+
|
9
|
+
def enqueue(message)
|
10
|
+
collection.save({:_id => generate_uid, :message => message, :priority => 3, :created_at => Time.now.to_i})
|
11
|
+
end
|
12
|
+
|
13
|
+
def message_count
|
14
|
+
collection.count # TODO: number of processed messages (create a collection for stats)
|
15
|
+
end
|
16
|
+
|
17
|
+
def queue_count
|
18
|
+
collection.count
|
19
|
+
end
|
20
|
+
|
21
|
+
def dequeue
|
22
|
+
if m = self.next_message
|
23
|
+
m["message"]
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def next_message
|
28
|
+
Magent.database.command(BSON::OrderedHash[:findandmodify, @name,
|
29
|
+
:sort, [{:priority => -1}, {:created_at => 1}],
|
30
|
+
:remove, true
|
31
|
+
])["value"]
|
32
|
+
end
|
33
|
+
|
34
|
+
def collection
|
35
|
+
@collection ||= Magent.database.collection(@name)
|
36
|
+
end
|
37
|
+
|
38
|
+
protected
|
39
|
+
def generate_uid
|
40
|
+
UUIDTools::UUID.random_create.hexdigest
|
41
|
+
end
|
42
|
+
end # GenericChannel
|
43
|
+
end
|
@@ -0,0 +1,68 @@
|
|
1
|
+
module Magent
|
2
|
+
class Processor
|
3
|
+
attr_reader :actor
|
4
|
+
|
5
|
+
def initialize(actor)
|
6
|
+
@actor = actor
|
7
|
+
@shutdown = false
|
8
|
+
|
9
|
+
@actor.class.actions.each do |action|
|
10
|
+
if !@actor.respond_to?(action)
|
11
|
+
raise ArgumentError, "action '#{action}' is not defined"
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
def run!
|
17
|
+
processed_messages = 0
|
18
|
+
delay = 0
|
19
|
+
|
20
|
+
trap('TERM') { shutdown!; exit 0 }
|
21
|
+
trap('SIGINT') { shutdown!; exit 0 }
|
22
|
+
|
23
|
+
loop do
|
24
|
+
break if @shutdown
|
25
|
+
|
26
|
+
delay = 0 if @actor._run_tasks
|
27
|
+
|
28
|
+
@method, @payload = @actor.class.channel.dequeue
|
29
|
+
|
30
|
+
if @method.nil?
|
31
|
+
delay += 0.1 if delay <= 5
|
32
|
+
else
|
33
|
+
delay = 0
|
34
|
+
$stderr.puts "#{@actor.class}##{@method}(#{@payload.inspect})"
|
35
|
+
begin
|
36
|
+
if @actor.class.can_handle?(@method)
|
37
|
+
processed_messages += 1
|
38
|
+
@actor.send(@method, @payload)
|
39
|
+
|
40
|
+
if processed_messages > 20
|
41
|
+
processed_messages = 0
|
42
|
+
GC.start
|
43
|
+
end
|
44
|
+
else
|
45
|
+
$stderr.puts "Unknown action: #{@method} (payload=#{@payload.inspect})"
|
46
|
+
end
|
47
|
+
rescue SystemExit
|
48
|
+
rescue Exception => e
|
49
|
+
$stderr.puts "Error while executing #{@method.inspect} #{@payload.inspect}"
|
50
|
+
$stderr.puts "#{e.to_s}\n#{e.backtrace.join("\t\n")}"
|
51
|
+
@actor.class.channel.failed(:message => e.message, :method => @method, :payload => @payload, :backtrace => e.backtrace, :date => Time.now.utc)
|
52
|
+
ensure
|
53
|
+
@method, @payload = nil
|
54
|
+
end
|
55
|
+
end
|
56
|
+
sleep delay
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
def shutdown!
|
61
|
+
@shutdown = true
|
62
|
+
$stderr.puts "Shutting down..."
|
63
|
+
if @method
|
64
|
+
@actor.class.channel.enqueue(@method, @payload)
|
65
|
+
end
|
66
|
+
end
|
67
|
+
end #Processor
|
68
|
+
end
|
data/lib/magent/push.rb
ADDED
data/lib/magent/utils.rb
ADDED
@@ -0,0 +1,13 @@
|
|
1
|
+
module Magent
|
2
|
+
module Utils
|
3
|
+
def self.underscore(word)
|
4
|
+
word.to_s.gsub(/::/, '.').
|
5
|
+
gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
|
6
|
+
gsub(/([a-z\d])([A-Z])/,'\1_\2').tr("-", "_").downcase
|
7
|
+
end
|
8
|
+
|
9
|
+
def self.camelize(word)
|
10
|
+
word.to_s.gsub(/\.(.?)/) { "::#{$1.upcase}" }.gsub(/(?:^|_)(.)/) { $1.upcase }
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
data/magent.gemspec
ADDED
@@ -0,0 +1,46 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
|
3
|
+
Gem::Specification.new do |s|
|
4
|
+
s.name = %q{thirtysixthspan-magent}
|
5
|
+
s.version = "0.4.1"
|
6
|
+
|
7
|
+
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
8
|
+
s.authors = ["David Cuadrado"]
|
9
|
+
s.date = %q{2010-05-10}
|
10
|
+
s.default_executable = %q{magent}
|
11
|
+
s.description = %q{Simple job queue system based on mongodb}
|
12
|
+
s.email = ["krawek@gmail.com"]
|
13
|
+
s.executables = ["magent"]
|
14
|
+
s.extra_rdoc_files = ["History.txt", "Manifest.txt", "PostInstall.txt"]
|
15
|
+
s.files = ["History.txt", "Manifest.txt", "PostInstall.txt", "README.rdoc", "Rakefile", "bin/magent", "examples/comm/run.rb", "examples/comm/worker.rb", "examples/error/error.rb", "examples/simple/bot.rb", "examples/stats/stats.rb", "lib/magent.rb", "lib/magent/actor.rb", "lib/magent/channel.rb", "lib/magent/generic_channel.rb", "lib/magent/processor.rb", "lib/magent/push.rb", "lib/magent/utils.rb", "magent.gemspec", "script/console", "test/test_helper.rb", "test/test_magent.rb"]
|
16
|
+
s.homepage = %q{http://github.com/dcu/magent}
|
17
|
+
s.post_install_message = %q{}
|
18
|
+
s.rdoc_options = ["--main", "README.rdoc"]
|
19
|
+
s.require_paths = ["lib"]
|
20
|
+
s.rubyforge_project = %q{magent}
|
21
|
+
s.rubygems_version = %q{1.3.6}
|
22
|
+
s.summary = %q{Simple job queue system based on mongodb}
|
23
|
+
s.test_files = ["test/test_helper.rb", "test/test_magent.rb"]
|
24
|
+
|
25
|
+
if s.respond_to? :specification_version then
|
26
|
+
current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
|
27
|
+
s.specification_version = 3
|
28
|
+
|
29
|
+
if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
|
30
|
+
s.add_runtime_dependency(%q<mongo>, [">= 0.1.0"])
|
31
|
+
s.add_runtime_dependency(%q<uuidtools>, [">= 2.0.0"])
|
32
|
+
s.add_development_dependency(%q<rubyforge>, [">= 2.0.4"])
|
33
|
+
s.add_development_dependency(%q<hoe>, [">= 2.6.0"])
|
34
|
+
else
|
35
|
+
s.add_dependency(%q<mongo>, [">= 0.1.0"])
|
36
|
+
s.add_dependency(%q<uuidtools>, [">= 2.0.0"])
|
37
|
+
s.add_dependency(%q<rubyforge>, [">= 2.0.4"])
|
38
|
+
s.add_dependency(%q<hoe>, [">= 2.6.0"])
|
39
|
+
end
|
40
|
+
else
|
41
|
+
s.add_dependency(%q<mongo>, [">= 0.1.0"])
|
42
|
+
s.add_dependency(%q<uuidtools>, [">= 2.0.0"])
|
43
|
+
s.add_dependency(%q<rubyforge>, [">= 2.0.4"])
|
44
|
+
s.add_dependency(%q<hoe>, [">= 2.6.0"])
|
45
|
+
end
|
46
|
+
end
|
data/script/console
ADDED
@@ -0,0 +1,9 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
# File: script/console
|
3
|
+
irb = RUBY_PLATFORM =~ /(:?mswin|mingw)/ ? 'irb.bat' : 'irb'
|
4
|
+
|
5
|
+
libs = " -r irb/completion"
|
6
|
+
#libs << " -r #{File.dirname(__FILE__) + '/../lib/console_lib/init.rb'}"
|
7
|
+
libs << " -r #{File.dirname(__FILE__) + '/../lib/magent.rb'}"
|
8
|
+
puts "Loading magent gem"
|
9
|
+
exec "#{irb} #{libs} --simple-prompt"
|
data/test/test_helper.rb
ADDED
data/test/test_magent.rb
ADDED
metadata
ADDED
@@ -0,0 +1,149 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: thirtysixthspan-magent
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
prerelease: false
|
5
|
+
segments:
|
6
|
+
- 0
|
7
|
+
- 4
|
8
|
+
- 1
|
9
|
+
version: 0.4.1
|
10
|
+
platform: ruby
|
11
|
+
authors:
|
12
|
+
- David Cuadrado
|
13
|
+
autorequire:
|
14
|
+
bindir: bin
|
15
|
+
cert_chain: []
|
16
|
+
|
17
|
+
date: 2010-05-10 00:00:00 -05:00
|
18
|
+
default_executable: magent
|
19
|
+
dependencies:
|
20
|
+
- !ruby/object:Gem::Dependency
|
21
|
+
name: mongo
|
22
|
+
prerelease: false
|
23
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
24
|
+
none: false
|
25
|
+
requirements:
|
26
|
+
- - ">="
|
27
|
+
- !ruby/object:Gem::Version
|
28
|
+
segments:
|
29
|
+
- 0
|
30
|
+
- 1
|
31
|
+
- 0
|
32
|
+
version: 0.1.0
|
33
|
+
type: :runtime
|
34
|
+
version_requirements: *id001
|
35
|
+
- !ruby/object:Gem::Dependency
|
36
|
+
name: uuidtools
|
37
|
+
prerelease: false
|
38
|
+
requirement: &id002 !ruby/object:Gem::Requirement
|
39
|
+
none: false
|
40
|
+
requirements:
|
41
|
+
- - ">="
|
42
|
+
- !ruby/object:Gem::Version
|
43
|
+
segments:
|
44
|
+
- 2
|
45
|
+
- 0
|
46
|
+
- 0
|
47
|
+
version: 2.0.0
|
48
|
+
type: :runtime
|
49
|
+
version_requirements: *id002
|
50
|
+
- !ruby/object:Gem::Dependency
|
51
|
+
name: rubyforge
|
52
|
+
prerelease: false
|
53
|
+
requirement: &id003 !ruby/object:Gem::Requirement
|
54
|
+
none: false
|
55
|
+
requirements:
|
56
|
+
- - ">="
|
57
|
+
- !ruby/object:Gem::Version
|
58
|
+
segments:
|
59
|
+
- 2
|
60
|
+
- 0
|
61
|
+
- 4
|
62
|
+
version: 2.0.4
|
63
|
+
type: :development
|
64
|
+
version_requirements: *id003
|
65
|
+
- !ruby/object:Gem::Dependency
|
66
|
+
name: hoe
|
67
|
+
prerelease: false
|
68
|
+
requirement: &id004 !ruby/object:Gem::Requirement
|
69
|
+
none: false
|
70
|
+
requirements:
|
71
|
+
- - ">="
|
72
|
+
- !ruby/object:Gem::Version
|
73
|
+
segments:
|
74
|
+
- 2
|
75
|
+
- 6
|
76
|
+
- 0
|
77
|
+
version: 2.6.0
|
78
|
+
type: :development
|
79
|
+
version_requirements: *id004
|
80
|
+
description: Simple job queue system based on mongodb
|
81
|
+
email:
|
82
|
+
- krawek@gmail.com
|
83
|
+
executables:
|
84
|
+
- magent
|
85
|
+
extensions: []
|
86
|
+
|
87
|
+
extra_rdoc_files:
|
88
|
+
- History.txt
|
89
|
+
- Manifest.txt
|
90
|
+
- PostInstall.txt
|
91
|
+
files:
|
92
|
+
- History.txt
|
93
|
+
- Manifest.txt
|
94
|
+
- PostInstall.txt
|
95
|
+
- README.rdoc
|
96
|
+
- Rakefile
|
97
|
+
- bin/magent
|
98
|
+
- examples/comm/run.rb
|
99
|
+
- examples/comm/worker.rb
|
100
|
+
- examples/error/error.rb
|
101
|
+
- examples/simple/bot.rb
|
102
|
+
- examples/stats/stats.rb
|
103
|
+
- lib/magent.rb
|
104
|
+
- lib/magent/actor.rb
|
105
|
+
- lib/magent/channel.rb
|
106
|
+
- lib/magent/generic_channel.rb
|
107
|
+
- lib/magent/processor.rb
|
108
|
+
- lib/magent/push.rb
|
109
|
+
- lib/magent/utils.rb
|
110
|
+
- magent.gemspec
|
111
|
+
- script/console
|
112
|
+
- test/test_helper.rb
|
113
|
+
- test/test_magent.rb
|
114
|
+
has_rdoc: true
|
115
|
+
homepage: http://github.com/dcu/magent
|
116
|
+
licenses: []
|
117
|
+
|
118
|
+
post_install_message: ""
|
119
|
+
rdoc_options:
|
120
|
+
- --main
|
121
|
+
- README.rdoc
|
122
|
+
require_paths:
|
123
|
+
- lib
|
124
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
125
|
+
none: false
|
126
|
+
requirements:
|
127
|
+
- - ">="
|
128
|
+
- !ruby/object:Gem::Version
|
129
|
+
segments:
|
130
|
+
- 0
|
131
|
+
version: "0"
|
132
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
133
|
+
none: false
|
134
|
+
requirements:
|
135
|
+
- - ">="
|
136
|
+
- !ruby/object:Gem::Version
|
137
|
+
segments:
|
138
|
+
- 0
|
139
|
+
version: "0"
|
140
|
+
requirements: []
|
141
|
+
|
142
|
+
rubyforge_project: magent
|
143
|
+
rubygems_version: 1.3.7
|
144
|
+
signing_key:
|
145
|
+
specification_version: 3
|
146
|
+
summary: Simple job queue system based on mongodb
|
147
|
+
test_files:
|
148
|
+
- test/test_helper.rb
|
149
|
+
- test/test_magent.rb
|