magent 0.4.2 → 0.5.0
Sign up to get free protection for your applications and to get access to all the features.
- data/.document +5 -0
- data/.gitignore +7 -0
- data/Gemfile +7 -0
- data/Gemfile.lock +21 -0
- data/LICENSE +20 -0
- data/README.rdoc +37 -1
- data/Rakefile +48 -20
- data/VERSION +1 -0
- data/bin/magent +31 -14
- data/examples/comm/worker.rb +1 -1
- data/examples/error/error.rb +1 -1
- data/examples/mongomapper/async.rb +41 -0
- data/examples/simple/bot.rb +2 -1
- data/examples/stats/stats.rb +1 -1
- data/lib/magent.rb +80 -28
- data/lib/magent/actor.rb +5 -1
- data/lib/magent/actor_channel.rb +34 -0
- data/lib/magent/async.rb +65 -0
- data/lib/magent/async_channel.rb +38 -0
- data/lib/magent/{channel.rb → failure.rb} +5 -9
- data/lib/magent/generic_channel.rb +7 -9
- data/lib/magent/processor.rb +25 -39
- data/lib/magent/push.rb +2 -2
- data/lib/magent/railtie.rb +10 -0
- data/lib/magent/web_socket_channel.rb +18 -0
- data/lib/magent/web_socket_server.rb +95 -0
- data/lib/tasks/magent.rake +55 -0
- data/magent.gemspec +73 -22
- metadata +49 -40
data/.document
ADDED
data/.gitignore
ADDED
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
GEM
|
2
|
+
remote: http://rubygems.org/
|
3
|
+
specs:
|
4
|
+
addressable (2.2.1)
|
5
|
+
bson (1.1)
|
6
|
+
em-websocket (0.1.4)
|
7
|
+
addressable (>= 2.1.1)
|
8
|
+
eventmachine (>= 0.12.9)
|
9
|
+
eventmachine (0.12.10)
|
10
|
+
mongo (1.1)
|
11
|
+
bson (>= 1.0.5)
|
12
|
+
uuidtools (2.1.1)
|
13
|
+
|
14
|
+
PLATFORMS
|
15
|
+
ruby
|
16
|
+
|
17
|
+
DEPENDENCIES
|
18
|
+
bson
|
19
|
+
em-websocket
|
20
|
+
mongo
|
21
|
+
uuidtools
|
data/LICENSE
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
Copyright (c) 2009 David A. Cuadrado
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
4
|
+
a copy of this software and associated documentation files (the
|
5
|
+
"Software"), to deal in the Software without restriction, including
|
6
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
7
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
8
|
+
permit persons to whom the Software is furnished to do so, subject to
|
9
|
+
the following conditions:
|
10
|
+
|
11
|
+
The above copyright notice and this permission notice shall be
|
12
|
+
included in all copies or substantial portions of the Software.
|
13
|
+
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
15
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
16
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
17
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
18
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
19
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
20
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.rdoc
CHANGED
@@ -18,7 +18,7 @@ see examples/
|
|
18
18
|
|
19
19
|
== REQUIREMENTS:
|
20
20
|
|
21
|
-
* mongodb >= 1.
|
21
|
+
* mongodb >= 1.6
|
22
22
|
* mongo >= 1.0.0 (gem install mongo)
|
23
23
|
|
24
24
|
== INSTALL:
|
@@ -26,6 +26,42 @@ see examples/
|
|
26
26
|
* rake gem
|
27
27
|
* sudo gem install pkg/*.gem
|
28
28
|
|
29
|
+
== USAGE:
|
30
|
+
|
31
|
+
include Magent::Async in your classes:
|
32
|
+
|
33
|
+
class MyClass
|
34
|
+
include Magent::Async
|
35
|
+
|
36
|
+
def self.process_task
|
37
|
+
puts "Processing task"
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
enqueue with priority 1:
|
42
|
+
|
43
|
+
MyClass.async.process_task.commit!(1)
|
44
|
+
|
45
|
+
start the worker:
|
46
|
+
|
47
|
+
magent start
|
48
|
+
|
49
|
+
|
50
|
+
== RAILS SUPPORT
|
51
|
+
|
52
|
+
= initializer
|
53
|
+
|
54
|
+
Magent.setup(YAML.load_file(Rails.root.join('config', 'magent.yml')),
|
55
|
+
Rails.env, { :logger => Rails.logger }
|
56
|
+
|
57
|
+
|
58
|
+
= starting the worker
|
59
|
+
|
60
|
+
to run the jobs in rails type:
|
61
|
+
|
62
|
+
rake magent:start
|
63
|
+
|
64
|
+
|
29
65
|
== LICENSE:
|
30
66
|
|
31
67
|
(The MIT License)
|
data/Rakefile
CHANGED
@@ -1,27 +1,55 @@
|
|
1
1
|
require 'rubygems'
|
2
|
-
|
3
|
-
require 'hoe'
|
4
|
-
require 'fileutils'
|
5
|
-
require './lib/magent'
|
2
|
+
require 'rake'
|
6
3
|
|
7
|
-
|
8
|
-
|
9
|
-
|
4
|
+
begin
|
5
|
+
require 'jeweler'
|
6
|
+
Jeweler::Tasks.new do |gem|
|
7
|
+
gem.name = "magent"
|
8
|
+
gem.summary = %Q{Simple job queue system based on mongodb}
|
9
|
+
gem.description = %Q{Simple job queue system based on mongodb}
|
10
|
+
gem.email = "krawek@gmail.com"
|
11
|
+
gem.homepage = "http://github.com/dcu/magent"
|
12
|
+
gem.authors = ["David A. Cuadrado"]
|
13
|
+
gem.add_development_dependency "thoughtbot-shoulda", ">= 0"
|
14
|
+
gem.add_dependency "mongo"
|
15
|
+
gem.add_dependency "em-websocket"
|
16
|
+
gem.add_dependency "uuidtools"
|
17
|
+
end
|
18
|
+
Jeweler::GemcutterTasks.new
|
19
|
+
rescue LoadError
|
20
|
+
puts "Jeweler (or a dependency) not available. Install it with: gem install jeweler"
|
21
|
+
end
|
22
|
+
|
23
|
+
require 'rake/testtask'
|
24
|
+
Rake::TestTask.new(:test) do |test|
|
25
|
+
test.libs << 'lib' << 'test'
|
26
|
+
test.pattern = 'test/**/test_*.rb'
|
27
|
+
test.verbose = true
|
28
|
+
end
|
10
29
|
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
30
|
+
begin
|
31
|
+
require 'rcov/rcovtask'
|
32
|
+
Rcov::RcovTask.new do |test|
|
33
|
+
test.libs << 'test'
|
34
|
+
test.pattern = 'test/**/test_*.rb'
|
35
|
+
test.verbose = true
|
36
|
+
end
|
37
|
+
rescue LoadError
|
38
|
+
task :rcov do
|
39
|
+
abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
|
40
|
+
end
|
19
41
|
end
|
20
42
|
|
21
|
-
|
22
|
-
Dir['tasks/**/*.rake'].each { |t| load t }
|
43
|
+
task :test => :check_dependencies
|
23
44
|
|
24
|
-
|
25
|
-
# remove_task :default
|
26
|
-
# task :default => [:spec, :features]
|
45
|
+
task :default => :test
|
27
46
|
|
47
|
+
require 'rake/rdoctask'
|
48
|
+
Rake::RDocTask.new do |rdoc|
|
49
|
+
version = File.exist?('VERSION') ? File.read('VERSION') : ""
|
50
|
+
|
51
|
+
rdoc.rdoc_dir = 'rdoc'
|
52
|
+
rdoc.title = "magent #{version}"
|
53
|
+
rdoc.rdoc_files.include('README*')
|
54
|
+
rdoc.rdoc_files.include('lib/**/*.rb')
|
55
|
+
end
|
data/VERSION
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
0.5.0
|
data/bin/magent
CHANGED
@@ -1,23 +1,33 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
2
|
|
3
|
+
if File.exist?("config/environments") && File.exist?("app/models") # rails
|
4
|
+
require Dir.getwd + '/config/environment'
|
5
|
+
end
|
6
|
+
|
3
7
|
$:.unshift File.dirname(__FILE__)+"/../lib/"
|
4
8
|
require 'magent'
|
5
9
|
require 'optparse'
|
6
10
|
require 'fileutils'
|
7
11
|
|
12
|
+
|
8
13
|
def usage(option_parser, error = nil)
|
9
14
|
$stderr.puts error if error
|
10
15
|
$stderr.puts option_parser
|
11
16
|
exit 1
|
12
17
|
end
|
13
18
|
|
14
|
-
options = {}
|
19
|
+
options = {:piddir => "/tmp", :type => :async, :queue => :default}
|
15
20
|
|
16
21
|
optparser = OptionParser.new do |opts|
|
17
|
-
opts.banner = "Usage: #{$0} [options] <start|stop|restart>\n\nExample: magent -
|
22
|
+
opts.banner = "Usage: #{$0} [options] -Q <queue> <start|stop|restart>\n\nExample: magent -d -P /tmp -Q default restart\n\nOptions:"
|
18
23
|
|
19
24
|
opts.on("-a PATH", "--agent=PATH", "Path to agent") do |o|
|
20
25
|
options[:agent] = o
|
26
|
+
options[:type] = :actor
|
27
|
+
end
|
28
|
+
|
29
|
+
opts.on("-Q QUEUE", "--queue=QUEUE", "queue to use. default=default") do |o|
|
30
|
+
options[:queue] = o
|
21
31
|
end
|
22
32
|
|
23
33
|
opts.on("-t ID", "--identifier=ID", "Identifier") do |o|
|
@@ -51,32 +61,35 @@ rescue => e
|
|
51
61
|
exit 0
|
52
62
|
end
|
53
63
|
|
54
|
-
if
|
55
|
-
usage(optparser, "
|
64
|
+
if args.empty?
|
65
|
+
usage(optparser, "not enough arguments")
|
56
66
|
end
|
57
67
|
|
58
|
-
|
59
|
-
|
60
|
-
if Magent.current_actor.nil?
|
61
|
-
usage(optparser, "Use Magent.register(YourActor.new) to register an actor")
|
68
|
+
if options[:agent]
|
69
|
+
load options[:agent]
|
62
70
|
end
|
63
71
|
|
64
72
|
class Controller
|
65
|
-
|
73
|
+
attr_reader :options
|
74
|
+
|
75
|
+
def initialize(opts)
|
66
76
|
@options = opts
|
67
|
-
|
77
|
+
|
78
|
+
@queue = @options[:queue]
|
68
79
|
|
69
80
|
@options[:log_path] ||= Dir.getwd
|
70
81
|
|
71
|
-
@identity = @options[:identifier] || Magent::Utils.underscore(
|
82
|
+
@identity = @options[:identifier] || Magent::Utils.underscore(@options[:queue].to_s)
|
72
83
|
@identity << "-#{Socket.gethostname.split('.')[0]}"
|
84
|
+
|
85
|
+
$stderr.puts ">> Starting magent in #{@options[:type]} model"
|
73
86
|
end
|
74
87
|
|
75
88
|
def start
|
76
89
|
if @options[:daemonize] && @options[:piddir]
|
77
90
|
run_as_daemon
|
78
91
|
else
|
79
|
-
Magent::Processor.new(
|
92
|
+
Magent::Processor.new(self.channel).run!
|
80
93
|
end
|
81
94
|
end
|
82
95
|
|
@@ -104,6 +117,10 @@ class Controller
|
|
104
117
|
start
|
105
118
|
end
|
106
119
|
|
120
|
+
def channel
|
121
|
+
@channel ||= (@options[:type] == :async) ? Magent::AsyncChannel.new(@queue) : Magent::ActorChannel.new(@queue)
|
122
|
+
end
|
123
|
+
|
107
124
|
private
|
108
125
|
def run_as_daemon
|
109
126
|
daemonize
|
@@ -115,7 +132,7 @@ class Controller
|
|
115
132
|
end
|
116
133
|
end
|
117
134
|
|
118
|
-
Magent::Processor.new(
|
135
|
+
Magent::Processor.new(self.channel).run!
|
119
136
|
end
|
120
137
|
|
121
138
|
def pid_file
|
@@ -136,4 +153,4 @@ class Controller
|
|
136
153
|
end
|
137
154
|
end
|
138
155
|
|
139
|
-
Controller.new(
|
156
|
+
Controller.new(options).send(args.shift)
|
data/examples/comm/worker.rb
CHANGED
data/examples/error/error.rb
CHANGED
@@ -0,0 +1,41 @@
|
|
1
|
+
$:.unshift File.dirname(__FILE__)+"/../../lib/"
|
2
|
+
require 'rubygems'
|
3
|
+
require 'magent'
|
4
|
+
require 'mongo_mapper'
|
5
|
+
|
6
|
+
|
7
|
+
MongoMapper.database = "test"
|
8
|
+
Magent.database = "test"
|
9
|
+
|
10
|
+
class Thing
|
11
|
+
include MongoMapper::Document
|
12
|
+
include Magent::Async
|
13
|
+
|
14
|
+
key :_id, String
|
15
|
+
key :name
|
16
|
+
|
17
|
+
def process_something(arg)
|
18
|
+
puts "Processing: #{arg}"
|
19
|
+
end
|
20
|
+
|
21
|
+
def process_nothing
|
22
|
+
puts "Processing..."
|
23
|
+
end
|
24
|
+
|
25
|
+
def self.foo
|
26
|
+
puts "MAX PRIORITY"
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
|
31
|
+
thing = Thing.create(:_id => "foo")
|
32
|
+
|
33
|
+
# 3 messages
|
34
|
+
thing.async.process_something("testing").commit!
|
35
|
+
thing.async.process_nothing.commit!
|
36
|
+
Thing.async.find(thing.id).process_something("testing2").commit!
|
37
|
+
Thing.async.foo.commit!(1)
|
38
|
+
|
39
|
+
Magent::Processor.new(Magent::AsyncChannel.new(:default)).run!
|
40
|
+
|
41
|
+
|
data/examples/simple/bot.rb
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
$:.unshift File.dirname(__FILE__)+"/../../lib/"
|
2
|
+
require 'rubygems'
|
2
3
|
require 'magent'
|
3
4
|
|
4
5
|
# Use: magent /path/to/this/file
|
@@ -32,6 +33,6 @@ end
|
|
32
33
|
Magent.register(Bot.new)
|
33
34
|
|
34
35
|
if $0 == __FILE__
|
35
|
-
Magent::Processor.new(
|
36
|
+
Magent::Processor.new(Bot.channel).run!
|
36
37
|
end
|
37
38
|
|
data/examples/stats/stats.rb
CHANGED
data/lib/magent.rb
CHANGED
@@ -1,62 +1,114 @@
|
|
1
1
|
$:.unshift(File.dirname(__FILE__)) unless
|
2
2
|
$:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__)))
|
3
3
|
|
4
|
+
require 'rubygems'
|
4
5
|
require 'mongo'
|
5
6
|
require 'set'
|
6
7
|
require 'uuidtools'
|
7
8
|
|
9
|
+
require 'magent/failure'
|
10
|
+
|
8
11
|
require 'magent/utils'
|
9
12
|
require 'magent/generic_channel'
|
10
|
-
require 'magent/
|
13
|
+
require 'magent/actor_channel'
|
11
14
|
require 'magent/push'
|
12
15
|
require 'magent/actor'
|
13
16
|
require 'magent/processor'
|
14
17
|
|
18
|
+
require 'magent/async'
|
19
|
+
require 'magent/async_channel'
|
20
|
+
|
21
|
+
require 'magent/railtie' if defined?(Rails)
|
22
|
+
|
23
|
+
if defined?(EventMachine::WebSocket)
|
24
|
+
require 'magent/web_socket_server'
|
25
|
+
end
|
26
|
+
|
15
27
|
module Magent
|
16
|
-
|
28
|
+
@@database_name = "magent"
|
17
29
|
|
18
|
-
@@
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
@@password = nil
|
30
|
+
@@config = {
|
31
|
+
"host" => "0.0.0.0",
|
32
|
+
"port" => 27017
|
33
|
+
}
|
23
34
|
|
24
|
-
def self.
|
25
|
-
@@
|
26
|
-
@@port = port
|
35
|
+
def self.connection
|
36
|
+
@@connection ||= Mongo::Connection.new
|
27
37
|
end
|
28
38
|
|
29
|
-
def self.
|
30
|
-
|
39
|
+
def self.logger
|
40
|
+
connection.logger
|
31
41
|
end
|
32
42
|
|
33
|
-
def self.
|
34
|
-
@@
|
35
|
-
@@password = password
|
43
|
+
def self.connection=(new_connection)
|
44
|
+
@@connection = new_connection
|
36
45
|
end
|
37
46
|
|
38
|
-
def self.
|
39
|
-
@@
|
47
|
+
def self.database=(name)
|
48
|
+
@@database = nil
|
49
|
+
@@database_name = name
|
40
50
|
end
|
41
51
|
|
42
|
-
def self.
|
43
|
-
|
44
|
-
|
45
|
-
@@connection.add_auth(@@db_name, @@username, @@password) if @@username && @@password
|
52
|
+
def self.database
|
53
|
+
@@database ||= Magent.connection.db(@@database_name)
|
54
|
+
end
|
46
55
|
|
47
|
-
|
56
|
+
def self.config
|
57
|
+
@@config
|
48
58
|
end
|
49
59
|
|
50
|
-
def self.
|
51
|
-
@@
|
60
|
+
def self.config=(config)
|
61
|
+
@@config = config
|
52
62
|
end
|
53
63
|
|
54
|
-
def self.
|
55
|
-
|
64
|
+
def self.connect(environment, options={})
|
65
|
+
raise 'Set config before connecting. Magent.config = {...}' if config.blank?
|
66
|
+
|
67
|
+
env = config_for_environment(environment)
|
68
|
+
Magent.connection = Mongo::Connection.new(env['host'], env['port'], options)
|
69
|
+
Magent.database = env['database']
|
70
|
+
Magent.database.authenticate(env['username'], env['password']) if env['username'] && env['password']
|
56
71
|
end
|
57
72
|
|
58
|
-
def self.
|
59
|
-
|
73
|
+
def self.setup(config, environment = nil, options = {})
|
74
|
+
self.config = config
|
75
|
+
connect(environment, options)
|
76
|
+
end
|
77
|
+
|
78
|
+
# deprecated
|
79
|
+
def self.host=(host)
|
80
|
+
@@config['host'] = host
|
81
|
+
end
|
82
|
+
|
83
|
+
def self.port=(port)
|
84
|
+
@@config['port'] = port
|
85
|
+
end
|
86
|
+
|
87
|
+
def self.db_name=(db_name)
|
88
|
+
@@database_name = db_name
|
89
|
+
end
|
90
|
+
|
91
|
+
def self.auth(username, passwd)
|
92
|
+
@@config['username'] = username
|
93
|
+
@@config['password'] = passwd
|
94
|
+
end
|
95
|
+
|
96
|
+
private
|
97
|
+
def self.config_for_environment(environment)
|
98
|
+
env = environment ? config[environment] : config
|
99
|
+
|
100
|
+
return env if env['uri'].blank?
|
101
|
+
|
102
|
+
uri = URI.parse(env['uri'])
|
103
|
+
raise InvalidScheme.new('must be mongodb') unless uri.scheme == 'mongodb'
|
104
|
+
|
105
|
+
{
|
106
|
+
'host' => uri.host,
|
107
|
+
'port' => uri.port,
|
108
|
+
'database' => uri.path.gsub(/^\//, ''),
|
109
|
+
'username' => uri.user,
|
110
|
+
'password' => uri.password,
|
111
|
+
}
|
60
112
|
end
|
61
113
|
end
|
62
114
|
|