qrpc 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- data/.document +5 -0
- data/Gemfile +14 -0
- data/Gemfile.lock +36 -0
- data/LICENSE.txt +20 -0
- data/README.md +126 -0
- data/Rakefile +37 -0
- data/TODO.md +1 -0
- data/VERSION +1 -0
- data/lib/qrpc/locator.rb +101 -0
- data/lib/qrpc/server/dispatcher.rb +87 -0
- data/lib/qrpc/server/job.rb +145 -0
- data/lib/qrpc/server.rb +264 -0
- data/qrpc.gemspec +69 -0
- data/test-client.rb +29 -0
- data/test-server.rb +13 -0
- metadata +168 -0
data/.document
ADDED
data/Gemfile
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
source "http://rubygems.org"
|
2
|
+
# Add dependencies required to use your gem here.
|
3
|
+
# Example:
|
4
|
+
gem "json-rpc-objects", ">= 0.1.2"
|
5
|
+
gem "depq", ">= 0.4"
|
6
|
+
gem "em-beanstalk", ">= 0.0.10"
|
7
|
+
gem "eventmachine", ">= 0.12.10"
|
8
|
+
|
9
|
+
# Add dependencies to develop your gem here.
|
10
|
+
# Include everything needed to run rake, tests, features, etc.
|
11
|
+
group :development do
|
12
|
+
gem "bundler", "~> 1.0.0"
|
13
|
+
gem "jeweler", "~> 1.5.2"
|
14
|
+
end
|
data/Gemfile.lock
ADDED
@@ -0,0 +1,36 @@
|
|
1
|
+
GEM
|
2
|
+
remote: http://rubygems.org/
|
3
|
+
specs:
|
4
|
+
addressable (2.2.2)
|
5
|
+
depq (0.4)
|
6
|
+
em-beanstalk (0.0.10)
|
7
|
+
eventmachine
|
8
|
+
eventmachine (0.12.10)
|
9
|
+
git (1.2.5)
|
10
|
+
hash-utils (0.3.0)
|
11
|
+
jeweler (1.5.2)
|
12
|
+
bundler (~> 1.0.0)
|
13
|
+
git (>= 1.2.5)
|
14
|
+
rake
|
15
|
+
json-rpc-objects (0.1.3)
|
16
|
+
addressable (>= 2.2.2)
|
17
|
+
hash-utils (>= 0.3.0)
|
18
|
+
multitype-introspection (>= 0.1.0)
|
19
|
+
types (>= 0.1.0)
|
20
|
+
yajl-ruby (>= 0.7.8)
|
21
|
+
multitype-introspection (0.1.0)
|
22
|
+
rake (0.8.7)
|
23
|
+
types (0.1.0)
|
24
|
+
multitype-introspection (>= 0.1.0)
|
25
|
+
yajl-ruby (0.7.9)
|
26
|
+
|
27
|
+
PLATFORMS
|
28
|
+
ruby
|
29
|
+
|
30
|
+
DEPENDENCIES
|
31
|
+
bundler (~> 1.0.0)
|
32
|
+
depq (>= 0.4)
|
33
|
+
em-beanstalk (>= 0.0.10)
|
34
|
+
eventmachine (>= 0.12.10)
|
35
|
+
jeweler (~> 1.5.2)
|
36
|
+
json-rpc-objects (>= 0.1.2)
|
data/LICENSE.txt
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
Copyright (c) 2011 Martin Kozák
|
2
|
+
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
4
|
+
a copy of this software and associated documentation files (the
|
5
|
+
"Software"), to deal in the Software without restriction, including
|
6
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
7
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
8
|
+
permit persons to whom the Software is furnished to do so, subject to
|
9
|
+
the following conditions:
|
10
|
+
|
11
|
+
The above copyright notice and this permission notice shall be
|
12
|
+
included in all copies or substantial portions of the Software.
|
13
|
+
|
14
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
15
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
16
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
17
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
18
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
19
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
20
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,126 @@
|
|
1
|
+
QRPC
|
2
|
+
====
|
3
|
+
|
4
|
+
**QRPC** currently implements queued JSON-RPC server which works as
|
5
|
+
normal RPC server, but through queue interface, so allows highly
|
6
|
+
scalable, distributed and asynchronous remote API implementation and
|
7
|
+
fast data processing.
|
8
|
+
|
9
|
+
It's based on [eventmachine][1] and [beanstalkd][2] so it's fast and
|
10
|
+
thread safe.
|
11
|
+
|
12
|
+
### Protocol
|
13
|
+
|
14
|
+
It utilizes [JSON-RPC][3] protocol in versions both [1.1][4] and [2.0][5].
|
15
|
+
Adds special data member `qrpc` with few options appropriate for queue
|
16
|
+
processing. Typicall request looks in Ruby hash notation like:
|
17
|
+
|
18
|
+
{
|
19
|
+
"jsonrpc" => "2.0",
|
20
|
+
"method" => "subtract",
|
21
|
+
"params" => [2, 1],
|
22
|
+
"id" => <some unique job id>,
|
23
|
+
"qrpc" => {
|
24
|
+
"version" => "1.0",
|
25
|
+
"client" => <some unique client id>,
|
26
|
+
"priority" => 30
|
27
|
+
}
|
28
|
+
}
|
29
|
+
|
30
|
+
The last `priority` member is optional, others are expected to be
|
31
|
+
present including them which are optional in classic JSON-RPC.
|
32
|
+
Default priority is 50.
|
33
|
+
|
34
|
+
Typical response looks like:
|
35
|
+
|
36
|
+
{
|
37
|
+
"jsonrpc" => "2.0",
|
38
|
+
"result" => 1,
|
39
|
+
"id" => <some unique job id>,
|
40
|
+
"qrpc" => {
|
41
|
+
"version" => "1.0",
|
42
|
+
}
|
43
|
+
}
|
44
|
+
|
45
|
+
And in case of exception:
|
46
|
+
|
47
|
+
{
|
48
|
+
"jsonrpc" => "2.0",
|
49
|
+
"error" => {
|
50
|
+
"code" => <some code>,
|
51
|
+
"message" => <some message>,
|
52
|
+
"data" => {
|
53
|
+
"name" => <exception class name>,
|
54
|
+
"message" => <exception message>,
|
55
|
+
"backtrace" => <array of Base64 encoded strings>,
|
56
|
+
"dump" => {
|
57
|
+
"raw" => <Base 54 encoded marshaled exception object>,
|
58
|
+
"format" => "Ruby"
|
59
|
+
}
|
60
|
+
}
|
61
|
+
},
|
62
|
+
|
63
|
+
"id" => <some unique job id>,
|
64
|
+
"qrpc" => {
|
65
|
+
"version" => "1.0",
|
66
|
+
}
|
67
|
+
}
|
68
|
+
|
69
|
+
Both `backtrace` and `dump` members are optional.
|
70
|
+
|
71
|
+
|
72
|
+
### Usage
|
73
|
+
|
74
|
+
Usage is simple. Look example:
|
75
|
+
|
76
|
+
require "qrpc/server"
|
77
|
+
|
78
|
+
class Foo
|
79
|
+
def subtract(x, y)
|
80
|
+
x - y
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
server = QRPC::Server::new(Foo::new)
|
85
|
+
server.listen! QRPC::Locator::new("test")
|
86
|
+
|
87
|
+
This creates an instance of `Foo` which will serve as API, creates
|
88
|
+
locator of the queue *test* at default server *localhost:11300*. Queue
|
89
|
+
name will be remapped to the real name *qrpc-test-input*. After call to
|
90
|
+
`#listen!`, it will run eventmachine and start listening for calls. If
|
91
|
+
you want to run it inside already run eventmachine, simply call
|
92
|
+
`#start_listening` with the same parameters.
|
93
|
+
|
94
|
+
Calls processing is thread safe because of eventmachine concept
|
95
|
+
similar to fibers. Default number at one time processed jobs is 20,
|
96
|
+
but it can be changed by setting `:max_jobs => <number>` to `#listen!`
|
97
|
+
or `#start_listening`.
|
98
|
+
|
99
|
+
Reponse will be put to the same queue server, to queue named
|
100
|
+
`qrpc-<client identifier>-output`, with structure described above.
|
101
|
+
Client isn't implemented at this time.
|
102
|
+
|
103
|
+
Contributing
|
104
|
+
------------
|
105
|
+
|
106
|
+
1. Fork it.
|
107
|
+
2. Create a branch (`git checkout -b 20101220-my-change`).
|
108
|
+
3. Commit your changes (`git commit -am "Added something"`).
|
109
|
+
4. Push to the branch (`git push origin 20101220-my-change`).
|
110
|
+
5. Create an [Issue][6] with a link to your branch.
|
111
|
+
6. Enjoy a refreshing Diet Coke and wait.
|
112
|
+
|
113
|
+
|
114
|
+
Copyright
|
115
|
+
---------
|
116
|
+
|
117
|
+
Copyright © 2011 [Martin Kozák][7]. See `LICENSE.txt` for
|
118
|
+
further details.
|
119
|
+
|
120
|
+
[1]: http://rubyeventmachine.com/
|
121
|
+
[2]: http://kr.github.com/beanstalkd/
|
122
|
+
[3]: http://en.wikipedia.org/wiki/JSON-RPC
|
123
|
+
[4]: http://groups.google.com/group/json-rpc/web/json-rpc-1-1-alt
|
124
|
+
[5]: http://groups.google.com/group/json-rpc/web/json-rpc-2-0
|
125
|
+
[6]: http://github.com/martinkozak/qrpc/issues
|
126
|
+
[7]: http://www.martinkozak.net/
|
data/Rakefile
ADDED
@@ -0,0 +1,37 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require 'rubygems'
|
3
|
+
require 'bundler'
|
4
|
+
begin
|
5
|
+
Bundler.setup(:default, :development)
|
6
|
+
rescue Bundler::BundlerError => e
|
7
|
+
$stderr.puts e.message
|
8
|
+
$stderr.puts "Run `bundle install` to install missing gems"
|
9
|
+
exit e.status_code
|
10
|
+
end
|
11
|
+
require 'rake'
|
12
|
+
|
13
|
+
require 'jeweler'
|
14
|
+
Jeweler::Tasks.new do |gem|
|
15
|
+
# gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
|
16
|
+
gem.name = "qrpc"
|
17
|
+
gem.homepage = "http://github.com/martinkozak/qrpc"
|
18
|
+
gem.license = "MIT"
|
19
|
+
gem.summary = 'Queued JSON-RPC server. Works as normal RPC server, but through queue interface, so allows highly scalable, distributed and asynchronous remote API implementation and fast data processing. It\'s based on eventmachine and beanstalkd, so it\'s fast and thread safe.'
|
20
|
+
gem.email = "martinkozak@martinkozak.net"
|
21
|
+
gem.authors = ["Martin Kozák"]
|
22
|
+
# Include your dependencies below. Runtime dependencies are required when using your gem,
|
23
|
+
# and development dependencies are only needed for development (ie running rake tasks, tests, etc)
|
24
|
+
# gem.add_runtime_dependency 'jabber4r', '> 0.1'
|
25
|
+
# gem.add_development_dependency 'rspec', '> 1.2.3'
|
26
|
+
end
|
27
|
+
Jeweler::RubygemsDotOrgTasks.new
|
28
|
+
|
29
|
+
require 'rake/rdoctask'
|
30
|
+
Rake::RDocTask.new do |rdoc|
|
31
|
+
version = File.exist?('VERSION') ? File.read('VERSION') : ""
|
32
|
+
|
33
|
+
rdoc.rdoc_dir = 'rdoc'
|
34
|
+
rdoc.title = "qrpc #{version}"
|
35
|
+
rdoc.rdoc_files.include('README*')
|
36
|
+
rdoc.rdoc_files.include('lib/**/*.rb')
|
37
|
+
end
|
data/TODO.md
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
* general queue interface for ability to use more queue servers.
|
data/VERSION
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
0.1.0
|
data/lib/qrpc/locator.rb
ADDED
@@ -0,0 +1,101 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
|
3
|
+
|
4
|
+
##
|
5
|
+
# General QRPC module.
|
6
|
+
#
|
7
|
+
|
8
|
+
module QRPC
|
9
|
+
|
10
|
+
##
|
11
|
+
# Resource locator.
|
12
|
+
#
|
13
|
+
|
14
|
+
class Locator
|
15
|
+
|
16
|
+
##
|
17
|
+
# Contains queue name.
|
18
|
+
#
|
19
|
+
|
20
|
+
@queue
|
21
|
+
attr_accessor :queue
|
22
|
+
|
23
|
+
##
|
24
|
+
# Contains host.
|
25
|
+
#
|
26
|
+
|
27
|
+
@host
|
28
|
+
attr_accessor :host
|
29
|
+
|
30
|
+
##
|
31
|
+
# Contains port.
|
32
|
+
#
|
33
|
+
|
34
|
+
@port
|
35
|
+
attr_accessor :port
|
36
|
+
|
37
|
+
##
|
38
|
+
# Parser.
|
39
|
+
#
|
40
|
+
|
41
|
+
PARSER = /^(.+)@(.+)(?:\:(\d+))?$/
|
42
|
+
|
43
|
+
##
|
44
|
+
# Default port.
|
45
|
+
#
|
46
|
+
|
47
|
+
DEFAULT_PORT = 11300
|
48
|
+
|
49
|
+
##
|
50
|
+
# Constructor.
|
51
|
+
#
|
52
|
+
# @param [String, Symbol] queue queue name
|
53
|
+
# @param [String] host host name
|
54
|
+
# @param [Integer] port port of the host
|
55
|
+
#
|
56
|
+
|
57
|
+
def initialize(queue, host = "localhost", port = 11300)
|
58
|
+
@queue = queue.to_s
|
59
|
+
@host = host
|
60
|
+
@port = port
|
61
|
+
end
|
62
|
+
|
63
|
+
##
|
64
|
+
# Parses the locator.
|
65
|
+
# Excpects form <queue>@<host>:<port>. Port is optional.
|
66
|
+
#
|
67
|
+
# @param [String] string locator in string form
|
68
|
+
# @return [QRPC::Locator] new instance
|
69
|
+
#
|
70
|
+
|
71
|
+
def self.parse(string)
|
72
|
+
match = string.match(self::PARSER)
|
73
|
+
|
74
|
+
queue = match[1]
|
75
|
+
host = match[2]
|
76
|
+
|
77
|
+
if match.length == 3
|
78
|
+
port = match[3]
|
79
|
+
else
|
80
|
+
port = self::DEFAULT_PORT
|
81
|
+
end
|
82
|
+
|
83
|
+
port = port.to_i
|
84
|
+
|
85
|
+
##
|
86
|
+
|
87
|
+
return self::new(queue, host, port);
|
88
|
+
end
|
89
|
+
|
90
|
+
##
|
91
|
+
# Converts back to string.
|
92
|
+
# @return [String] locator in string form
|
93
|
+
#
|
94
|
+
|
95
|
+
def to_s
|
96
|
+
@queue.dup << "@" << @host << ":" << @port.to_s
|
97
|
+
end
|
98
|
+
|
99
|
+
end
|
100
|
+
|
101
|
+
end
|
@@ -0,0 +1,87 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "depq"
|
3
|
+
|
4
|
+
|
5
|
+
##
|
6
|
+
# General QRPC module.
|
7
|
+
#
|
8
|
+
|
9
|
+
module QRPC
|
10
|
+
class Server
|
11
|
+
|
12
|
+
##
|
13
|
+
# Queue RPC job.
|
14
|
+
#
|
15
|
+
|
16
|
+
class Dispatcher
|
17
|
+
|
18
|
+
##
|
19
|
+
# Holds running EM fibers count.
|
20
|
+
#
|
21
|
+
|
22
|
+
@count
|
23
|
+
|
24
|
+
##
|
25
|
+
# Holds unprocessed jobs queue.
|
26
|
+
#
|
27
|
+
|
28
|
+
@queue
|
29
|
+
|
30
|
+
##
|
31
|
+
# Holds max jobs count.
|
32
|
+
#
|
33
|
+
|
34
|
+
@max_jobs
|
35
|
+
|
36
|
+
##
|
37
|
+
# Constructor.
|
38
|
+
#
|
39
|
+
|
40
|
+
def initialize(max_jobs = 20)
|
41
|
+
@count = 0
|
42
|
+
@queue = Depq::new
|
43
|
+
@max_jobs = max_jobs
|
44
|
+
|
45
|
+
if @max_jobs.nil?
|
46
|
+
@max_jobs = 20
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
##
|
51
|
+
# Puts job to dispatcher.
|
52
|
+
# @param [QRPC::Server::Job] job job for dispatching
|
53
|
+
#
|
54
|
+
|
55
|
+
def put(job)
|
56
|
+
begin
|
57
|
+
@queue.put(job, job.priority)
|
58
|
+
rescue ::Exception => e
|
59
|
+
return
|
60
|
+
end
|
61
|
+
|
62
|
+
if @count < @max_jobs
|
63
|
+
self.process_next!
|
64
|
+
@count += 1
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
##
|
69
|
+
# Sets up next job for processing.
|
70
|
+
#
|
71
|
+
|
72
|
+
def process_next!
|
73
|
+
job = @queue.pop
|
74
|
+
job.callback do
|
75
|
+
if (@count < @max_jobs) and not @queue.empty?
|
76
|
+
self.process_next!
|
77
|
+
else
|
78
|
+
@count -= 1
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
job.process!
|
83
|
+
end
|
84
|
+
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
@@ -0,0 +1,145 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "eventmachine"
|
3
|
+
require "json-rpc-objects/request"
|
4
|
+
require "json-rpc-objects/response"
|
5
|
+
require "json-rpc-objects/error"
|
6
|
+
|
7
|
+
|
8
|
+
##
|
9
|
+
# General QRPC module.
|
10
|
+
#
|
11
|
+
|
12
|
+
module QRPC
|
13
|
+
class Server
|
14
|
+
|
15
|
+
##
|
16
|
+
# Queue RPC job.
|
17
|
+
#
|
18
|
+
|
19
|
+
class Job
|
20
|
+
include EM::Deferrable
|
21
|
+
|
22
|
+
##
|
23
|
+
# Indicates default priority.
|
24
|
+
#
|
25
|
+
|
26
|
+
DEFAULT_PRIORITY = 50
|
27
|
+
|
28
|
+
##
|
29
|
+
# Holds beanstalk job.
|
30
|
+
#
|
31
|
+
|
32
|
+
@job
|
33
|
+
|
34
|
+
##
|
35
|
+
# Holds JSON-RPC request.
|
36
|
+
#
|
37
|
+
|
38
|
+
@request
|
39
|
+
|
40
|
+
##
|
41
|
+
# Holds API object.
|
42
|
+
#
|
43
|
+
|
44
|
+
@api
|
45
|
+
|
46
|
+
##
|
47
|
+
# Constructor.
|
48
|
+
#
|
49
|
+
# @param [Object] object which will serve as API
|
50
|
+
# @param [EM::Beanstalk::Job] job beanstalk job
|
51
|
+
#
|
52
|
+
|
53
|
+
def initialize(api, job)
|
54
|
+
@api = api
|
55
|
+
@job = job
|
56
|
+
end
|
57
|
+
|
58
|
+
##
|
59
|
+
# Starts processing.
|
60
|
+
#
|
61
|
+
|
62
|
+
def process!
|
63
|
+
result = nil
|
64
|
+
error = nil
|
65
|
+
request = self.request
|
66
|
+
|
67
|
+
begin
|
68
|
+
result = @api.send(request.method, *request.params)
|
69
|
+
rescue ::Exception => e
|
70
|
+
error = self.generate_error(request, e)
|
71
|
+
end
|
72
|
+
|
73
|
+
response = request.class::version.response::create(result, error, :id => request.id)
|
74
|
+
response.qrpc = { :version => :"1.0" }
|
75
|
+
|
76
|
+
@job.delete()
|
77
|
+
self.set_deferred_status(:succeeded, response.to_json)
|
78
|
+
end
|
79
|
+
|
80
|
+
##
|
81
|
+
# Returns job in request form.
|
82
|
+
# @return [JsonRpcObjects::Generic::Object] request associated to job
|
83
|
+
#
|
84
|
+
|
85
|
+
def request
|
86
|
+
if @request.nil?
|
87
|
+
@request = JsonRpcObjects::Request::parse(@job.body)
|
88
|
+
end
|
89
|
+
|
90
|
+
return @request
|
91
|
+
end
|
92
|
+
|
93
|
+
##
|
94
|
+
# Returns job priority according to request.
|
95
|
+
#
|
96
|
+
# Default priority is 50. You can scale up and down according
|
97
|
+
# to your needs in fact without limits.
|
98
|
+
#
|
99
|
+
# @return [Integer] priority level
|
100
|
+
#
|
101
|
+
|
102
|
+
def priority
|
103
|
+
priority = self.request.qrpc["priority"]
|
104
|
+
if priority.nil?
|
105
|
+
priority = self.class::DEFAULT_PRIORITY
|
106
|
+
else
|
107
|
+
priority = priority.to_i
|
108
|
+
end
|
109
|
+
|
110
|
+
return priority
|
111
|
+
end
|
112
|
+
|
113
|
+
##
|
114
|
+
# Returns client identifier.
|
115
|
+
# @return [String] client identifier
|
116
|
+
#
|
117
|
+
|
118
|
+
def client
|
119
|
+
self.request.qrpc["client"]
|
120
|
+
end
|
121
|
+
|
122
|
+
|
123
|
+
protected
|
124
|
+
|
125
|
+
##
|
126
|
+
# Generates error from exception.
|
127
|
+
#
|
128
|
+
|
129
|
+
def generate_error(request, exception)
|
130
|
+
data = {
|
131
|
+
:name => exception.class.name,
|
132
|
+
:message => exception.message,
|
133
|
+
:backtrace => exception.backtrace.map { |s| Base64.encode64(s) },
|
134
|
+
:dump => {
|
135
|
+
:raw => Base64.encode64(Marshal.dump(exception)),
|
136
|
+
:format => :Ruby,
|
137
|
+
}
|
138
|
+
}
|
139
|
+
|
140
|
+
request.class::version.error::create(100, "exception raised during processing the request", :error => data)
|
141
|
+
end
|
142
|
+
|
143
|
+
end
|
144
|
+
end
|
145
|
+
end
|
data/lib/qrpc/server.rb
ADDED
@@ -0,0 +1,264 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "qrpc/server/job"
|
3
|
+
require "qrpc/server/dispatcher"
|
4
|
+
require "qrpc/locator"
|
5
|
+
require "em-beanstalk"
|
6
|
+
require "eventmachine"
|
7
|
+
require "base64"
|
8
|
+
|
9
|
+
|
10
|
+
##
|
11
|
+
# General QRPC module.
|
12
|
+
#
|
13
|
+
|
14
|
+
module QRPC
|
15
|
+
|
16
|
+
##
|
17
|
+
# Queue RPC server.
|
18
|
+
#
|
19
|
+
|
20
|
+
class Server
|
21
|
+
|
22
|
+
##
|
23
|
+
# Prefix for handled queues.
|
24
|
+
#
|
25
|
+
|
26
|
+
QRPC_PREFIX = "qrpc"
|
27
|
+
|
28
|
+
##
|
29
|
+
# Input queue postfix.
|
30
|
+
#
|
31
|
+
|
32
|
+
QRPC_POSTFIX_INPUT = "input"
|
33
|
+
|
34
|
+
##
|
35
|
+
# Output queue postfix.
|
36
|
+
#
|
37
|
+
|
38
|
+
QRPC_POSTFIX_OUTPUT = "output"
|
39
|
+
|
40
|
+
##
|
41
|
+
# Holds API instance.
|
42
|
+
#
|
43
|
+
|
44
|
+
@api
|
45
|
+
|
46
|
+
##
|
47
|
+
# Holds input locator.
|
48
|
+
#
|
49
|
+
|
50
|
+
@locator
|
51
|
+
|
52
|
+
##
|
53
|
+
# Holds output queue name.
|
54
|
+
#
|
55
|
+
|
56
|
+
@output_name
|
57
|
+
|
58
|
+
##
|
59
|
+
# Holds input queue instance.
|
60
|
+
#
|
61
|
+
|
62
|
+
@input_queue
|
63
|
+
|
64
|
+
##
|
65
|
+
# Holds output queue instance.
|
66
|
+
#
|
67
|
+
|
68
|
+
@output_queue
|
69
|
+
|
70
|
+
##
|
71
|
+
# Holds job dispatcher.
|
72
|
+
#
|
73
|
+
|
74
|
+
@dispatcher
|
75
|
+
|
76
|
+
##
|
77
|
+
# Cache of output names.
|
78
|
+
#
|
79
|
+
|
80
|
+
@output_name_cache
|
81
|
+
|
82
|
+
##
|
83
|
+
# Indicates currently used output queue.
|
84
|
+
#
|
85
|
+
|
86
|
+
@output_used
|
87
|
+
|
88
|
+
##
|
89
|
+
# Holds servers for finalizing.
|
90
|
+
#
|
91
|
+
|
92
|
+
@@servers = { }
|
93
|
+
|
94
|
+
##
|
95
|
+
# Constructor.
|
96
|
+
# @param [Object] api some object which will be used as RPC API
|
97
|
+
#
|
98
|
+
|
99
|
+
def initialize(api)
|
100
|
+
@api = api
|
101
|
+
@output_name_cache = { }
|
102
|
+
|
103
|
+
# Destructor
|
104
|
+
ObjectSpace.define_finalizer(self, self.class.method(:finalize).to_proc)
|
105
|
+
@@servers[self.object_id] = self
|
106
|
+
end
|
107
|
+
|
108
|
+
##
|
109
|
+
# Finalizer handler.
|
110
|
+
# @param [Integer] id id of finalized instance
|
111
|
+
#
|
112
|
+
|
113
|
+
def self.finalize(id)
|
114
|
+
if @@servers.has_key? id
|
115
|
+
@@servers[id].finalize!
|
116
|
+
end
|
117
|
+
end
|
118
|
+
|
119
|
+
##
|
120
|
+
# Destructor.
|
121
|
+
#
|
122
|
+
|
123
|
+
def finalize!
|
124
|
+
if @input_queue
|
125
|
+
@input_queue.watch("default") do
|
126
|
+
@input_queue.ignore(@input_name) do
|
127
|
+
@input_queue.close
|
128
|
+
end
|
129
|
+
end
|
130
|
+
end
|
131
|
+
|
132
|
+
if @output_queue
|
133
|
+
@output_queue.use("default") do
|
134
|
+
@output_queue.close
|
135
|
+
end
|
136
|
+
end
|
137
|
+
end
|
138
|
+
|
139
|
+
|
140
|
+
##
|
141
|
+
# Listens to the queue.
|
142
|
+
# (Blocking call which starts eventmachine.)
|
143
|
+
#
|
144
|
+
# @param [QRPC::Locator] locator of the input queue
|
145
|
+
# @param [Hash] opts options for the server
|
146
|
+
#
|
147
|
+
|
148
|
+
def listen!(locator, opts = { })
|
149
|
+
EM.run do
|
150
|
+
self.start_listening(locator, opts)
|
151
|
+
end
|
152
|
+
end
|
153
|
+
|
154
|
+
##
|
155
|
+
# Starts listening to the queue.
|
156
|
+
# (Blocking queue which expect, eventmachine is started.)
|
157
|
+
#
|
158
|
+
# @param [QRPC::Locator] locator of the input queue
|
159
|
+
# @param [Hash] opts options for the server
|
160
|
+
#
|
161
|
+
|
162
|
+
def start_listening(locator, opts)
|
163
|
+
@locator = locator
|
164
|
+
@locator.queue = self.class::QRPC_PREFIX.dup << "-" << @locator.queue << "-" << self.class::QRPC_POSTFIX_INPUT
|
165
|
+
@dispatcher = QRPC::Server::Dispatcher::new(opts[:max_jobs])
|
166
|
+
|
167
|
+
# Cache cleaning dispatcher
|
168
|
+
EM.add_periodic_timer(20) do
|
169
|
+
@output_name_cache.clear
|
170
|
+
end
|
171
|
+
|
172
|
+
# Process input queue
|
173
|
+
self.input_queue do |queue|
|
174
|
+
queue.each_job do |job|
|
175
|
+
self.process_job(job)
|
176
|
+
end
|
177
|
+
end
|
178
|
+
end
|
179
|
+
|
180
|
+
##
|
181
|
+
#
|
182
|
+
|
183
|
+
##
|
184
|
+
# Returns input queue.
|
185
|
+
# (Callable from EM only.)
|
186
|
+
#
|
187
|
+
# @param [Proc] block block to which will be input queue given
|
188
|
+
#
|
189
|
+
|
190
|
+
def input_queue(&block)
|
191
|
+
if not @input_queue
|
192
|
+
@input_queue = EM::Beanstalk::new(:host => @locator.host, :port => @locator.port)
|
193
|
+
@input_queue.watch(@locator.queue) do
|
194
|
+
@input_queue.ignore("default") do
|
195
|
+
block.call(@input_queue)
|
196
|
+
end
|
197
|
+
end
|
198
|
+
else
|
199
|
+
block.call(@input_queue)
|
200
|
+
end
|
201
|
+
end
|
202
|
+
|
203
|
+
##
|
204
|
+
# Returns output queue.
|
205
|
+
# (Callable from EM only.)
|
206
|
+
#
|
207
|
+
# @return [EM::Beanstalk] output queue Beanstalk connection
|
208
|
+
#
|
209
|
+
|
210
|
+
def output_queue
|
211
|
+
if not @output_queue
|
212
|
+
@output_queue = EM::Beanstalk::new(:host => @locator.host, :port => @locator.port)
|
213
|
+
end
|
214
|
+
|
215
|
+
return @output_queue
|
216
|
+
end
|
217
|
+
|
218
|
+
##
|
219
|
+
# Returns output name for client name.
|
220
|
+
#
|
221
|
+
# @param [String, Symbol] client client identifier
|
222
|
+
# @return [Symbol] output name
|
223
|
+
#
|
224
|
+
|
225
|
+
def output_name(client)
|
226
|
+
client_index = client.to_sym
|
227
|
+
|
228
|
+
if not @output_name_cache.include? client_index
|
229
|
+
output_name = self.class::QRPC_PREFIX.dup << "-" << client.to_s << "-" << self.class::QRPC_POSTFIX_OUTPUT
|
230
|
+
output_name = output_name.to_sym
|
231
|
+
@output_name_cache[client_index] = output_name
|
232
|
+
else
|
233
|
+
output_name = @output_name_cache[client_index]
|
234
|
+
end
|
235
|
+
|
236
|
+
return output_name
|
237
|
+
end
|
238
|
+
|
239
|
+
|
240
|
+
protected
|
241
|
+
|
242
|
+
##
|
243
|
+
# Process one job.
|
244
|
+
#
|
245
|
+
|
246
|
+
def process_job(job)
|
247
|
+
our_job = QRPC::Server::Job::new(@api, job)
|
248
|
+
our_job.callback do |result|
|
249
|
+
call = Proc::new { self.output_queue.put(result, :priority => our_job.priority) }
|
250
|
+
output_name = self.output_name(our_job.client)
|
251
|
+
|
252
|
+
if @output_used != output_name
|
253
|
+
@output_used = output_name
|
254
|
+
self.output_queue.use(output_name.to_s, &call)
|
255
|
+
else
|
256
|
+
call.call
|
257
|
+
end
|
258
|
+
end
|
259
|
+
|
260
|
+
@dispatcher.put(our_job)
|
261
|
+
end
|
262
|
+
|
263
|
+
end
|
264
|
+
end
|
data/qrpc.gemspec
ADDED
@@ -0,0 +1,69 @@
|
|
1
|
+
# Generated by jeweler
|
2
|
+
# DO NOT EDIT THIS FILE DIRECTLY
|
3
|
+
# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
|
4
|
+
# -*- encoding: utf-8 -*-
|
5
|
+
|
6
|
+
Gem::Specification.new do |s|
|
7
|
+
s.name = %q{qrpc}
|
8
|
+
s.version = "0.1.0"
|
9
|
+
|
10
|
+
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
|
+
s.authors = ["Martin Kozák"]
|
12
|
+
s.date = %q{2011-01-18}
|
13
|
+
s.email = %q{martinkozak@martinkozak.net}
|
14
|
+
s.extra_rdoc_files = [
|
15
|
+
"LICENSE.txt",
|
16
|
+
"README.md"
|
17
|
+
]
|
18
|
+
s.files = [
|
19
|
+
".document",
|
20
|
+
"Gemfile",
|
21
|
+
"Gemfile.lock",
|
22
|
+
"LICENSE.txt",
|
23
|
+
"README.md",
|
24
|
+
"Rakefile",
|
25
|
+
"TODO.md",
|
26
|
+
"VERSION",
|
27
|
+
"lib/qrpc/locator.rb",
|
28
|
+
"lib/qrpc/server.rb",
|
29
|
+
"lib/qrpc/server/dispatcher.rb",
|
30
|
+
"lib/qrpc/server/job.rb",
|
31
|
+
"qrpc.gemspec",
|
32
|
+
"test-client.rb",
|
33
|
+
"test-server.rb"
|
34
|
+
]
|
35
|
+
s.homepage = %q{http://github.com/martinkozak/qrpc}
|
36
|
+
s.licenses = ["MIT"]
|
37
|
+
s.require_paths = ["lib"]
|
38
|
+
s.rubygems_version = %q{1.3.7}
|
39
|
+
s.summary = %q{Queued JSON-RPC server. Works as normal RPC server, but through queue interface, so allows highly scalable, distributed and asynchronous remote API implementation and fast data processing. It's based on eventmachine and beanstalkd, so it's fast and thread safe.}
|
40
|
+
|
41
|
+
if s.respond_to? :specification_version then
|
42
|
+
current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
|
43
|
+
s.specification_version = 3
|
44
|
+
|
45
|
+
if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
|
46
|
+
s.add_runtime_dependency(%q<json-rpc-objects>, [">= 0.1.2"])
|
47
|
+
s.add_runtime_dependency(%q<depq>, [">= 0.4"])
|
48
|
+
s.add_runtime_dependency(%q<em-beanstalk>, [">= 0.0.10"])
|
49
|
+
s.add_runtime_dependency(%q<eventmachine>, [">= 0.12.10"])
|
50
|
+
s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
|
51
|
+
s.add_development_dependency(%q<jeweler>, ["~> 1.5.2"])
|
52
|
+
else
|
53
|
+
s.add_dependency(%q<json-rpc-objects>, [">= 0.1.2"])
|
54
|
+
s.add_dependency(%q<depq>, [">= 0.4"])
|
55
|
+
s.add_dependency(%q<em-beanstalk>, [">= 0.0.10"])
|
56
|
+
s.add_dependency(%q<eventmachine>, [">= 0.12.10"])
|
57
|
+
s.add_dependency(%q<bundler>, ["~> 1.0.0"])
|
58
|
+
s.add_dependency(%q<jeweler>, ["~> 1.5.2"])
|
59
|
+
end
|
60
|
+
else
|
61
|
+
s.add_dependency(%q<json-rpc-objects>, [">= 0.1.2"])
|
62
|
+
s.add_dependency(%q<depq>, [">= 0.4"])
|
63
|
+
s.add_dependency(%q<em-beanstalk>, [">= 0.0.10"])
|
64
|
+
s.add_dependency(%q<eventmachine>, [">= 0.12.10"])
|
65
|
+
s.add_dependency(%q<bundler>, ["~> 1.0.0"])
|
66
|
+
s.add_dependency(%q<jeweler>, ["~> 1.5.2"])
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
data/test-client.rb
ADDED
@@ -0,0 +1,29 @@
|
|
1
|
+
# encoding: utf-8
|
2
|
+
require "beanstalk-client"
|
3
|
+
require "json-rpc-objects/request"
|
4
|
+
|
5
|
+
b = Beanstalk::Pool::new(["localhost:11300"])
|
6
|
+
req1 = JsonRpcObjects::Request::create(:subtract, [2, 3], :id => "job1", :qrpc => { :version => "1.0", :client => :cc })
|
7
|
+
req2 = JsonRpcObjects::Request::create(:something_bad, nil, :id => "job2", :qrpc => { :version => "1.0", :client => :cc, :priority => 20 })
|
8
|
+
|
9
|
+
b.use("qrpc-test-input")
|
10
|
+
b.watch("qrpc-cc-output")
|
11
|
+
b.put(req1.to_json)
|
12
|
+
b.put(req2.to_json)
|
13
|
+
|
14
|
+
job = b.reserve
|
15
|
+
puts job.body
|
16
|
+
job.delete
|
17
|
+
|
18
|
+
job = b.reserve
|
19
|
+
puts job.body
|
20
|
+
job.delete
|
21
|
+
|
22
|
+
=begin
|
23
|
+
100.times do
|
24
|
+
b.put(req1.to_json)
|
25
|
+
job = b.reserve
|
26
|
+
puts job.body
|
27
|
+
job.delete
|
28
|
+
end
|
29
|
+
=end
|
data/test-server.rb
ADDED
metadata
ADDED
@@ -0,0 +1,168 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: qrpc
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
prerelease: false
|
5
|
+
segments:
|
6
|
+
- 0
|
7
|
+
- 1
|
8
|
+
- 0
|
9
|
+
version: 0.1.0
|
10
|
+
platform: ruby
|
11
|
+
authors:
|
12
|
+
- "Martin Koz\xC3\xA1k"
|
13
|
+
autorequire:
|
14
|
+
bindir: bin
|
15
|
+
cert_chain: []
|
16
|
+
|
17
|
+
date: 2011-01-18 00:00:00 +01:00
|
18
|
+
default_executable:
|
19
|
+
dependencies:
|
20
|
+
- !ruby/object:Gem::Dependency
|
21
|
+
name: json-rpc-objects
|
22
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
23
|
+
none: false
|
24
|
+
requirements:
|
25
|
+
- - ">="
|
26
|
+
- !ruby/object:Gem::Version
|
27
|
+
segments:
|
28
|
+
- 0
|
29
|
+
- 1
|
30
|
+
- 2
|
31
|
+
version: 0.1.2
|
32
|
+
type: :runtime
|
33
|
+
prerelease: false
|
34
|
+
version_requirements: *id001
|
35
|
+
- !ruby/object:Gem::Dependency
|
36
|
+
name: depq
|
37
|
+
requirement: &id002 !ruby/object:Gem::Requirement
|
38
|
+
none: false
|
39
|
+
requirements:
|
40
|
+
- - ">="
|
41
|
+
- !ruby/object:Gem::Version
|
42
|
+
segments:
|
43
|
+
- 0
|
44
|
+
- 4
|
45
|
+
version: "0.4"
|
46
|
+
type: :runtime
|
47
|
+
prerelease: false
|
48
|
+
version_requirements: *id002
|
49
|
+
- !ruby/object:Gem::Dependency
|
50
|
+
name: em-beanstalk
|
51
|
+
requirement: &id003 !ruby/object:Gem::Requirement
|
52
|
+
none: false
|
53
|
+
requirements:
|
54
|
+
- - ">="
|
55
|
+
- !ruby/object:Gem::Version
|
56
|
+
segments:
|
57
|
+
- 0
|
58
|
+
- 0
|
59
|
+
- 10
|
60
|
+
version: 0.0.10
|
61
|
+
type: :runtime
|
62
|
+
prerelease: false
|
63
|
+
version_requirements: *id003
|
64
|
+
- !ruby/object:Gem::Dependency
|
65
|
+
name: eventmachine
|
66
|
+
requirement: &id004 !ruby/object:Gem::Requirement
|
67
|
+
none: false
|
68
|
+
requirements:
|
69
|
+
- - ">="
|
70
|
+
- !ruby/object:Gem::Version
|
71
|
+
segments:
|
72
|
+
- 0
|
73
|
+
- 12
|
74
|
+
- 10
|
75
|
+
version: 0.12.10
|
76
|
+
type: :runtime
|
77
|
+
prerelease: false
|
78
|
+
version_requirements: *id004
|
79
|
+
- !ruby/object:Gem::Dependency
|
80
|
+
name: bundler
|
81
|
+
requirement: &id005 !ruby/object:Gem::Requirement
|
82
|
+
none: false
|
83
|
+
requirements:
|
84
|
+
- - ~>
|
85
|
+
- !ruby/object:Gem::Version
|
86
|
+
segments:
|
87
|
+
- 1
|
88
|
+
- 0
|
89
|
+
- 0
|
90
|
+
version: 1.0.0
|
91
|
+
type: :development
|
92
|
+
prerelease: false
|
93
|
+
version_requirements: *id005
|
94
|
+
- !ruby/object:Gem::Dependency
|
95
|
+
name: jeweler
|
96
|
+
requirement: &id006 !ruby/object:Gem::Requirement
|
97
|
+
none: false
|
98
|
+
requirements:
|
99
|
+
- - ~>
|
100
|
+
- !ruby/object:Gem::Version
|
101
|
+
segments:
|
102
|
+
- 1
|
103
|
+
- 5
|
104
|
+
- 2
|
105
|
+
version: 1.5.2
|
106
|
+
type: :development
|
107
|
+
prerelease: false
|
108
|
+
version_requirements: *id006
|
109
|
+
description:
|
110
|
+
email: martinkozak@martinkozak.net
|
111
|
+
executables: []
|
112
|
+
|
113
|
+
extensions: []
|
114
|
+
|
115
|
+
extra_rdoc_files:
|
116
|
+
- LICENSE.txt
|
117
|
+
- README.md
|
118
|
+
files:
|
119
|
+
- .document
|
120
|
+
- Gemfile
|
121
|
+
- Gemfile.lock
|
122
|
+
- LICENSE.txt
|
123
|
+
- README.md
|
124
|
+
- Rakefile
|
125
|
+
- TODO.md
|
126
|
+
- VERSION
|
127
|
+
- lib/qrpc/locator.rb
|
128
|
+
- lib/qrpc/server.rb
|
129
|
+
- lib/qrpc/server/dispatcher.rb
|
130
|
+
- lib/qrpc/server/job.rb
|
131
|
+
- qrpc.gemspec
|
132
|
+
- test-client.rb
|
133
|
+
- test-server.rb
|
134
|
+
has_rdoc: true
|
135
|
+
homepage: http://github.com/martinkozak/qrpc
|
136
|
+
licenses:
|
137
|
+
- MIT
|
138
|
+
post_install_message:
|
139
|
+
rdoc_options: []
|
140
|
+
|
141
|
+
require_paths:
|
142
|
+
- lib
|
143
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
144
|
+
none: false
|
145
|
+
requirements:
|
146
|
+
- - ">="
|
147
|
+
- !ruby/object:Gem::Version
|
148
|
+
hash: -3341094115904611072
|
149
|
+
segments:
|
150
|
+
- 0
|
151
|
+
version: "0"
|
152
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
153
|
+
none: false
|
154
|
+
requirements:
|
155
|
+
- - ">="
|
156
|
+
- !ruby/object:Gem::Version
|
157
|
+
segments:
|
158
|
+
- 0
|
159
|
+
version: "0"
|
160
|
+
requirements: []
|
161
|
+
|
162
|
+
rubyforge_project:
|
163
|
+
rubygems_version: 1.3.7
|
164
|
+
signing_key:
|
165
|
+
specification_version: 3
|
166
|
+
summary: Queued JSON-RPC server. Works as normal RPC server, but through queue interface, so allows highly scalable, distributed and asynchronous remote API implementation and fast data processing. It's based on eventmachine and beanstalkd, so it's fast and thread safe.
|
167
|
+
test_files: []
|
168
|
+
|