qrpc 0.4.0 → 0.9.0

data/CHANGES.txt

@@ -1,9 +1,11 @@
 
-0.4.0
-* #precall and #postcall handler support on API object
-* missing default value for options in Server#start_listening
-* in some responses ID isn't available, but tried to be converted to 
-  the symbol
+0.9.0 (2012-05-24)
+* transfer agents generic support
+* protocols generic support
+* ID generators generic support
+* serializers generic support
+* server synchronicity option support
+* substantial code reorganization
 
 0.3.2 (2011-08-28)
 * UUID of the client now generated in compact format
@@ -13,7 +15,8 @@
 * UUID of the jobs now generated in compact format
 
 0.3.0 (2011-02-27)
-* invalid json-rpc-object dependency
+* bad json-rpc-object dependency (but noncritical, some special cases
+    performance lag only)
 * all public methods from Client moved to Client::Dispatcher
 
 0.2.2 (2011-02-13)

data/Gemfile

@@ -1,11 +1,17 @@
 source "http://rubygems.org"
 # Add dependencies required to use your gem here.
 # Example:
-   gem "json-rpc-objects", ">= 0.3.3"
-   gem "depq", ">= 0.4"
-   gem "em-jack", ">= 0.1.3"
+   gem "json-rpc-objects", ">= 0.4.1"
+   gem "PriorityQueue", ">= 0.1.2"
+   gem "unified-queues", ">= 0"
+   gem "evented-queue", ">= 0"
    gem "eventmachine", ">= 0"
-   gem "uuid", ">= 2.3.2"
+   gem "hash-utils", ">= 1.1.0"
+   gem "em-wrapper", ">= 0"
+   gem "abstract", ">= 0"
+   gem "em-jack", ">= 0.1.3"
+   gem "hashie", ">= 1.0.0"
+   gem "uuid", ">= 2.0.0"
 
 # Add dependencies to develop your gem here.
 # Include everything needed to run rake, tests, features, etc.

data/Gemfile.lock

@@ -1,29 +1,45 @@
 GEM
   remote: http://rubygems.org/
   specs:
+    PriorityQueue (0.1.2)
     abstract (1.0.0)
-    addressable (2.2.6)
-    depq (0.6)
+    addressable (2.2.8)
     em-jack (0.1.5)
       eventmachine (>= 0.12.10)
-    eventmachine (1.0.0.beta.4)
+    em-wrapper (0.1.0)
+      eventmachine
+      hash-utils (>= 0.17.0)
+      object-proxy (>= 0.3.0)
+    evented-queue (0.1.0)
+      abstract
+      unified-queues
+    eventmachine (0.12.10)
     git (1.2.5)
-    hash-utils (1.1.0)
+    hash-utils (2.0.0)
       ruby-version
+    hashie (1.2.0)
     jeweler2 (2.0.9)
       git (>= 1.2.5)
-    json-rpc-objects (0.4.3)
+    json-rpc-objects (0.4.2)
       abstract (>= 1.0.0)
       addressable (>= 2.2.2)
       hash-utils (>= 0.18.0)
       json-rpc-objects-json (>= 0.1.1)
     json-rpc-objects-json (0.1.1)
       multi_json
-    macaddr (1.5.0)
-      systemu (>= 2.4.0)
-    multi_json (1.0.4)
+    lookup-hash (0.2.0)
+      hash-utils (>= 0.11.0)
+    macaddr (1.6.0)
+      systemu (~> 2.2.0)
+    multi_json (1.3.5)
+    object-proxy (0.3.0)
+      hash-utils (>= 0.12.1)
     ruby-version (0.3.1)
-    systemu (2.4.2)
+    systemu (2.2.0)
+    unified-queues (0.1.0)
+      abstract (>= 1.0.0)
+      hash-utils (>= 0.18.1)
+      lookup-hash
     uuid (2.3.5)
       macaddr (~> 1.0)
 
@@ -31,10 +47,16 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
+  PriorityQueue (>= 0.1.2)
+  abstract
   bundler (>= 1.0.0)
-  depq (>= 0.4)
   em-jack (>= 0.1.3)
+  em-wrapper
+  evented-queue
   eventmachine
+  hash-utils (>= 1.1.0)
+  hashie (>= 1.0.0)
   jeweler2 (>= 2.0.0)
-  json-rpc-objects (>= 0.3.3)
-  uuid (>= 2.3.2)
+  json-rpc-objects (>= 0.4.1)
+  unified-queues
+  uuid (>= 2.0.0)

data/LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2011 Martin Kozák
+Copyright (c) 2011 - 2012 Martin Kozák (martinkozak@martinkozak.net)
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/README.md

@@ -6,14 +6,15 @@ which works as normal RPC server, but through queue interface, so allows
 highly scalable, distributed and asynchronous remote API implementation 
 and fast data processing. 
 
-It's based on [eventmachine][1] and [beanstalkd][2] so it's fast and 
-thread safe. 
+It's based on [eventmachine][1] and the default implementation uses 
+[beanstalkd][2] so it's fast and thread safe. 
 
 ### Protocol
 
-It utilizes [JSON-RPC][3] protocol in versions both [1.1][4] and [2.0][5].
-Adds special data member `qrpc` with few options appropriate for queue 
-processing. Typicall request looks in Ruby hash notation like:
+It utilizes [JSON-RPC][3] protocol by default in versions both [1.1][4] 
+and [2.0][5] altthough different protocols can be implemented. Adds special 
+data member `qrpc` with few options appropriate for queue processing. 
+Typicall request looks in Ruby hash notation like:
     
     {
         "jsonrpc" => "2.0",
@@ -54,7 +55,7 @@ And in case of exception:
                 "message" => <exception message>,
                 "backtrace" => <array of Base64 encoded strings>,
                 "dump" => {
-                    "raw" => <Base 54 encoded marshaled exception object>,
+                    "raw" => <Base64 encoded marshaled exception object>,
                     "format" => "ruby"
                 }
             }
@@ -68,13 +69,61 @@ And in case of exception:
     
 Both `backtrace` and `dump` members are optional.
 
+### Variety of Platforms
+
+#### Transfer Agents (“Queues”)
+
+QRPC uses the [Unified Queues][10] interface which ensures connecting
+to different queue interfaces using an unified API. It can be used on all 
+platforms which have unified queues driver available by this way. By default,
+two of them are available:
+
+* [beanstalkd][2] queue protocol,
+* 8 different queues-like in-memory objects (in fact, all priority queue 
+implementations available for Ruby to date).
+
+See the [Unified Queues][10] documentation.
+
+#### Communication Protocols (“Protocols”)
+
+Different protocols can be implemented provided that it will provide the 
+required API. By default, two of them are available:
+
+* [JSON-RPC][3] protocol (default, see description above),
+* object protocol (in fact, the pure Ruby objects for in-memory communication) 
+
+#### Identification Generators (“Generators”)
+
+The request identificators can be generated by various ways. It's possible
+to implement an own generator so have benefits for logging or in corporate 
+use. By default two  generators are available:
+
+* [UUID][11] generator (default),
+* [Ruby object ID][12] generator (fast, intended for in-memory communication in 
+single Ruby instance).
+
+#### Encoding Agents (“Serializers”)
+    
+The JSON-RPC protocol is intended for use with the [JSON][13] encoding, but can 
+be encoded by various other methods. Various serializers support is part of the 
+[JSON-RPC Objects][13] gem. See its documentation. The following protocols are 
+supported for example, to date:
+
+* [JSON][14]
+* [BSON][15]
+* [YAML][16]
+* [MessagePack][17]
+ 
     
 ### Server Usage
 
-Usage is simple. Look example:
+Usage is simple. Look example of *Beanstalked* *JSON-RPC* service encoded 
+by the *MsgPack* format working *synchronously*:
 
-    require "qrpc/server"
-    require "qrpc/locator"
+    require "qrpc/server"                           # server
+    require "qrpc/locator/em-jack"                  # queue transfer agent
+    require "qrpc/protocol/json-rpc"                # RPC protocol
+    require "json-rpc-objects/serializer/msgpack"   # serializer
     
     class Foo
         def subtract(x, y)
@@ -82,8 +131,12 @@ Usage is simple. Look example:
         end
     end
 
-    server = QRPC::Server::new Foo::new
-    server.listen! QRPC::Locator::new "test"
+    serializer = JsonRpcObjects::Serializer::MsgPack::new
+    protocol = QRPC::Protocol::JsonRpc::new(:serializer => serializer)
+    locator = QRPC::Locator::EMJackLocator::new("test")
+
+    server = QRPC::Server::new(Foo::new, :synchronous, protocol)
+    server.listen! locator
     
 This creates an instance of `Foo` which will serve as API, creates
 locator of the queue *test* at default server *localhost:11300*. Queue 
@@ -93,27 +146,33 @@ you want to run it inside already run eventmachine, simply call
 `#start_listening` with the same parameters.
 
 Calls processing is thread safe because of eventmachine concept 
-similar to fibers. Default number at one time processed jobs is 20,
-but it can be changed by setting `:max_jobs => <number>` to `#listen!` 
-or `#start_listening`.
+similar to fibers.  Reponse will be put to the same queue server, 
+to queue named `qrpc-<client identifier>-output`, with structure 
+described above. 
 
-The `#precall` and `#postcall` handlers are supported on the API class
-called if API responds to them. Currently, they don't receive any
-parameters and are intended for simple call initialization only.
+Server can work by synchronous or asynchronous way. In the first case, 
+sends the returned values as result, in the other case yielded.
 
-Reponse will be put to the same queue server, to queue named 
-`qrpc-<client identifier>-output`, with structure described above. 
 
 ### Client Usage
 
-Client usage is simple too. Look example:
+Client usage is simple too. Look example complement the example above:
 
     require "eventmachine"
-    require "qrpc/client"
-    require "qrpc/locator"
+    
+    require "qrpc/client"                           # client 
+    require "qrpc/generator/uuid"                   # ID generator
+    require "qrpc/locator/em-jack"                  # queue transfer agent
+    require "qrpc/protocol/json-rpc"                # RPC protocol
+    require "json-rpc-objects/serializer/msgpack"   # serializer
     
     EM::run do
-        client = QRPC::Client::new QRPC::Locator::new "test"
+        generator = QRPC::Generator::ObjectID::new
+        locator = QRPC::Locator::EMJackLocator::new(:test)
+        serializer = JsonRpcObjects::Serializer::JSON::new
+        protocol = QRPC::Protocol::JsonRpc::new(:serializer => serializer)
+        
+        client = QRPC::Client::new(locator, generator, protocol) 
         client.subtract(2, 3) { |result| puts result }  # prints out -1
     end
     
@@ -126,7 +185,7 @@ its principle.
     
 Client is implemented as [evented][1] too, but in case of need you can 
 implement another one with non-evented interface using whatever 
-[beanstalkd][2] client you like of sure.
+[beanstalkd][2] client you like of sure or any other transfer agent.
 
 Contributing
 ------------
@@ -142,7 +201,7 @@ Contributing
 Copyright
 ---------
 
-Copyright &copy; 2011 [Martin Kozák][7]. See `LICENSE.txt` for
+Copyright &copy; 2011-2012 [Martin Kozák][7]. See `LICENSE.txt` for
 further details.
 
 [1]: http://rubyeventmachine.com/
@@ -152,3 +211,11 @@ further details.
 [5]: http://groups.google.com/group/json-rpc/web/json-rpc-2-0
 [6]: http://github.com/martinkozak/qrpc/issues
 [7]: http://www.martinkozak.net/
+[10]: http://github.com/martinkozak/unified-queues
+[11]: http://en.wikipedia.org/wiki/Universally_unique_identifier
+[12]: http://ruby-doc.org/core-1.9.3/Object.html#method-i-object_id
+[13]: http://github.com/martinkozak/json-rpc-objects
+[14]: http://www.json.org/
+[15]: http://bsonspec.org/
+[16]: http://yaml.org/
+[17]: http://msgpack.org/

data/Rakefile

@@ -18,7 +18,8 @@ Jeweler::Tasks.new do |gem|
   gem.name = "qrpc"
   gem.homepage = "http://github.com/martinkozak/qrpc"
   gem.license = "MIT"
-  gem.summary = 'Queued JSON-RPC client and server. Works as normal RPC server, but through queue interface, so allows highly scalable, distributed and asynchronous remote API implementation and fast data processing. It\'s based on eventmachine and beanstalkd, so it\'s fast and thread safe.'
+  gem.summary = 'Queued RPC client and server. Works as normal RPC server, but through queue interface, so allows highly scalable, distributed and asynchronous remote API implementation and fast data processing. It\'s based on EventMachine and typically on Beanstalk, so it\'s fast and thread safe.'
+  gem.post_install_message = "\nQRPC: API of the 0.9.x version (and 1.0 in future) is partialy incompatible with the older versions. Modifications of your current applications may be necessary for upgrading them to the latest version. \n\n"
   gem.email = "martinkozak@martinkozak.net"
   gem.authors = ["Martin Kozák"]
   # Include your dependencies below. Runtime dependencies are required when using your gem,

data/TODO.md

@@ -1,3 +1 @@
-* general queue interface for ability to use more queue servers (starling, sparrow, kestrel, stomp, AMQP...)
-* copyright headers to all files
-* think about Protocol Buffers or an binary format (BSON!) in general
+* plugins for more queue servers (starling, sparrow, kestrel, stomp, AMQP...)

data/VERSION

@@ -1 +1 @@
-0.4.0
+0.9.0

data/lib/qrpc/client.rb

@@ -1,5 +1,9 @@
 # encoding: utf-8
+# (c) 2011 Martin Kozák (martinkozak@martinkozak.net)
+
 require "qrpc/client/dispatcher"
+require "qrpc/generator/uuid"
+require "qrpc/general"
 
 ##
 # General QRPC module.
@@ -30,22 +34,26 @@ module QRPC
         
         ##
         # Constructor.
-        # @param [QRPC::Locator] locator of the output queue
+        #
+        # @param [QRPC::Locator] locator of the queues
+        # @param [QRPC::Generator] ID generator
+        # @param [QRPC::Protocol::Abstract] protocol protocol of the session
         #
         
-        def initialize(locator)
-            @dispatcher = QRPC::Client::Dispatcher::new(locator)
+        def initialize(locator, generator = QRPC::default_generator, protocol = QRPC::default_protocol)
+            @dispatcher = QRPC::Client::Dispatcher::new(locator, generator, protocol)
         end
                 
         ##
         # Handles call to RPC. (*********)
         #
         # Be warn, arguments will be serialized to JSON, so they should
-        # be serializable nativelly or implement +#to_json+ method.
+        # be serializable nativelly or implement +#to_s+ or +#to_json+ 
+        # method.
         #
         # @param [Symbol] name name of the called methods
         # @param [Array] args arguments of the called methods
-        # @param [Proc] block callback for returning result
+        # @param [Proc] block callback for returning 
         #
         
         def method_missing(name, *args, &block)

data/lib/qrpc/client/dispatcher.rb

@@ -1,8 +1,11 @@
 # encoding: utf-8
+# (c) 2011 Martin Kozák (martinkozak@martinkozak.net)
+
 require "em-jack"
-require "uuid"
+require "em-batch"
 require "qrpc/general"
 require "qrpc/client/job"
+require "hash-utils/object"   # >= 1.1.0
 require "json-rpc-objects/response"
 
 ##
@@ -23,18 +26,30 @@ module QRPC
     class Client
     
         ##
-        # Queue RPC client dispatcher (worker).
+        # Queue RPC client dispaxtcher (worker).
         # @since 0.3.0
         #
         
         class Dispatcher
+          
+            ##
+            # Holds current protocol object.
+            #
+            
+            @protocol
 
             ##
             # Holds locator of the target queue.
             #
             
             @locator
+
+            ##
+            # Holds generator of IDs.
+            #
             
+            @generator
+                        
             ##
             # Holds client session ID.
             #
@@ -77,17 +92,21 @@ module QRPC
             
             @@clients = { }
             
-            
             ##
             # Constructor.
+            #
             # @param [QRPC::Locator] locator of the output queue
+            # @param [QRPC::Generator] ID generator
+            # @param [QRPC::Protocol::Abstract] protocol protocol of the session
             #
             
-            def initialize(locator)
+            def initialize(locator, generator = QRPC::default_generator, protocol = QRPC::default_protocol)
+                @protocol = protocol
                 @locator = locator
+                @generator = generator
                 @pooling = false
                 @jobs = { }
-            
+
                 # Destructor
                 ObjectSpace.define_finalizer(self, self.class.method(:finalize).to_proc)
                 @@clients[self.object_id] = self
@@ -110,16 +129,16 @@ module QRPC
             
             def finalize!
                 if not @input_queue.nil?
-                    @input_queue.watch("default") do
-                        @input_queue.ignore(@input_name.to_s) do
-                            @input_queue.close
+                    @input_queue.subscribe("default") do
+                        @input_queue.unsubscribe(@input_name.to_s) do
+                            @input_queue.close!
                         end
                     end
                 end
                 
                 if not @output_queue.nil?
                     @output_queue.use("default") do
-                        @output_queue.close
+                        @output_queue.close!
                     end
                 end
             end
@@ -135,7 +154,7 @@ module QRPC
             #
             
             def create_job(name, args, priority = QRPC::DEFAULT_PRIORITY, &block)
-                Client::Job::new(self.id, name, args, priority, &block)
+                Client::Job::new(self.id, name, args, priority, @generator, @protocol, &block)
             end
             
             ##
@@ -144,16 +163,19 @@ module QRPC
             
             def put(job)
                 if not job.notification?
-                    @jobs[job.id] = job
-                end
-                
-                self.output_queue do |queue|
-                    queue.put(job.to_json)
+                    id = job.id
+                    id = id.to_sym if not id.kind_of? Integer
+                    
+                    @jobs[id] = job
                 end
                 
                 if (not @pooling) and (@jobs.length > 0)
                     self.pool!
                 end
+                
+                self.output_queue do |queue|
+                    queue.push(job.serialize)
+                end
             end
          
             ##
@@ -164,28 +186,26 @@ module QRPC
                 
                 # Results processing logic
                 processor = Proc::new do |job|
-                    response = JsonRpcObjects::Response::parse(job.body)
-                    job.delete()
+                    response = @protocol.response::parse(job)
                     
                     if not response.id.nil?
-                        id = response.id.to_sym
+                        id = response.id
+                        id = id.to_sym if not id.kind_of? Integer
+                        
                         if @jobs.include? id
                             @jobs[id].assign_result(response)
+                            @jobs.delete(id)
                         end
                     end
-                    
-                    @jobs.delete(id)
                 end
                 
-                # Runs processor for each job            
-                parent = self
-                worker = EM.spawn do
-                    parent.input_queue { |q| q.each_job(&processor) }
+                # Runs processor for each job (expects recurring #pop)   
+                self.input_queue do |queue|
+                    queue.pop(true, &processor)
                 end
                 
                 ##
                 
-                worker.run
                 @pooling = true
                 
             end
@@ -197,10 +217,10 @@ module QRPC
             
             def input_name
                 if @input_name.nil?
-                    @input_name = (QRPC::QUEUE_PREFIX.dup << "-" << self.id.to_s << "-" << QRPC::QUEUE_POSTFIX_OUTPUT).to_sym
+                    @input_name = (QRPC::QUEUE_PREFIX + "-" + self.id.to_s + "-" + QRPC::QUEUE_POSTFIX_OUTPUT).to_sym
+                else
+                    @input_name
                 end
-                
-                return @input_name
             end
             
             ##
@@ -210,22 +230,16 @@ module QRPC
             
             def input_queue(&block)
                 if @input_queue.nil?
-                    @input_queue = EMJack::Connection::new(:host => @locator.host, :port => @locator.port)
-                    @input_queue.watch(self.input_name.to_s) do
-                        @input_queue.ignore("default") do
-
-                            # Results pooler error handler
-                            @input_queue.on_error do |error|
-                                raise ::Exception::new("Beanstalk error: " << error.to_s)
-                            end
-                            
-                            # Returns
-                            block.call(@input_queue)
-                            
+                    @input_queue = @locator.input_queue
+                    @input_queue.subscribe(self.input_name.to_s) do
+                        @input_queue.unsubscribe("default") do
+                            yield @input_queue 
                         end
                     end
                 else
-                    block.call(@input_queue)
+                    @input_queue.subscribe(self.input_name.to_s) do
+                        yield @input_queue
+                    end
                 end
             end
             
@@ -236,10 +250,10 @@ module QRPC
             
             def output_name
                 if @output_name.nil?
-                    @output_name = (QRPC::QUEUE_PREFIX.dup << "-" << @locator.queue << "-" << QRPC::QUEUE_POSTFIX_INPUT).to_sym
+                    @output_name = (QRPC::QUEUE_PREFIX + "-" + @locator.queue_name + "-" + QRPC::QUEUE_POSTFIX_INPUT).to_sym
+                else
+                    @output_name
                 end
-                
-                return @output_name
             end
             
             ##
@@ -249,12 +263,14 @@ module QRPC
             
             def output_queue(&block)
                 if @output_queue.nil?
-                    @output_queue = EMJack::Connection::new(:host => @locator.host, :port => @locator.port)
+                    @output_queue = @locator.output_queue
                     @output_queue.use(self.output_name.to_s) do
-                        block.call(@output_queue)
+                        yield @output_queue
                     end
                 else
-                    block.call(@output_queue)
+                    @output_queue.use(self.output_name.to_s) do
+                        yield @output_queue
+                    end
                 end
             end
             
@@ -265,10 +281,10 @@ module QRPC
             
             def id
                 if @id.nil?
-                    @id = UUID.generate(:compact).to_sym
+                    @id = @generator.generate(self)
+                else
+                    @id
                 end
-                
-                return @id
             end
                
         end