nebulous_stomp 2.0.2 → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a4ac7c514b6d925e6186924b2dd93a175de94e02
-  data.tar.gz: 8aca7aa180be4c84fe6ccbe89b172980cd2a56a1
+  metadata.gz: 2b884a207187065896c92f83149c9a9701fb4b32
+  data.tar.gz: 18ec13584623af59f629a7c1e4ebc2d904c59cf3
 SHA512:
-  metadata.gz: abb4de512e2aec3d8720505f9f06d7a00b2d5aa4caba2a1e9dab977f6c36cb6970f791289509cc6fbe96b945d6eeec84a65d7fe86b54d7402b97104dcd71c18d
-  data.tar.gz: 3e44c452fa7f94cd331cfc637b90b427c5171bd7fa9663b4ebe5118320e8d1ea8554628b57bcecfcc55cbc2f7f69aed9d13f49951c8e1ce82836db55cb7b26d5
+  metadata.gz: 25d7c1f7cda1c2a509c837b4de72a998fbca6354bf29f9ec1a9e4931df1a22458b19b9cf57a4bf7e5c61fbb1f3d22ceeb7ff66dc93f1327756c0a98e7293e93b
+  data.tar.gz: b9cfecbfdb21aaa42f36162b9baf57221a53706779721a584f5998a3a4f853fd0929789722b1a8ca896231b75123a43fdecfb76a051826f4fd7ff73cebe94f5b

data/.hgignore

@@ -17,3 +17,5 @@ syntax: glob
 mkmf.log
 .ruby-*
 .Project
+
+feature/connection.yaml

data/.hgtags

@@ -17,3 +17,4 @@ cff04defabb0895ff2f032087b97ea9616bee6a9 1.14
 6da65ce15fab9722deb05d8c0ed2d0c8e2cfe2ff 2.0.0
 9c4ec9af8f6b8a848082bf118be037696ff357c7 2.0.1
 63eb0028eeedf7cc3938fd8ecda1add9246fb905 2.02
+c4e3ca2874b81f2b8f7008322ff0e5a18f29c35e 3.0.0

data/README.md

@@ -1,38 +1,235 @@
-Nebulous
+Introduction
+============
+
+A little module that implements The Nebulous Protocol, a way of passing data over STOMP between
+different systems. Specifically, it allows you to send a message, a *Request* and receive another
+message in answer, a *Response*.  (Which is not something STOMP does, out of the box).
+
+This library covers two specific use cases (three if you are picky):
+
+1) Request-Response: a program that consumes incoming messages, works out what message to send in
+response, and sends it.
+
+2) Question-Answer: a program that sends a request and then waits for a response; the other end of
+the Request-Response use case. We support optional caching of responses in Redis, to speed things up
+if your program is likely to make the same request repeatedly within a short time.
+
+3) Since we are talking to Redis, we expose a basic, simple interface for you to talk to it
+yourself.
+
+
+Thanks
+======
+
+This code was developed, by me, during working hours at [James Hall & Co.
+Ltd](https://www.jameshall.co.uk/). I'm incredibly greatful that they have permitted me to
+open-source it.
+
+
+
+A Quick Example
+===============
+
+Before we get too bogged down, some code. 
+
+    require "nebulous_stomp"
+
+    NebulousStomp.init( my_init_hash )
+    NebulousStomp.add_target(:target1, my_target)
+
+    message  = NebulousStomp::Message.new(verb: "ping")
+    request  = NebulousStomp::Request.new(:target1, message)
+    response = request.send
+
+This example is for the question-answer use case. `response` will contain a NebulousStomp::Message
+-- unless the target fails to respond in time, in which case a NebulousStomp::MessageTimeout will
+be raised.
+
+
+The Protocol
+============
+
+I natter on about this in far too much detail elsewhere, but the highly condensed version is:
+
+* Every request always gets a response; if you don't get one, then something is wrong at the other
+  end.
+
+* Request-response programs consume all messages on the queue that they listen on. They place the
+  response on a *different* queue, the name of which is given by the request. 
+
+* Each message has a 'unique' ID; the response has the ID of the request it responds to.
+
+* Messages can "follow the protocol" by being in the form: verb, parameters, descripton. Requests
+  *must* be in this form; responses don't have to be.
+
+* The special verb "success" in a response means "I got the message, everything is fine, nothing to
+  report here".
+
+* The special verb "error" in a response means something went wrong. The description should say
+  what.
+
+
+Targets
+=======
+
+When you have a system running a request-response loop, then the simplest way to proceed is to
+assign it a pair of queues on your Stomp server: one for incoming requests, and one for it to post
+responses to. 
+
+We call such a system a Target. Any other, question-answer, system (which wants to throw Requests at
+that target and get a response) will need to know what those queues are; so we configure a list of
+targets at startup.
+
+Note that it is perfectly okay for a target to use more than one request queue (desirable, even,
+if some requests will take time to fulfil). But we don't directly support that in Nebulous: a
+Target is always one request queue, one response queue. In this case, the simplest way forward is
+to define two targets.
+
+
+Examples
 ========
 
-A little module that implements the Nebulous Protocol, a way of passing data
-over STOMP between different systems. We also support message cacheing via
-Redis.
+Request-Response
+----------------
+
+This revisits the example from the start, but with more detail. For completeness, we configure a
+Redis server for caching respsonses (which is optional) and show all the config hashes (which
+certainly want to come from a config file in practice).
+
+    require "nebulous_stomp"
+
+    host   = {login: "guest", passcode: "guest", host: "10.11.12.13", ssl: false}
+    stomp  = {hosts: [host], reliable: false}
+    redis  = {host: '127.0.0.1', port: 6379, db: 0}
+
+    config = { stompConnectHash: stomp, 
+               redisCOnnectHash: redis, 
+               messageTimeout:   5,
+               cacheTimeout:     30 }
+
+    target = { sendQueue:      "/queue/in", 
+               receiveQueue:   "/queue/out", 
+               messageTimeout: 7 }
+
+    NebulousStomp.init(config)
+    NebulousStomp.add_target(:target1, target)
+
+    message  = NebulousStomp::Message.new(verb: "ping")
+    request  = NebulousStomp::Request.new(:target1, message)
+
+    response1 = request.send
+    response2 = request.send
+
+`response1` will be filled from the target; `response2` will be filled from the Redis cache
+(provided that line gets called within 30 seconds of the previous line). (Obviously this is
+pointless and for example only.)
+
+The stomp hash is passed unchanged to the Stomp gem; the redis hash is passed unchanged to the
+Redis gem. See these gems for details about what they should contain.
+
+Message.new takes a single hash as an argument; it accepts a long list of possible keys, but mostly
+I imagine you will be using 'verb', 'params', and 'desc'. It's worth also noting 'replyTo', which
+sets the queue to reply to; if missing then Request sets it from the Target, of course.
+
+This rather specific example contains three seperate timeout values. The message timeout is the
+time we wait for a response before raising MessageTimeout. The value in the config hash is a
+default; in this example it is overidden on the target.  The cache timeout is, of course, the time
+that the response is kept on the cache. These values can be further overridden for specific
+messages.
+
+Often even with a cache set up, you don't want to use it (for requests that trigger database
+updates, for example); in which case the method to call is `send_no_cache`.
+
+Question-Answer
+---------------
+
+    require "nebulous_stomp"
+
+    NebulousStomp.init(config)
+    target = NebulousStomp.add_target(:target1, target)
+
+    listener = NebulousStomp::Listener.new(target)
+
+    listener.consume_messages do |msg|
+      begin
+
+        case msg.verb
+          when "ping" 
+            listener.reply *msg.respond_with_success 
+          when "time" 
+            listener.reply *msg.respond_with_protocol("timeresponse", Time.now)
+          else
+            listener.reply *msg.respond_with_error("Bad verb #{msg.verb}")
+        end
+
+      rescue
+        listener.reply *msg.respond_with_error($!)
+      end
+    end
+
+    loop { sleep 5 }
+
+This example implements a target that responds to two verbs.  In responce to "ping" it sends a
+success verb, indicating it got the message. In response to "time" it sends a "timeresponse" verb
+with the current time as a parameter. For any other verb on its receive queue, it responds with an
+error verb.
+
+`Listener.new` requires either a Target object or a queue name. This is different from Request,
+which can take a target name. You can always retreive a Target object yourself, though, like this:
+
+    target = NebulousParam.get_target(targetname)
+
+If you want to respond with a message that does not follow the verb-parameter-description part of
+the protocol, then you can pass an arbitrary message body to `msg.respond()`.
+
+Note the error handling. This is especially important because the body that you pass to the
+`consume_messages` method is actually being run in a thread, by the Stomp gem; by default all
+errors will be swallowed silently. As you can see, `message.respond_with_error` can take an
+exception as a parameter.
+
+Note also, for the same reason, that your program must hold the main thread open while
+`consume_messages` is running; if the main thread ends, the program stops.
+
+Redis
+-----
+
+    require "nebulous_stomp/redis_helper"
+
+    # ...parameters get set here...
+
+    redis = NebulousStomp::RedisHelper.new
+
+    redis.set(:thing, "thingy")
+    redes.set(:gone_in_30_seconds, "thingy", 30)
+
+    value = redis.get(:thing)
+
+    redis.del(:thing)
 
-There are two use cases:
+Obviously this is not so much an example as it is some random calls to RedisHelper. But hopefully
+it is fairly self-explanatory.
 
-First, sending a request for information and waiting for a response, which
-might come from a cache of previous responses, if you allow it. To do
-this you should create a Nebulous::NebRequest, which will return a
-Nebulous::Message.
 
-Second, the other end of the deal: hanging around waiting for requests and
-sending responses. To do this, you need to use the Nebulous::StompHandler
-class, which will again furnish Nebulous::Meessage objects, and allow you to
-create them.
+A list of classes
+=================
 
-Some configuratuion is required: see Nebulous.init, Nebulous.add_target &
-Nebulous.add_logger.
+To help you drill down to the API documentation.  These are the externally-facing classes:
 
-Since you are setting the Redis connection details as part of initialisation,
-you can also use it to connect to Redis, if you want. See
-Nebulous::RedisHandler.
+* Listener      -- implements the request-response use case
+* Message       -- a Nebulous message
+* NebulousStomp -- main class
+* RedisHelper   -- implements the Redis use case 
+* Request       -- implements the Request-Response use case; a wrapper for Message
+* Target        -- represents a single Target
+ 
+These classes are used internally:
 
-a complete list of classes & modules:
+* Param            -- helper class to store and return configuration
+* RedisHandler     -- internal class to wrap the Redis gem
+* RedisHandlerNull -- a "mock" version of  RedisHandler for use in testing
+* StompHandler     -- internal class to wrap the Stomp gem
+* StompHandlerNull -- a "mock" version of StompHandler for use in testing
 
-* Nebulous
-* Nebulous::Param
-* Nebulous::NebRequest
-* Nebulous::NebRequestNull
-* Nebulous::Message
-* Nebulous::StompHandler
-* Nebulous::StompHandlerNull
-* Nebulous::RedisHandler
-* Nebulous::RedisHandlerNull
+You might find the null classes useful in your own tests; both Listener and Request allow the
+injection of mock handler objects. You must require them seperately, though.
 

data/feature/connection_example.yaml

@@ -0,0 +1,24 @@
+---
+
+:init:
+  :stompConnectHash:
+    hosts:
+      - login:    guest
+        passcode: guest
+        host:     '10.11.12.13'
+        port:     61613
+        ssl:      false
+    reliable: false
+
+  :redisConnectHash:
+    host: '127.0.0.1'
+    port: 6379
+    db  : 0   
+
+  :messageTimeout: 2
+  :cacheTimeout  : 10
+
+:target:
+  :sendQueue:    '/queue/featuretestsend'
+  :receiveQueue: '/queue/featuretestreceive'
+

data/feature/feature_test_spec.rb

@@ -0,0 +1,247 @@
+require 'nebulous_stomp'
+require 'nebulous_stomp/redis_helper'
+require 'nebulous_stomp/redis_handler'
+
+require_relative 'gimme'
+
+
+##
+# These are the feature tests for Nebulous. They are not run when you type `rspec`; that only gets
+# you the unit tests. You have to name this directory to run it: `rspec feature`.
+#
+# These tests require an actual, working STOMP server and an actual, working Redis server (and ones
+# which you don't mind sending test messages to, at that). You should configure connection to this
+# in features/connection.yaml; an example file is provided, features/connection_example.yaml.
+#
+describe 'stomp use cases:' do
+
+  def init_nebulous(configfile)
+    config = YAML.load(File.open configfile)
+    NebulousStomp.init config[:init]
+    NebulousStomp.add_target("featuretest", config[:target] )
+  end
+
+  def new_request(verb)
+    message = NebulousStomp::Message.new(verb: verb)
+    NebulousStomp::Request.new("featuretest", message)
+  end
+
+  before(:all) do
+    init_nebulous 'feature/connection.yaml'
+    Thread.new{ Gimme.new("feature/connection.yaml").run; sleep 15 }
+  end
+
+  let(:hash) do
+    { "verb"        => "foo", 
+      "parameters"  => "bar", 
+      "description" => "baz" }
+
+  end
+
+  let(:redis) { NebulousStomp::RedisHelper.new }
+
+  # Plain redis access for debugging
+  def redis_backdoor(cmd, arg)
+    hash = NebulousStomp::Param.get :redisConnectHash
+    handler = NebulousStomp::RedisHandler.new hash
+    handler.connect unless handler.connected?
+    handler.send(cmd.to_sym, arg.to_s)
+  end
+
+  # Plain stomp access for debugging
+  def stomp_backdoor(cmd, queue, msg=nil)
+    hash = NebulousStomp::Param.get :stompConnectHash
+    handler = NebulousStomp::StompHandler.new hash
+
+    case cmd
+      when :send 
+        handler.send_message(queue, msg)
+        return true
+      when :listen
+        messages = []
+        handler.listen_with_timeout(queue, 1) {|msg| messages << msg; false } rescue nil
+        return messages, handler.connected?
+    end
+  end
+
+
+  ##
+  # tests for the request-response use case - a server that consumes messages and responds with
+  # other messages.
+  #
+  # Note that it's the Gimme class, in the thread above, that is actually doing the responding; we
+  # just send a message to it and check the response.
+  #
+  describe "request-response:" do
+
+    it "can respond to a message with a success verb" do
+      response = new_request("gimmesuccess").send_no_cache
+
+      expect( response      ).to be_kind_of(NebulousStomp::Message)
+      expect( response.verb ).to eq "success"
+    end
+
+    it "can respond to a message with an error verb" do
+      response = new_request("gimmeerror").send_no_cache
+
+      expect( response      ).to be_kind_of(NebulousStomp::Message)
+      expect( response.verb ).to eq "error"
+    end
+
+    it "can respond to a message with a specific Protocol message" do
+      response = new_request("gimmeprotocol").send_no_cache
+
+      expect( response      ).to be_kind_of(NebulousStomp::Message)
+      expect( response.verb ).to eq hash["verb"]
+    end
+
+    it "can respond to a message with a non-Protocol message" do
+      message  = NebulousStomp::Message.new(verb: 'gimmemessage', contentType: 'text')
+      response = NebulousStomp::Request.new("featuretest", message).send_no_cache
+
+      expect( response      ).to be_kind_of(NebulousStomp::Message)
+      expect( response.verb ).to be_nil
+      expect( response.body ).to eq "weird message body"
+    end
+
+    it "can send a message >32k" do
+      message  = NebulousStomp::Message.new( verb: 'gimmebigmessage', 
+                                             params: "32", 
+                                             contentType: 'text' )
+
+      response = NebulousStomp::Request.new("featuretest", message).send_no_cache
+
+      expect( response           ).to be_kind_of(NebulousStomp::Message)
+      expect( response.body.size ).to be > (1024 * 32)
+      expect( response.body[0..2] ).to eq "foo"
+      expect( response.body[-3..-1] ).to eq "bar"
+    end
+
+    it "can send a message >128k" do
+      message  = NebulousStomp::Message.new( verb: 'gimmebigmessage', 
+                                             params: "128", 
+                                             contentType: 'text' )
+
+      response = NebulousStomp::Request.new("featuretest", message).send_no_cache
+
+      expect( response           ).to be_kind_of(NebulousStomp::Message)
+      expect( response.body.size ).to be > (1024 * 128)
+      expect( response.body[0..2] ).to eq "foo"
+      expect( response.body[-3..-1] ).to eq "bar"
+    end
+
+    it "can send a message >512k" do
+      message  = NebulousStomp::Message.new( verb: 'gimmebigmessage', 
+                                             params: "512", 
+                                             contentType: 'text' )
+
+      response = NebulousStomp::Request.new("featuretest", message).send_no_cache
+
+      expect( response           ).to be_kind_of(NebulousStomp::Message)
+      expect( response.body.size ).to be > (1024 * 512)
+      expect( response.body[0..2] ).to eq "foo"
+      expect( response.body[-3..-1] ).to eq "bar"
+    end
+
+  end
+  ##
+
+
+  ##
+  # Tests for the question-and-answer use case -- a process that sends a request to a
+  # request-response server and waits for an answering response
+  #
+  describe "question-and-answer:" do
+
+
+    it "can send a JSON message and get a JSON response" do
+      message  = NebulousStomp::Message.new(verb: 'gimmeprotocol', contentType: 'application/json')
+      response = NebulousStomp::Request.new("featuretest", message).send_no_cache
+
+      expect( response.content_type ).to eq 'application/json'
+      expect( response.body ).to eq hash
+      expect( response.stomp_body ).to eq hash.to_json
+    end
+
+    it "can send a text message and get a text response" do
+      message  = NebulousStomp::Message.new(verb: 'gimmeprotocol', contentType: 'application/text')
+      response = NebulousStomp::Request.new("featuretest", message).send_no_cache
+
+      expect( response.content_type ).to eq 'application/text'
+      expect( response.body ).to eq hash
+
+      hash.each do |k,v|
+        expect( response.stomp_body ).to match(/#{k}: *#{v}/)
+      end
+    end
+
+    it "can cache a response in Redis" do
+      message = NebulousStomp::Message.new(verb: 'gimmeprotocol', contentType: 'application/text')
+      request = NebulousStomp::Request.new("featuretest", message)
+      signature = {verb:"gimmeprotocol"}.to_json
+
+      redis_backdoor(:del, signature)
+      request.send
+      expect( redis_backdoor(:get, signature) ).not_to be_nil
+    end
+
+    it "will receive its response without disturbing any others" do
+      msg1   = NebulousStomp::Message.new(verb: 'backdoor',      contentType: 'application/text')
+      msg2   = NebulousStomp::Message.new(verb: 'gimmeprotocol', contentType: 'application/text')
+      target = NebulousStomp.get_target(:featuretest)
+
+      # Place msg1 on the queue directly
+      stomp_backdoor(:send, target.send_queue, msg1)
+
+      # Send Msg2 to the target, so that Gimme puts the response on the queue and then we read it
+      response2 = NebulousStomp::Request.new(target, msg2).send_no_cache
+
+      # Now read the messages left on the queue
+      leftovers, connected = stomp_backdoor(:listen, target.send_queue)
+
+      expect( response2      ).to be_kind_of NebulousStomp::Message
+      expect( response2.verb ).to eq "foo"
+      expect( leftovers.map(&:verb) ).to include("backdoor")
+      expect( leftovers.map(&:verb) ).not_to include("foo")
+      expect( connected ).to be_truthy
+    end
+
+  end
+  ##
+
+
+  ##
+  # Tests for the Redis use case -- user wants to access Redis so we grant them access through our
+  # connection to it.
+  #
+  describe "redis:" do
+
+    it "can set a value in the store" do
+      redis.del(:foo) rescue nil
+      redis.set(:foo, "bar")
+      expect( redis.get(:foo) ).to eq "bar"
+    end
+
+    it "can set a value in the store with a timeout" do
+      redis.set(:foo, "bar", 1)
+      expect( redis.get :foo ).to eq "bar"
+      sleep 2
+      expect( redis.get :foo ).to be_nil
+    end
+
+    it "can get a value from the store" do
+      redis.set(:foo, bar: "baz")
+      expect( redis.get(:foo) ).to eq( {bar: "baz"} )
+    end
+
+    it "can remove a value from the store" do
+      redis.set(:foo, "bar")
+      redis.del(:foo)
+      expect( redis.get :foo ).to be_nil
+    end
+
+  end
+  ##
+
+end
+

data/feature/gimme.rb

@@ -0,0 +1,91 @@
+$: << "./lib" if __FILE__ == $0
+
+require 'nebulous_stomp'
+require 'yaml'
+require "pry"
+
+
+##
+# A little request-reponse server for the feature test
+#
+class Gimme
+
+  def initialize(configfile)
+    @config   = load_config configfile
+    @target   = init_nebulous
+    #@listener = NebulousStomp::Listener.new(@target)
+    @listener = NebulousStomp::Listener.new("/queue/featuretestreceive")
+  end
+
+  def run
+    @listener.consume_messages{|msg| reply msg}
+  end
+
+  def quit
+    @listener.quit
+  end
+
+  private
+
+  def load_config(file)
+    YAML.load(File.open file)
+  end
+
+  def init_nebulous
+    NebulousStomp.init @config[:init]
+    NebulousStomp.add_target("featuretest", @config[:target] )
+  end
+
+  def reply(msg)
+    queue, message = 
+      case msg.verb
+        when "gimmesuccess" 
+          msg.respond_with_success
+
+        when "gimmeerror" 
+          msg.respond_with_error("the error you wanted")
+
+        when "gimmeprotocol" 
+          msg.respond_with_protocol("foo", "bar", "baz")
+
+        when "gimmemessage" 
+          msg.respond("weird message body")
+
+        when "gimmebigmessage"
+          body = big_body msg.params
+          msg.respond body
+
+        else fail "unknown verb #{msg.verb} in Gimme"
+      end
+
+    @listener.reply(queue, message)
+
+  rescue
+    puts "ERROR: #{$!}" 
+    $!.backtrace.each{|e| puts e }
+  end
+
+  def big_body(params)
+    kb = params.to_f; fail "not a size" if kb == 0
+
+    body = "foo"
+    body << "Q" * (1024 * kb)
+    body << "bar"
+
+    body
+  end
+
+end
+
+
+if __FILE__ == $0
+
+  begin
+    g = Gimme.new('./feature/connection.yaml')
+    g.run
+    loop { sleep 5 }
+  ensure
+    g.quit
+  end
+
+end