thrift_amqp_transport 0.1.0

data/History.txt

@@ -0,0 +1,7 @@
+=== 0.1.1 / ???
+
+* FIXED: Race condition on server/client creation
+
+=== 0.1.0 / 2009-08-27
+
+* One-way communication with a thrift service, using a header exchange.

data/Manifest.txt

@@ -0,0 +1,14 @@
+History.txt
+Manifest.txt
+README.txt
+Rakefile
+lib/thrift_amqp/server_transport.rb
+lib/thrift_amqp/transport.rb
+lib/thrift_amqp_transport.rb
+spec/integration/oneway_spec.rb
+spec/integration/test.thrift
+spec/spec.opts
+spec/spec_helper.rb
+spec/unit/server_transport_spec.rb
+spec/unit/transport_spec.rb
+thrift_amqp_transport.gemspec

data/README.txt

@@ -0,0 +1,93 @@
+= Thrift AMQP transport
+
+== DESCRIPTION
+
+Transports thrift messages over a the advanced message queue protocol. (AMQP)
+Because of the unconnected broadcasting nature of the message queue, this 
+transport supports only one-way communication. 
+
+The usage scenario is that you would use this to broadcast information about 
+services (1 producer, n consumers) and then create point to point connections 
+from client to service using normal (TCP) thrift. You gain the advantage of
+using only one interface definition language (IDL). 
+
+== SYNOPSIS
+
+Using the following interface definition: 
+
+  service AwesomeService {
+    // a little hommage 
+    oneway void battleCry(1:string battlecry);
+  }
+
+On the server (consumer of messages): 
+
+  class MyAwesomeHandler
+    def battleCry(message)
+      puts messages
+    end
+  end
+  
+  handler = MyAwesomeHandler.new()
+  processor = AwesomeService::Processor.new(handler)
+  
+  # Connecting to the exchange called 'battle_cry'. This must match the
+  # clients setting.    
+  server_transport = Thrift::AMQP::ServerTransport.new('battle_cry')
+  
+  server = Thrift::SimpleServer.new(processor, transport)
+  
+  server.serve    # never returns
+
+On the client: 
+
+  transport = Thrift::AMQP::Transport.connect('battle_cry')
+  protocol = Thrift::BinaryProtocol.new(transport)
+  client = AwesomeService::Client.new(protocol)
+  
+  client.battleCry('chunky bacon!')   # prints 'chunky bacon!' on the server
+
+== SPECIFICATION
+
+To run the specs, just type
+
+  rake specs
+  
+Integration tests need a AMQP-Server to run on localhost. This has so far only
+been tested with RabbitMQ. (http://www.rabbitmq.com/)
+
+  rake thrift
+  rake integration
+  
+== REQUIREMENTS
+
+rabbitmq: local server for integration tests
+thrift: full installation needed if you want to compile service descriptions
+bunny
+
+
+== LICENSE
+
+(The MIT License)
+
+Copyright (c) 2009 Kaspar Schiess
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+

data/Rakefile

@@ -0,0 +1,36 @@
+require 'hoe'
+ 
+namespace :hoe do # Too much out of the box... 
+  Hoe.spec 'thrift_amqp_transport' do 
+    developer('Kaspar Schiess', 'eule@space.ch')
+  end
+end
+
+desc "Runs all unit examples."
+Spec::Rake::SpecTask.new('spec:unit') do |t|
+  t.spec_files = FileList['spec/unit/*_spec.rb']
+  t.spec_opts = ['--options', 'spec/spec.opts']
+end
+
+desc "Runs all integration examples."
+Spec::Rake::SpecTask.new('spec:integration') do |t|
+  t.spec_files = FileList['spec/integration/*_spec.rb']
+  t.spec_opts = ['--options', 'spec/spec.opts']
+end
+
+desc "Runs all examples."
+Spec::Rake::SpecTask.new('spec:all') do |t|
+  t.spec_files = FileList['spec/**/*_spec.rb']
+  t.spec_opts = ['--options', 'spec/spec.opts']
+end
+
+
+compile_integration = file 'spec/integration/gen-rb/test.rb' => %w(spec/integration/test.thrift) do
+  rm_rf 'spec/integration/gen-rb' rescue nil
+  sh %Q{thrift --gen rb -o spec/integration spec/integration/test.thrift }
+end
+
+desc "Compiles thrift IDL definitions into Ruby code."
+task :thrift => compile_integration
+
+task :default => 'spec:all'

data/lib/thrift_amqp/server_transport.rb

@@ -0,0 +1,54 @@
+
+
+# Server transport for using oneway thrift over an AMQP-queue. 
+#
+# Example: 
+#
+#   class MyAwesomeHandler
+#     def battleCry(message)
+#       puts messages
+#     end
+#   end
+# 
+#   handler = MyAwesomeHandler.new()
+#   processor = AwesomeService::Processor.new(handler)
+# 
+#   # Connecting to the exchange called 'battle_cry'. This must match the
+#   # clients setting.    
+#   server_transport = Thrift::AMQP::ServerTransport.new('battle_cry')
+# 
+#   server = Thrift::SimpleServer.new(processor, transport)
+# 
+#   server.serve    # never returns
+#
+class Thrift::AMQP::ServerTransport < Thrift::BaseServerTransport
+  # Initializes a connection to the AMQP queue. If you provide a headers 
+  # argument, only messages that match ALL headers will be accepted. 
+  #
+  # Example: 
+  #
+  #   Thrift::AMQP::ServerTransport.new('queue', :version => 1)
+  #   # Will only match messages that have 'version' == '1'
+  # 
+  def initialize(exchange_name, headers={})
+    @exchange_name = exchange_name
+    @header_filter = transform_hash_to_string(headers)
+  end
+  
+  # Part of the server 
+  def listen
+    @transport = Thrift::AMQP::Transport.connect(@exchange_name)
+  end
+
+  def accept
+    @transport
+  end
+private
+  def transform_hash_to_string(hash)
+    hash.inject({}) do |new_hash, (k,v)|
+      new_hash[k.to_s] = v.to_s
+
+      new_hash
+    end
+  end
+end

data/lib/thrift_amqp/transport.rb

@@ -0,0 +1,91 @@
+
+# Transport of thrift messages (oneway) using an AMQP backend. 
+#
+# Example: 
+#
+#   transport = Thrift::AMQP::Transport.connect('battle_cry')
+#   protocol = Thrift::BinaryProtocol.new(transport)
+#   client = AwesomeService::Client.new(protocol)
+#   
+#   client.battleCry('chunky bacon!')   # prints 'chunky bacon!' on the server
+#
+class Thrift::AMQP::Transport < Thrift::BaseTransport
+  POLL_SLEEP = 0.01
+  
+  # Connects the transport to the queue. Since client and server use much of
+  # the same method to to this (and since queues/exchanges must be declared
+  # with the same parameters), this is code shared between the server and 
+  # the client.
+  # 
+  # Parameters: 
+  # * +exchange_name+ - The name of the exchange to connect to. 
+  # * +headers+ - The headers to look for (receive) or the headers to send. 
+  #   Client and server should provide the same arguments. 
+  #
+  def self.connect(exchange_name, headers={})
+    connection = Bunny.new
+    connection.start
+    
+    exchange = begin
+      connection.exchange(exchange_name, 
+        :type => :fanout)
+    rescue Bunny::ProtocolError
+      raise "Could not create exchange #{@exchange_name}, maybe it exists (with different params)?"
+    end
+    
+    # Make sure the queue exists. The server doesn't use this, but no harm 
+    # in creating this anyway. 
+    queue = connection.queue(exchange_name, 
+      :auto_delete => true)
+    queue.bind(exchange)
+    
+    instance = new(connection, exchange, queue)
+    instance
+  end
+  
+  # Constructs a transport based on an existing connection and a message
+  # exchange (of the headers type). 
+  #
+  # It might be more simple to use the Transport.connect method.
+  #
+  def initialize(connection, exchange, queue)
+    @connection = connection
+    @exchange   = exchange
+    @queue      = queue
+    
+    @buffered_message = ''
+    @write_buffer = ''
+  end
+  
+  def read(sz)
+    # loop and pop until we have something to show 
+    loop do
+      if buffered_message?
+        return buffered_message.slice!(0,sz) || ''
+      end
+      
+      self.buffered_message = @queue.pop
+      sleep POLL_SLEEP unless buffered_message?
+    end
+  end
+  
+  def write(buffer)
+    write_buffer << buffer
+  end
+  def flush
+    @exchange.publish(write_buffer)
+    self.write_buffer = ''
+  end
+  
+private
+  attr_accessor :buffered_message
+  attr_accessor :write_buffer
+  
+  def buffered_message?
+    not (
+      buffered_message == :queue_empty ||
+      buffered_message.nil? ||
+      buffered_message.empty?
+    )
+  end
+end

data/lib/thrift_amqp_transport.rb

@@ -0,0 +1,12 @@
+
+require 'thrift'    # for messaging
+require 'bunny'     # for AMQP interaction
+
+module Thrift
+  module AMQP
+    VERSION = '0.1.0'
+  end
+end
+
+require 'thrift_amqp/server_transport'
+require 'thrift_amqp/transport'

data/spec/integration/oneway_spec.rb

@@ -0,0 +1,116 @@
+require File.dirname(__FILE__) + '/../spec_helper'
+
+$:.unshift File.dirname(__FILE__) + "/gen-rb"
+begin
+  require 'test'
+rescue LoadError
+  puts "No test interface found. Maybe you should run 'rake thrift' first?"
+end
+
+describe "AMQP Transport Integration (oneway)" do
+  EXCHANGE_NAME = 'integration_spec_oneway'
+  
+  class SpecHandler
+    attr_reader :messages
+    def initialize
+      @messages = []
+    end
+    def sendMessage(message)
+      @messages << message
+    end
+  end
+  
+  class SpecTestServer 
+    attr_reader :handler
+    def initialize()
+      @handler = SpecHandler.new()
+      @processor = Test::Processor.new(handler)
+      @server_transport = Thrift::AMQP::ServerTransport.new(EXCHANGE_NAME)
+    end
+    
+    # Spins the server and makes it read the next +message_count+ messages.
+    #
+    def spin_server(message_count)
+      @server_transport.listen
+      client = @server_transport.accept
+      prot = Thrift::BinaryProtocol.new(client)
+      
+      messages_left = message_count
+      while messages_left > 0
+        @processor.process(prot, prot)
+        messages_left -= 1
+      end
+    ensure
+      client.close
+    end
+    
+    def close
+      @server_transport.close
+    end
+  end
+  
+  context 'with a server in the background' do
+    attr_reader :server, :client
+    before(:each) do
+      # Server setup
+      @server = SpecTestServer.new
+      
+      # Client setup
+      begin
+        @transport = Thrift::AMQP::Transport.connect(EXCHANGE_NAME)
+        protocol = Thrift::BinaryProtocol.new(@transport)
+        @client = Test::Client.new(protocol)
+      rescue Bunny::ServerDownError
+        raise "Could not connect - is your local RabbitMQ running?"
+      end
+
+      @transport.open
+    end
+    after(:each) do
+      @server.close
+      @transport.close
+      
+      # Try to kill the exchange and the queue 
+      connection = Bunny.new 
+    end
+    
+    it "should successfully send a message" do
+      client.sendMessage("a message")
+
+      server.spin_server 1
+      server.handler.messages.should include('a message')
+    end 
+    it "should send several messages" do
+      10.times do |i|
+        client.sendMessage("a message (#{i})")
+      end
+
+      server.spin_server 10
+      server.handler.messages.should have(10).messages
+    end 
+    context "with a second server" do
+      attr_reader :second_server
+      before(:each) do
+        # Server setup
+        @second_server = SpecTestServer.new
+      end
+      after(:each) do
+        second_server.close
+      end
+      
+      it "should allow only one server to receive the message/call" do
+        10.times do |i|
+          client.sendMessage("a message (#{i})")
+        end
+
+        server.spin_server 5
+        second_server.spin_server 5
+        
+        messages = server.handler.messages + second_server.handler.messages
+        10.times do |i|
+          messages.should include("a message (#{i})")
+        end          
+      end 
+    end
+  end
+end

data/spec/integration/test.thrift

@@ -0,0 +1,3 @@
+service Test {
+  oneway void sendMessage(1:string message)
+}

data/spec/spec.opts

@@ -0,0 +1 @@
+--colour

data/spec/spec_helper.rb

@@ -0,0 +1,19 @@
+require 'spec'
+
+$:.unshift File.dirname(__FILE__) + '/../lib'
+
+require 'thrift_amqp_transport'
+
+unless defined?(PROJECT_BASE)
+  PROJECT_BASE = File.join(File.dirname(__FILE__), '..')
+end
+
+Spec::Runner.configure do |config|
+  config.mock_with :flexmock
+end
+
+def set_defaults(mock, hash)
+  hash.each do |k, v|
+    mock.should_receive(k).and_return(v).by_default
+  end
+end

data/spec/unit/server_transport_spec.rb

@@ -0,0 +1,64 @@
+require File.dirname(__FILE__) + '/../spec_helper'
+
+describe Thrift::AMQP::ServerTransport do
+  it "should be constructed with an exchange name" do
+    Thrift::AMQP::ServerTransport.new('exchange')
+  end 
+  it "should be constructed with an exchange and a set of headers" do
+    Thrift::AMQP::ServerTransport.new('exchange', :foo => :bar)
+  end 
+  
+  context '#new(exchange)' do
+    attr_reader :transport
+    attr_reader :bunny, :queue, :exchange
+
+    before(:each) do
+      @transport = Thrift::AMQP::ServerTransport.new('exchange')
+
+      # Stub Bunny subsystem
+      @bunny = flexmock(:bunny)
+      @exchange = flexmock(:exchange, :name => 'exchange')
+      @queue = flexmock(:queue)
+      
+      set_defaults(bunny, 
+        :start => nil, 
+        :exchange => exchange, 
+        :queue => queue)
+
+      set_defaults(queue, 
+        :bind => nil)
+
+      flexmock(Bunny, :new => bunny)
+    end
+
+    describe "#listen (1: setup the connection)" do
+      it "should create a bunny connection" do
+        bunny.should_receive(:start).once
+        
+        transport.listen
+      end 
+      it "should create an exchange of type headers" do
+        bunny.should_receive(:exchange).with('exchange', Hash).once
+        
+        transport.listen
+      end 
+    end
+    describe "#accept (2: accept a connection, returns a transport)" do
+      before(:each) do
+        transport.listen
+      end
+      it "should return a queue transport" do
+        transport.accept.should be_instance_of(Thrift::AMQP::Transport)
+      end 
+    end
+    describe "#close (3: closes the server)" do
+      before(:each) do
+        transport.listen
+      end
+
+      it "should not fail" do
+        transport.close
+      end
+    end
+  end
+end

data/spec/unit/transport_spec.rb

@@ -0,0 +1,115 @@
+require File.dirname(__FILE__) + '/../spec_helper'
+
+describe Thrift::AMQP::Transport, 'when constructed with an exchange (server)' do
+  attr_reader :transport, :connection, :exchange, :queue
+  before(:each) do
+    @connection = flexmock(:connection)
+    @exchange = flexmock(:exchange)
+    @queue = flexmock(:queue)
+    
+    # Stub an interface for the connection 
+    set_defaults(connection, 
+      :queue => queue
+    )
+    
+    set_defaults(exchange, 
+      :name => 'exchange'
+    )
+    
+    # Stub an interface for the queue
+    set_defaults(queue, 
+      :bind => nil, 
+      :pop  => :queue_empty
+    )
+    
+    @transport = Thrift::AMQP::Transport.new(connection, exchange)
+  end
+  
+  describe "#read(sz)" do
+    it "should poll for a new message on the queue" do
+      queue.should_receive(:pop).and_return(:queue_empty,:queue_empty,'message')
+      
+      transport.read(1).should == 'm'
+    end
+    it "should return sz bytes" do
+      queue.should_receive(:pop).and_return(' '*1000)
+      
+      transport.read(999).size.should == 999
+      transport.read(10).size.should == 1
+    end
+    it "should return at most the whole message" do
+      queue.should_receive(:pop).and_return(' '*1000)
+      
+      transport.read(2000).size.should == 1000
+    end
+  end
+end
+
+describe Thrift::AMQP::Transport, 'when using .connect (client)' do
+  attr_reader :bunny, :exchange
+  before(:each) do
+    @bunny = flexmock(:bunny)
+    @exchange = flexmock(:exchange)
+    
+    set_defaults(bunny, 
+      :start => nil,
+      :exchange => exchange)
+    
+    set_defaults(exchange, 
+      :name => 'exchange')
+    
+    flexmock(Bunny).should_receive(:new).and_return bunny
+  end
+  describe ".connect(exchange_name)" do
+    it "should create a connection to the AMQP server" do
+      bunny.should_receive(:start).once
+      
+      Thrift::AMQP::Transport.connect('exchange_name')
+    end
+    it "should access the exchange specified" do
+      bunny.should_receive(:exchange).with('exchange_name', Hash).once
+      
+      Thrift::AMQP::Transport.connect('exchange_name')
+    end
+    it "should return a transport instance" do
+      transport = Thrift::AMQP::Transport.connect('exchange_name')
+      
+      transport.should be_instance_of(Thrift::AMQP::Transport)
+    end 
+    it "should raise a nice error when exchange can't be created" do
+      bunny.should_receive(:exchange).and_raise(Bunny::ProtocolError)
+
+      lambda {
+        transport = Thrift::AMQP::Transport.connect('exchange_name')
+      }.should raise_error()
+    end 
+  end
+
+  context 'connected' do
+    attr_reader :transport
+    before(:each) do
+      @transport = Thrift::AMQP::Transport.connect('exchange_name')
+    end
+    
+    describe "write(buffer)" do
+      it "should publish messages to the exchange" do
+        exchange.should_receive(:publish).with("some message (encoded by thrift)").once
+
+        transport.write("some message (encoded by thrift)")
+        transport.flush
+      end
+      it "should not publish on simple write" do
+        exchange.should_receive(:publish).never
+
+        transport.write("some message (encoded by thrift)")
+      end 
+      it "should publish on #flush" do
+        exchange.should_receive(:publish).with("foobar").once
+
+        transport.write("foo")
+        transport.write("bar")
+        transport.flush
+      end 
+    end
+  end
+end

data/thrift_amqp_transport.gemspec

@@ -0,0 +1,34 @@
+(in /Users/kaspar/git/own/thrift_amqp_transport)
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{thrift_amqp_transport}
+  s.version = "0.1.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Kaspar Schiess"]
+  s.date = %q{2009-08-27}
+  s.description = %q{Transports thrift messages over a the advanced message queue protocol. (AMQP)
+This is an early release that features only one-way communication.}
+  s.email = ["eule@space.ch"]
+  s.extra_rdoc_files = ["History.txt", "Manifest.txt", "README.txt"]
+  s.files = ["History.txt", "Manifest.txt", "README.txt", "Rakefile", "lib/thrift_amqp/server_transport.rb", "lib/thrift_amqp/transport.rb", "lib/thrift_amqp_transport.rb", "spec/integration/test.thrift", "spec/integration/thrift_amqp_transport_spec.rb", "spec/spec.opts", "spec/spec_helper.rb", "spec/unit/server_transport_spec.rb", "spec/unit/transport_spec.rb"]
+  s.rdoc_options = ["--main", "README.txt"]
+  s.require_paths = ["lib"]
+  s.rubyforge_project = %q{thrift_amqp_transport}
+  s.rubygems_version = %q{1.3.5}
+  s.summary = %q{Transports thrift messages over a the advanced message queue protocol}
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+      s.add_development_dependency(%q<hoe>, [">= 2.3.2"])
+    else
+      s.add_dependency(%q<hoe>, [">= 2.3.2"])
+    end
+  else
+    s.add_dependency(%q<hoe>, [">= 2.3.2"])
+  end
+end

metadata

@@ -0,0 +1,79 @@
+--- !ruby/object:Gem::Specification 
+name: thrift_amqp_transport
+version: !ruby/object:Gem::Version 
+  version: 0.1.0
+platform: ruby
+authors: 
+- Kaspar Schiess
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-08-31 00:00:00 +02:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: hoe
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 2.3.3
+    version: 
+description: Transports thrift messages over a the advanced message queue protocol. (AMQP) Because of the unconnected broadcasting nature of the message queue, this  transport supports only one-way communication.   The usage scenario is that you would use this to broadcast information about  services (1 producer, n consumers) and then create point to point connections  from client to service using normal (TCP) thrift. You gain the advantage of using only one interface definition language (IDL).
+email: 
+- eule@space.ch
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- History.txt
+- Manifest.txt
+- README.txt
+files: 
+- History.txt
+- Manifest.txt
+- README.txt
+- Rakefile
+- lib/thrift_amqp/server_transport.rb
+- lib/thrift_amqp/transport.rb
+- lib/thrift_amqp_transport.rb
+- spec/integration/oneway_spec.rb
+- spec/integration/test.thrift
+- spec/spec.opts
+- spec/spec_helper.rb
+- spec/unit/server_transport_spec.rb
+- spec/unit/transport_spec.rb
+- thrift_amqp_transport.gemspec
+has_rdoc: true
+homepage: 
+post_install_message: 
+rdoc_options: 
+- --main
+- README.txt
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: thrift_amqp_transport
+rubygems_version: 1.3.1
+signing_key: 
+specification_version: 2
+summary: Transports thrift messages over a the advanced message queue protocol
+test_files: []
+