experella-proxy 0.0.6 → 0.0.9

data/.gitignore

@@ -11,5 +11,13 @@
 
 .rspec
 
+#rvm
+.ruby-version
+.ruby-gemset
+
+#code coverage
+coverage
+
+
 Gemfile.lock
 *.gem

data/README.md

@@ -1,6 +1,9 @@
 #Experella-Proxy
 
-A balancing EventMachine reverse proxy based on [em-proxy](https://github.com/igrigorik/em-proxy)
+[![Gem Version](https://badge.fury.io/rb/experella-proxy.png)](http://badge.fury.io/rb/experella-proxy)
+
+A balancing EventMachine reverse proxy based on [em-proxy](https://github.com/igrigorik/em-proxy). 
+See our [presentation](http://experteer.github.io/experella-proxy/index.html) for a more detailed overview.
 
 Configurable in pure ruby!
 
@@ -17,7 +20,7 @@ Supports:
 Proxy uses [http_parser](https://github.com/tmm1/http_parser.rb) to parse http data and is thereby subject to the parsers restrictions
 
 The proxy is build for low proxy to server latency and does not support persistent connections to servers. Keep that in mind
-as i can severely influence proxy performance overhead.
+as it can severely influence proxy performance overhead.
 
 It balances for every single http-request and not per client/connection.
 
@@ -205,6 +208,18 @@ Override server's run function
     end
 ```
 
+## Development
+
+In the dev folder a development binary is provided which allows execution without installation as gem.
+
+The test folder provides simple sinatra servers for testing/debugging which can be run with rake tasks.
+
+Additionally you can activate simplecov code coverage analysis for specs by setting COVERAGE=true
+
+```
+$> COVERAGE=true rake spec
+```
+
 ## Additional Information
 
 + [em-proxy](https://github.com/igrigorik/em-proxy)

data/Rakefile

@@ -1,3 +1,4 @@
+require 'bundler/gem_tasks'
 require 'rake'
 require 'rspec/core/rake_task'
 require 'yard'

data/bin/experella-proxy

@@ -1,5 +1,13 @@
 #!/usr/bin/env ruby
 
+if ENV["COVERAGE"]
+  #simplecov support for integration specs
+  require 'simplecov'
+  SimpleCov.start do
+    command_name ENV["TESTNAME"] || "integration-test"
+  end
+end
+
 lib = File.expand_path(File.join(File.dirname(__FILE__), '../lib/'))
 
 require 'fileutils'

data/config/default/config.rb

@@ -1,5 +1,5 @@
 #Backend Server
-base_backend_port=(ENV["BASE_PORT"] : 4000).to_i
+base_backend_port=(ENV["BASE_PORT"] || 4000).to_i
 backend1_port=base_backend_port+1
 backend2_port=base_backend_port+2
 
@@ -47,7 +47,7 @@ logger.level = Logger::WARN
 # you can add one pair per call to set_proxy(hsh)
 # additionally you can activate ssl with provided private_key and cert_chain files
 set_proxy(:host => "localhost", :port => base_backend_port)
-set_proxy(:host => "localhost", :port => 443,
+set_proxy(:host => "localhost", :port => base_backend_port + 443,
           :options => {:tls => true,
                        :private_key_file => 'ssl/private/experella-proxy.key',
                        :cert_chain_file => 'ssl/certs/experella-proxy.pem'}

data/dev/experella-proxy

@@ -7,6 +7,14 @@ Bundler.setup
 
 #NOTE: This is exactly the same as /bin/experella-proxy from here
 
+if ENV["COVERAGE"]
+  #simplecov support for integration specs
+  require 'simplecov'
+  SimpleCov.start do
+    command_name ENV["TESTNAME"] || "integration-test"
+  end
+end
+
 lib = File.expand_path(File.join(File.dirname(__FILE__), '../lib/'))
 
 require 'fileutils'

data/experella-proxy.gemspec

@@ -28,6 +28,8 @@ Gem::Specification.new do |spec|
   # documentation tool
   spec.add_development_dependency "yard", "~> 0.8.7.3"
   spec.add_development_dependency "redcarpet", "~> 2.3.0"
+  #code coverage
+  spec.add_development_dependency "simplecov", "~> 0.7.1"
 
   spec.files         = Dir["bin/*"] + Dir["dev/*"] + Dir["lib/**/*"] + Dir["config/default/**/*"]
   spec.files        += Dir["spec/**/*"] + Dir["test/sinatra/*"]

data/lib/experella-proxy/backend.rb

@@ -27,7 +27,7 @@ module ExperellaProxy
     #
     # @param data [String] Opaque response data
     def receive_data(data)
-      log.debug [:receive_backend, @name, data]
+      log.debug [:receive_backend, @name]
       @plexer.relay_from_backend(@name, data)
     end
 

data/lib/experella-proxy/backend_server.rb

@@ -2,13 +2,13 @@ module ExperellaProxy
 
   # BackendServer objects contain information on available BackendServers
   #
-  # Accepts Requests based on Request header information matched to it's message_pattern
+  # Accepts Requests based on Request header information and it's message_matcher
   #
   # See {#initialize}
   class BackendServer
 
     attr_accessor :host, :port, :concurrency, :workload, :name
-    attr_reader :message_pattern, :mangle
+    attr_reader :message_matcher, :mangle
 
     # Constructor of the BackendServer
     #
@@ -39,8 +39,8 @@ module ExperellaProxy
     # @param [Hash] options
     # @option options [String] :name name used in logs and for storage. will use Host:Port if no name is specified
     # @option options [String] :concurrency concurrency. will use 1 as default
-    # @option options [Hash|Proc] :accepts  message_pattern for request headers this BackendServer accepts or a lambda where the request is passed
-    #   Keys get symbolized and values create Regexp objects. Empty Hash is default
+    # @option options [Hash|Proc] :accepts  message_pattern that will be converted to a message_matcher or an arbitrary message_matcher as proc
+    #   Empty Hash is default
     # @option options [Hash] :mangle Hash which can modify request headers. Keys get symbolized. nil is default
     def initialize(host, port, options = {})
       @host = host #host URL as string
@@ -53,7 +53,7 @@ module ExperellaProxy
       end
       @workload = 0
 
-      update_message_pattern(options[:accepts])
+      make_message_matcher(options[:accepts])
 
       #mangle can be nil
       @mangle = options[:mangle]
@@ -61,22 +61,21 @@ module ExperellaProxy
       @mangle = @mangle.inject({}) { |memo, (k, v)| memo[k.to_sym] = v; memo } unless @mangle.nil?
     end
 
-    # compares Backend servers accepted message_pattern to request object
+    # compares Backend servers message_matcher to request object
     #
     # @param request [Request] a request object
     # @return [Boolean] true if BackendServer accepts the Request, false otherwise
     def accept?(request)
-      res=@message_pattern.call(request)
+      res=@message_matcher.call(request)
       #puts "#{name} #{request.header['request_url']} #{res}"
       res
     end
 
-    # Updates the message_pattern
+    # Makes a message matching block from the message_pattern.
     #
-    # @param hsh [Hash] hash containing additional message_pattern information. Keys get symbolized and values create
-    #   Regexp objects. Values of duplicate keys will be overwritten
-    def update_message_pattern(obj)
-      @message_pattern =if obj.respond_to?(:call)
+    # @param obj [Hash|Proc] hash containing a message_pattern that will be converted to a message_matcher proc or an arbitrary own message_matcher
+    def make_message_matcher(obj)
+      @message_matcher =if obj.respond_to?(:call)
         obj
       else
         #precompile message pattern keys to symbols and values to regexp objects

data/lib/experella-proxy/connection.rb

@@ -105,20 +105,10 @@ module ExperellaProxy
     def connect_backendserver(backend)
       @backend = backend
       connection_manager.free_connection(self)
-      #mangle http header if backend wants to
-      unless @backend.mangle.nil?
-        @backend.mangle.each do |k, v|
-          if v.respond_to?(:call)
-            get_request.update_header({k => v.call(get_request.header[k])})
-          else
-            get_request.update_header({k => v})
-          end
-        end
-      end
-
+      #mangle http headers
+      mangle
       # reconstruct the request header
       get_request.reconstruct_header
-
       #special web support for unknown hosts
       if @backend.name.eql?("web")
         xport = get_request.header[:Host].match(/:[0-9]+/)
@@ -197,7 +187,7 @@ module ExperellaProxy
     # @param data [String] opaque response data
     def relay_from_backend(name, data)
       log.info msec + 'on_response'.ljust(12) + " @" + @signature.to_s + " from #{name}"
-      log.debug msec + "#{name.inspect} " + data
+      log.debug [msec, "#{name.inspect}", data]
       @got_response = true
       data = @on_response.call(name, data) if @on_response
       get_request.response << data
@@ -449,8 +439,12 @@ module ExperellaProxy
 
         # remove all hop-by-hop header fields
         unless h["Connection"].nil?
-          h["Connection"].each do |s|
-            h.delete(s)
+          if h["Connection"].is_a?(String)
+            h.delete(h["Connection"])
+          else
+            h["Connection"].each do |s|
+              h.delete(s)
+            end
           end
         end
         HOP_HEADERS.each do |s|
@@ -514,6 +508,20 @@ module ExperellaProxy
 
     end
 
+    # Mangles http headers based on backend specific mangle configuration
+    #
+    def mangle
+      unless @backend.mangle.nil?
+        @backend.mangle.each do |k, v|
+          if v.respond_to?(:call)
+            get_request.update_header({k => v.call(get_request.header[k])})
+          else
+            get_request.update_header({k => v})
+          end
+        end
+      end
+    end
+
     # This method sends the first requests send_buffer to the backend server, if
     # any backend is set and there is request data to dispatch
     #

data/lib/experella-proxy/proxy.rb

@@ -53,7 +53,6 @@ module ExperellaProxy
     #
     def self.stop
       if EM.reactor_running?
-        log.info("Terminating experella-proxy")
         EventMachine::stop_event_loop
       end
     end

data/lib/experella-proxy/response.rb

@@ -18,6 +18,8 @@ module ExperellaProxy
       @conn = request.conn
       @header = {}
       @status_code = 500
+      @no_length = false #set true if no content-length or transfer-encoding given
+      @keep_parsing = true #used for special no length case
       @chunked = false # if true the parsed body will be chunked
       @buffer = false # default is false, so incoming data will be streamed,
       # used for http1.0 clients and transfer-encoding chunked backend responses
@@ -35,7 +37,15 @@ module ExperellaProxy
     # @param str [String] data as string
     def <<(str)
       begin
-        @response_parser << str
+        if @keep_parsing
+          offset = @response_parser << str
+
+        #edge case for message without content-length and transfer encoding
+          @conn.send_data str[offset..-1] unless @keep_parsing
+        else
+          @conn.send_data str
+        end
+
       rescue Http::Parser::Error
         log.warn ["Parser error caused by invalid response data", "@#{@conn.signature}"]
         # on error unbind response_parser object, so additional data doesn't get parsed anymore
@@ -128,6 +138,7 @@ module ExperellaProxy
           # if no transfer-encoding and no content-length is present, set Connection: close
           if h["Content-Length"].nil?
             @request.keep_alive = false
+            @no_length = true
             @header[:Connection] = "close"
           end
         #chunked encoded
@@ -144,8 +155,12 @@ module ExperellaProxy
 
         # remove all hop-by-hop header fields
         unless h["Connection"].nil?
-          h["Connection"].each do |s|
-            h.delete(s)
+          if h["Connection"].is_a?(String)
+            h.delete(h["Connection"])
+          else
+            h["Connection"].each do |s|
+              h.delete(s)
+            end
           end
         end
         HOP_HEADERS.each do |s|
@@ -161,7 +176,6 @@ module ExperellaProxy
         end
         @header[:Via] = via
 
-
         update_header(h)
         unless @buffer
           # called before any data is put into send_buffer
@@ -197,8 +211,11 @@ module ExperellaProxy
           @send_buffer << body
           @conn.send_data flush
         end
-      end
 
+        if @no_length
+          @keep_parsing = false
+        end
+      end
     end
   end
 end

data/lib/experella-proxy/version.rb

@@ -1,5 +1,15 @@
 module ExperellaProxy
   # ExperellaProxy Gemversion
+  # 0.0.9
+  # * added simplecov code coverage support.
+  # * removed an unescaped duplicate debug log output
+  # 0.0.8
+  # * fixed a parsing bug where no data was send when backend used connection close to indicate message end
+  # 0.0.7
+  # * fixed minor issues with ruby 2.0
+  # * fixed a typo in default config
+  # * refactored mangling in own method
+  # * refactored message pattern and matching
   # 0.0.6
   # * updated homepage
   # 0.0.5
@@ -11,5 +21,5 @@ module ExperellaProxy
   # * added self-signed SSL certificate for TLS/HTTPS
   # * added config template init functionality
   #
-  VERSION = "0.0.6"
+  VERSION = "0.0.9"
 end

data/spec/experella-proxy/backend_server_spec.rb

@@ -4,15 +4,15 @@ describe ExperellaProxy::BackendServer do
 
   let(:backend) {
     ExperellaProxy::BackendServer.new("host", "port", {:concurrency => "2", :name => "name",
-                                                       :accepts => {"Host" => "experella"},
+                                                       :accepts => {"Host" => "experella", :path => "ella"},
                                                        :mangle => {"Connection" => "close"}
                                                       })
   }
   let(:min_backend) {
     ExperellaProxy::BackendServer.new("host", "port")
   }
-  let(:pattern) {
-    backend.message_pattern
+  let(:matcher) {
+    backend.message_matcher
   }
 
   describe "#new" do
@@ -39,9 +39,9 @@ describe ExperellaProxy::BackendServer do
       min_backend.concurrency.should eql 1
     end
 
-    it "has a message pattern" do
-      backend.message_pattern.should_not be_nil
-      min_backend.message_pattern.should_not be_nil
+    it "has a message matcher" do
+      backend.message_matcher.should_not be_nil
+      min_backend.message_matcher.should_not be_nil
     end
 
     it "mangle should be nil" do
@@ -54,45 +54,53 @@ describe ExperellaProxy::BackendServer do
     end
   end
 
-  shared_examples "the message pattern" do
+  shared_examples "the message matcher" do
 
     it "returns a proc" do
-      pattern.should be_an_instance_of Proc
+      matcher.should be_an_instance_of Proc
     end
 
   end
 
-
-  describe "#update_message_pattern" do
-
-    #it "adds the hash parameter to the message_pattern and overwrites values of duplicate keys" do
-    #  backend.update_message_pattern({"Host" => "experella.de", "Connection" => "keep-alive"})
-    #  pattern = backend.message_pattern
-    #  pattern[:Host].should eql Regexp.new("experella.de")
-    #  pattern[:Connection].should eql Regexp.new("keep-alive")
-    #end
-
-    it_should_behave_like "the message pattern"
-  end
-
   describe "#accept?" do
 
 
-    it "returns false if request header doesn't have all keys of message_pattern" do
+    it "returns false if desired request header missing" do
       request = ExperellaProxy::Request.new("")
       request.update_header(:request_url => "/docs")
       backend.accept?(request).should be_false
     end
 
-    it "returns false if request headers don't match full message_pattern" do
+
+    it "returns false if request headers doesn't match the message_matcher" do
       request = ExperellaProxy::Request.new("")
-      request.update_header(:Host => "google.com", :request_url => "/docs")
+      request.update_header(:Host => "experella.com", :request_url => "/docs")
+      backend.accept?(request).should be_false
+      request.uri.update(:path => "godzilla")
       backend.accept?(request).should be_false
     end
 
-    it "returns true if full message pattern finds matches in request headers" do
+    it "accepts a block as message matcher" do
+      lambdaBackend = ExperellaProxy::BackendServer.new("host", "port", {:concurrency => "2", :name => "name",
+                                                         :accepts => lambda { |req|
+                                                           if req.header[:Host] == "google.com"
+                                                             true
+                                                           else
+                                                             false
+                                                           end
+                                                         }
+      })
       request = ExperellaProxy::Request.new("")
+      request.update_header(:Host => "google.com", :request_url => "/docs")
+      lambdaBackend.accept?(request).should be_true
       request.update_header(:Host => "experella.com", :request_url => "/docs")
+      lambdaBackend.accept?(request).should be_false
+    end
+
+    it "returns true if full message_matcher finds matches in request headers" do
+      request = ExperellaProxy::Request.new("")
+      request.update_header(:Host => "experella.com")
+      request.uri.update(:path => "experella")
       backend.accept?(request).should be_true
     end
 

data/spec/experella-proxy/configuration_spec.rb

@@ -5,10 +5,6 @@ describe ExperellaProxy::Configuration do
     ExperellaProxy.config
   }
 
-  let(:no_config){
-    ExperellaProxy::Configuration.new(:configfile => "")
-  }
-
   it "should load a config file" do
     config.backends.size.should == 4
     config.timeout.should == 6.0

data/spec/experella-proxy/experella-proxy_spec.rb

@@ -16,7 +16,20 @@ describe ExperellaProxy do
     File.expand_path("../../../bin/experella-proxy", __FILE__)
   }
 
-
+  # static testnames send to spawned experella for simplecov
+  ENV_TESTNAMES = {
+    "should get response from the echoserver via the proxy" => "response",
+    "should respond with 404" => "404",
+    "should respond with 400 on malformed request" => "400",
+    "should respond with 503" => "503",
+    "should reuse keep-alive connections" => "keep-alive",
+    "should handle chunked post requests and strip invalid Content-Length" => "chunked-request",
+    "should rechunk and stream Transfer-Encoding chunked responses" => "chunked-response",
+    "should timeout inactive connections after config.timeout" => "timeout",
+    "should handle pipelined requests correctly" => "pipelined",
+    "should accept requests on all set proxy domains" => "multiproxy",
+    "should be able to handle post requests" => "post"
+  }
 
   describe "EchoServer" do
     before :each do
@@ -46,7 +59,10 @@ describe ExperellaProxy do
   end
 
   describe "Proxy" do
-    before :each do
+    before :each do |test|
+      if ENV["COVERAGE"]
+        ENV["TESTNAME"] = ENV_TESTNAMES[test.example.description]
+      end
       @pid = spawn("ruby", "#{echo_server}", "127.0.0.10", "7654")
       @pid2 = spawn("#{experella_proxy}", "run", "--", "--config=#{File.join(File.dirname(__FILE__),"/../fixtures/test_config.rb")}")
       sleep(0.8) #let the server startup, specs may fail if this is set to low
@@ -335,6 +351,7 @@ describe ExperellaProxy do
             req1 = conn.get({:connect_timeout => 1, :inactivity_timeout => config.timeout + 5,
                              :keepalive => true, :head => {"Host" => "experella.com"}})
             req1.errback {
+              #this shouldnt happen, but when it does it should at least be because of a timeout
               time = Time.now - time
               time.should >= config.timeout
               time.should < config.timeout + 5

data/spec/fixtures/test_config.rb

@@ -14,7 +14,7 @@ backend( :name => "exp proxy", :host => "127.0.0.11", :port => "7655", :concurre
   :accepts => {"Host" => "experella", "request_url" => "/(#{request_part})($|/)"}
 )
 
-backend( :name => "web", :host => "0.0.0.0", :port => "80", :concurrency => "1000",
+backend( :name => "web", :host_port => "0.0.0.0:80", :concurrency => "1000",
   :accepts => {"Host" => "^((?!(experella|127)).)*$"}
 )
 

data/spec/spec_helper.rb

@@ -8,6 +8,15 @@ require 'rubygems'
 require 'bundler'
 Bundler.setup
 
+if ENV["COVERAGE"]
+  require "simplecov"
+  SimpleCov.start do
+    add_group "Proxy", "lib"
+    add_group "Specs", "spec"
+  end
+end
+
+
 require 'pathname'
 
 LIB_ROOT= Pathname.new(File.dirname(__FILE__) +"/../")

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: experella-proxy
 version: !ruby/object:Gem::Version 
-  hash: 19
+  hash: 13
   prerelease: 
   segments: 
   - 0
   - 0
-  - 6
-  version: 0.0.6
+  - 9
+  version: 0.0.9
 platform: ruby
 authors: 
 - Dennis-Florian Herr
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2014-02-12 00:00:00 +01:00
+date: 2014-04-07 00:00:00 +02:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -191,6 +191,22 @@ dependencies:
         version: 2.3.0
   type: :development
   version_requirements: *id011
+- !ruby/object:Gem::Dependency 
+  name: simplecov
+  prerelease: false
+  requirement: &id012 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 1
+        segments: 
+        - 0
+        - 7
+        - 1
+        version: 0.7.1
+  type: :development
+  version_requirements: *id012
 description: a balancing & routing proxy, see README for more details
 email: 
 - dennis.herr@experteer.com