bowtie-io-rack-streaming-proxy 2.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: cea9bd8a7ee62c0e0606871db98bba8f5de3e3e8
+  data.tar.gz: 7c5cfcf36d77ed6e53a6f478a2fce63f9b7037bd
+SHA512:
+  metadata.gz: 5e1f9c1c0f8471331bdd2baead76a7754d1a88aad4691b15757bdf892814760faa2118135c5a9cb1caf065f931490f3e2a54f4eed2a8bf66dcec6a5fd20ab1e0
+  data.tar.gz: 1e78c5b5686d47e41d933658c8af6b813ea10dc7a2ceeaec74c74b5273569cee33f7719fab178f677a4fe68baaf5bcb9f05c8c030455fbb8fa01630ba002d7b2

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in rack-streaming_proxy.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2009-2013 Fred Ngo, Nathan Witmer
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,95 @@
+# Rack::StreamingProxy
+
+A transparent streaming proxy to be used as rack middleware.
+
+* Streams the response from the downstream server to minimize memory usage
+* Handles chunked encoding if used
+* Proxies GET/PUT/POST/DELETE, XHR, and cookies
+
+Now updated to be compatible with Rails 3 and 4, and fixes major concurrency issues that were present in 1.0.
+
+Use Rack::StreamingProxy when you need to have the response streamed back to the client, for example when handling large file requests that could be proxied directly but need to be authenticated against the rest of your middleware stack.
+
+Note that this will not work well with EventMachine. EM buffers the entire rack response before sending it to the client. When testing, try Unicorn or Passenger rather than the EM-based Thin (See [discussion](http://groups.google.com/group/thin-ruby/browse_thread/thread/4762f8f851b965f6)).
+
+A simple streamer app has been included for testing and development.
+
+## Usage
+
+To use inside a Rails app, add a `config/initializers/streaming_proxy.rb` initialization file, and place in it:
+
+```ruby
+require 'rack/streaming_proxy'
+
+YourRailsApp::Application.configure do
+  config.streaming_proxy.logger             = Rails.logger                          # stdout by default
+  config.streaming_proxy.log_verbosity      = Rails.env.production? ? :low : :high  # :low or :high, :low by default
+  config.streaming_proxy.num_retries_on_5xx = 5                                     # 0 by default
+  config.streaming_proxy.raise_on_5xx       = true                                  # false by default
+
+  # Will be inserted at the end of the middleware stack by default.
+  config.middleware.use Rack::StreamingProxy::Proxy do |request|
+
+    # Inside the request block, return the full URI to redirect the request to,
+    # or nil/false if the request should continue on down the middleware stack.
+    if request.path.start_with?('/search')
+      "http://www.some-other-service.com/search?#{request.query}"
+    end
+  end
+end
+```
+
+To use as a Rack app:
+
+```ruby
+require 'rack/streaming_proxy'
+
+use Rack::StreamingProxy::Proxy do |request|
+  if request.path.start_with?('/proxy')
+    # You probably want to get rid of the '/proxy' in the path, when requesting from the destination.
+    proxy_path = request.path.sub %r{^/proxy}, ''
+    "http://www.another-server.com#{proxy_path}"
+  end
+end
+```
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'rack-streaming-proxy'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install rack-streaming-proxy
+
+## Requirements
+
+* Ruby = 1.9.3
+* rack >= 1.4
+* servolux ~> 0.10
+
+These requirements (other than Ruby) will be automatically installed via Bundler.
+
+This gem has not been tested with versions lower than those indicated.
+
+This gem works with Ubuntu 10.04. It has not been tested with later versions of Ubuntu or other Linuxes, but it should work just fine. It has not been tested with OS X but should work as well. However, I doubt it will work on any version of Windows, as it does process-based stuff. You are most welcome to try it and report back.
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Implement your changes, and make sure to add tests!
+4. Commit your changes (`git commit -am 'Add some feature'`)
+5. Push to the branch (`git push origin my-new-feature`)
+6. Create new Pull Request
+
+## Thanks To
+
+* [Nathan Witmer](http://github.com/zerowidth) for the 1.0 implementation of [Rack::StreamingProxy](http://github.com/zerowidth/rack-streaming-proxy)
+* [Tom Lea](http://github.com/cwninja) for [Rack::Proxy](http://gist.github.com/207938), which inspired Rack::StreamingProxy.
+* [Tim Pease](http://github.com/TwP) for [Servolux](https://github.com/Twp/servolux)

data/README.txt

@@ -0,0 +1,78 @@
+rack-streaming-proxy
+    by Nathan Witmer <nwitmer@gmail.com>
+    http://github.com/zerowidth/rack-streaming-proxy
+
+== DESCRIPTION:
+
+Streaming proxy for Rack, the rainbows to Rack::Proxy's unicorn.
+
+== FEATURES/PROBLEMS:
+
+Provides a transparent streaming proxy to be used as rack middleware.
+
+* Streams the response from the downstream server to minimize memory usage
+* Handles chunked encoding if used
+* Proxies GET/PUT/POST/DELETE, XHR, and cookies
+
+Use this when you need to have the response streamed back to the client,
+for example when handling large file requests that could be proxied
+directly but need to be authenticated against the rest of your middleware
+stack.
+
+Note that this will not work well with EventMachine. EM buffers the entire
+rack response before sending it to the client. When testing, try
+mongrel (via rackup) or passenger, rather than the EM-based thin. See
+http://groups.google.com/group/thin-ruby/browse_thread/thread/4762f8f851b965f6
+for more discussion.
+
+I've included a simple streamer app for testing and development.
+
+Thanks to:
+
+* Tom Lea (cwninja) for Rack::Proxy (http://gist.github.com/207938)
+* Tim Pease for bones, servolux, &c
+
+== SYNOPSIS:
+
+  require "rack/streaming_proxy"
+
+  use Rack::StreamingProxy do |request|
+    # inside the request block, return the full URI to redirect the request to,
+    # or nil/false if the request should continue on down the middleware stack.
+    if request.path.start_with?("/proxy")
+      "http://another_server#{request.path}"
+    end
+  end
+
+== REQUIREMENTS:
+
+* servolux (gem install servolux)
+
+== INSTALL:
+
+* sudo gem install rack-streaming-proxy --source http://gemcutter.org
+
+== LICENSE:
+
+(The MIT License)
+
+Copyright (c) 2009 Nathan Witmer
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/dev/client.rb

@@ -0,0 +1,24 @@
+#!/usr/bin/env ruby
+# coding: utf-8
+# cf. https://gist.github.com/sonots/7751554
+
+require 'net/http'
+require 'uri'
+
+# unicorn spec/app.ru -p 4321
+# unicorn spec/proxy.ru -p 4322
+PORT = ARGV[0] || 8080
+
+http = Net::HTTP.new "localhost", PORT
+request = Net::HTTP::Get.new "/slow_stream"
+#request['Transfer-Encoding'] = 'chunked'
+request['Connection'] = 'keep-alive'
+http.request(request){|response|
+  puts "content-length: #{response.content_length}"
+  body = []
+  response.read_body{|x|
+    body << Time.now
+    puts "read_block: #{body.length}, #{x.size}byte(s)"
+  }
+  puts body
+}

data/dev/proxy.ru

@@ -0,0 +1,13 @@
+require File.expand_path(
+    File.join(File.dirname(__FILE__), %w[.. lib rack streaming_proxy]))
+
+use Rack::Reloader, 1
+# use Rack::CommonLogger # rackup already has commonlogger loaded
+use Rack::Lint
+use Rack::StreamingProxy do |req|
+  url = "http://localhost:4000#{req.path}"
+  url << "?#{req.query_string}" unless req.query_string.empty?
+  url
+end
+
+run lambda { |env| [200, {"Content-Type" => "text/plain"}, ""] }

data/dev/streamer.ru

@@ -0,0 +1,54 @@
+class Streamer
+  include Rack::Utils
+
+  def call(env)
+    req = Rack::Request.new(env)
+    headers = {"Content-Type" => "text/plain"}
+
+    @chunked = req.path.start_with?("/chunked")
+
+    if count = req.path.match(/(\d+)$/)
+      count = count[0].to_i
+    else
+      count = 100
+    end
+    @strings = count.times.collect {|n| "~~~~~ #{n} ~~~~~\n" }
+
+    if chunked?
+      headers["Transfer-Encoding"] = "chunked"
+    else
+      headers["Content-Length"] = @strings.inject(0) {|sum, s| sum += bytesize(s)}.to_s
+    end
+
+    [200, headers, self.dup]
+  end
+
+  def each
+    term = "\r\n"
+    @strings.each do |chunk|
+      if chunked?
+        size = bytesize(chunk)
+        yield [size.to_s(16), term, chunk, term].join
+      else
+        yield chunk
+      end
+      sleep 0.05
+    end
+    yield ["0", term, "", term].join if chunked?
+  end
+
+  protected
+
+  def chunked?
+    @chunked
+  end
+end
+
+# use Rack::CommonLogger # rackup already has commonlogger loaded
+use Rack::Lint
+
+# GET /
+# GET /10
+# GET /chunked
+# GET /chunked/10
+run Streamer.new

data/lib/rack/streaming_proxy.rb

@@ -0,0 +1,2 @@
+require 'rack/streaming_proxy/version'
+require 'rack/streaming_proxy/proxy'

data/lib/rack/streaming_proxy/errors.rb

@@ -0,0 +1,5 @@
+module Rack::StreamingProxy
+  class Error           < RuntimeError; end
+  class UnknownError    < Error;        end
+  class HttpServerError < Error;        end
+end

data/lib/rack/streaming_proxy/proxy.rb

@@ -0,0 +1,120 @@
+require 'rack'
+require 'logger'
+require 'rack/streaming_proxy/session'
+require 'rack/streaming_proxy/request'
+require 'rack/streaming_proxy/response'
+
+class Rack::StreamingProxy::Proxy
+  class << self
+    # Logs to stdout by default unless configured with another logger via Railtie.
+    def logger
+      Logger.new(STDOUT)
+    end
+
+    # At :low verbosity by default -- will not output :debug level messages.
+    # :high verbosity outputs :debug level messages.
+    # This is independent of the Logger's log_level, as set in Rails, for example,
+    # although the Logger's level can override this setting.
+    def log_verbosity
+      :low
+    end
+
+    # No retries are performed by default.
+    def num_retries_on_5xx
+      0
+    end
+
+    # If the proxy cannot recover from 5xx's through retries (see num_retries_on_5xx),
+    # then it by default passes through the content from the destination
+    # e.g. the Apache error page. If you want an exception to be raised instead so
+    # you can handle it yourself (i.e. display your own error page), set raise_on_5xx to true.
+    def raise_on_5xx
+      false
+    end
+
+    def log(level, message)
+      unless log_verbosity == :low && level == :debug
+        logger.send level, "[Rack::StreamingProxy] #{message}"
+      end
+    end
+  end
+
+  # The block provided to the initializer is given a Rack::Request
+  # and should return:
+  #
+  #   * nil/false to skip the proxy and continue down the stack
+  #   * a complete uri (with query string if applicable) to proxy to
+  #
+  # Example:
+  #
+  #   use Rack::StreamingProxy::Proxy do |req|
+  #     if req.path.start_with?('/search')
+  #       "http://some_other_service/search?#{req.query}"
+  #     end
+  #   end
+  #
+  # Most headers, request body, and HTTP method are preserved.
+  #
+  def initialize(app, &block)
+    @app   = app
+    @block = block
+  end
+
+  def call(env)
+    current_request = Rack::Request.new(env)
+    destination_uri = destination_uri(current_request)
+
+    # Decide whether this request should be proxied.
+    if destination_uri
+      self.class.log :info, "Starting proxy request to: #{destination_uri}"
+
+      request  = Rack::StreamingProxy::Request.new(destination_uri, current_request)
+      begin
+        response = Rack::StreamingProxy::Session.new(request).start
+      rescue Exception => e # Rescuing only for the purpose of logging to rack.errors
+        log_rack_error(env, e)
+        raise e
+      end
+
+      # Notify client http version to the instance of Response class.
+      response.client_http_version = env['HTTP_VERSION'].sub(/HTTP\//, '') if env.has_key?('HTTP_VERSION')
+      # Ideally, both a Content-Length header field and a Transfer-Encoding 
+      # header field are not expected to be present from servers which 
+      # are compliant with RFC2616. However, irresponsible servers may send 
+      # both to rack-streaming-proxy.
+      # RFC2616 says if a message is received with both a Transfer-Encoding 
+      # header field and a Content-Length header field, the latter MUST be 
+      # ignored. So I deleted a Content-Length header here.
+      #
+      # Though there is a case that rack-streaming-proxy deletes both a 
+      # Content-Length and a Transfer-Encoding, a client can acknowledge the 
+      # end of body by closing the connection when the entire response has 
+      # been sent without a Content-Length header. So a Content-Length header 
+      # does not have to be required here in our understaing.
+      response.headers.delete('Content-Length') if response.headers.has_key?('Transfer-Encoding')
+      if env.has_key?('HTTP_VERSION') && env['HTTP_VERSION'] < 'HTTP/1.1'
+        # Be compliant with RFC2146
+        response.headers.delete('Transfer-Encoding')
+      end
+
+      self.class.log :info, "Finishing proxy request to: #{destination_uri}"
+      [response.status, response.headers, response]
+
+    # Continue down the middleware stack if the request is not to be proxied.
+    else
+      @app.call(env)
+    end
+  end
+
+  private
+  def destination_uri(rack_request)
+    nil
+  end
+
+  def log_rack_error(env, e)
+    env['rack.errors'].puts e.message
+    env['rack.errors'].puts e.backtrace #.collect { |line| "\t" + line }
+    env['rack.errors'].flush
+  end
+
+end

data/lib/rack/streaming_proxy/request.rb

@@ -0,0 +1,80 @@
+require 'uri'
+require 'net/https'
+
+class Rack::StreamingProxy::Request
+
+  attr_reader :http_request, :rack_request
+
+  def initialize(destination_uri, current_request)
+    @destination_uri = URI.parse(destination_uri)
+    @rack_request    = current_request
+    @http_request    = translate_request(current_request, @destination_uri)
+  end
+
+  def host
+    @destination_uri.host
+  end
+
+  def port
+    @destination_uri.port
+  end
+
+  def use_ssl?
+    @destination_uri.is_a? URI::HTTPS
+  end
+
+  def use_basic_auth?
+    @destination_uri.user && @destination_uri.password
+  end
+
+  def uri
+    @destination_uri.to_s
+  end
+
+private
+
+  def translate_request(current_request, uri)
+    method = current_request.request_method.downcase
+    method[0..0] = method[0..0].upcase
+
+    request = Net::HTTP.const_get(method).new("#{uri.path}#{"?" if uri.query}#{uri.query}")
+
+    if request.request_body_permitted? and current_request.body
+      request.body_stream    = current_request.body
+      request.content_length = current_request.content_length if current_request.content_length
+      request.content_type   = current_request.content_type   if current_request.content_type
+    end
+
+    log_headers :debug, 'Current Request Headers', current_request.env
+
+    current_headers = current_request.env.reject { |key, value| !(key.match /^HTTP_/) }
+    current_headers.each do |key, value|
+      fixed_name = key.sub(/^HTTP_/, '').gsub('_', '-')
+      request[fixed_name] = value unless fixed_name.downcase == 'host'
+    end
+    request['X-Forwarded-For'] = (current_request.env['X-Forwarded-For'].to_s.split(/, +/) + [current_request.env['REMOTE_ADDR']]).join(', ')
+
+    if rack_request.env[:'proxy_addon_headers']
+      rack_request.env[:'proxy_addon_headers'].each do |k,v|
+        request[k] = v
+      end
+    end
+
+    if use_basic_auth?
+      request.basic_auth @destination_uri.user, @destination_uri.password
+    end
+
+    log_headers :debug, 'Proxy Request Headers:', request
+
+    request
+  end
+
+  def log_headers(level, title, headers)
+    Rack::StreamingProxy::Proxy.log level, "+-------------------------------------------------------------"
+    Rack::StreamingProxy::Proxy.log level, "| #{title}"
+    Rack::StreamingProxy::Proxy.log level, "+-------------------------------------------------------------"
+    headers.each { |key, value| Rack::StreamingProxy::Proxy.log level, "| #{key} = #{value.to_s}" }
+    Rack::StreamingProxy::Proxy.log level, "+-------------------------------------------------------------"
+  end
+
+end

data/lib/rack/streaming_proxy/response.rb

@@ -0,0 +1,79 @@
+require 'rack/streaming_proxy/errors'
+
+class Rack::StreamingProxy::Response
+  include Rack::Utils # For HeaderHash
+
+  attr_reader :status, :headers
+  attr_accessor :client_http_version
+
+  def initialize(piper)
+    @piper = piper
+    @client_http_version = '1.1'
+    receive
+  end
+
+  # This method is called by Rack itself, to iterate over the proxied contents.
+  def each
+    if @body_permitted
+      term = "\r\n"
+
+      while chunk = read_from_destination
+        break if chunk == :done
+        if @chunked
+          size = bytesize(chunk)
+          next if size == 0
+          if @client_http_version >= '1.1'
+            yield [size.to_s(16), term, chunk, term].join
+          else
+            yield chunk
+          end
+        else
+          yield chunk
+        end
+      end
+
+      finish
+
+      if @chunked && @client_http_version >= '1.1'
+        yield ['0', term, '', term].join
+      end
+    end
+  end
+
+private
+
+  def receive
+    # The first item received from the child will either be an HTTP status code or an Exception.
+    @status = read_from_destination
+
+    if @status.nil? # This should never happen
+      Rack::StreamingProxy::Proxy.log :error, "Parent received unexpected nil status!"
+      finish
+      raise Rack::StreamingProxy::UnknownError
+    elsif @status.kind_of? Exception
+      e = @status
+      Rack::StreamingProxy::Proxy.log :error, "Parent received an Exception from Child: #{e.class}: #{e.message}"
+      finish
+      raise e
+    end
+
+    Rack::StreamingProxy::Proxy.log :debug, "Parent received: Status = #{@status}."
+    @body_permitted = read_from_destination
+    Rack::StreamingProxy::Proxy.log :debug, "Parent received: Reponse has body? = #{@body_permitted}."
+    @headers = HeaderHash.new(read_from_destination)
+    @chunked = (@headers['Transfer-Encoding'] == 'chunked')
+    finish unless @body_permitted # If there is a body, finish will be called inside each.
+  end
+
+  # parent needs to wait for the child, or it results in the child process becoming defunct, resulting in zombie processes!
+  # This is very important. See: http://siliconisland.ca/2013/04/26/beware-of-the-zombie-process-apocalypse/
+  def finish
+    Rack::StreamingProxy::Proxy.log :info, "Parent process #{Process.pid} waiting for child process #{@piper.pid} to exit."
+    @piper.wait
+  end
+
+  def read_from_destination
+    @piper.gets
+  end
+
+end

data/lib/rack/streaming_proxy/session.rb

@@ -0,0 +1,119 @@
+require 'uri'
+require 'net/https'
+require 'servolux'
+require 'rack/streaming_proxy/errors'
+
+class Rack::StreamingProxy::Session
+
+  def initialize(request)
+    @request = request
+  end
+
+  # Returns a Rack::StreamingProxy::Response
+  def start
+    @piper = Servolux::Piper.new 'r', timeout: 30
+    @piper.child  { child }
+    @piper.parent { parent }
+  end
+
+private
+
+  def child
+    begin
+      Rack::StreamingProxy::Proxy.log :debug, "Child starting request to #{@request.uri}"
+      perform_request
+
+    rescue Exception => e
+      # Rescue all exceptions to help with development and debugging, as otherwise when exceptions
+      # occur the child process doesn't crash the parent process. Normally rescuing from Exception is a bad idea,
+      # but it's the only way to get a stacktrace here for all exceptions including SyntaxError etc,
+      # and we are simply passing it on so catastrophic exceptions will still be raised up the chain.
+      Rack::StreamingProxy::Proxy.log :debug, "Child process #{Process.pid} passing on #{e.class}: #{e.message}"
+      @piper.puts e # Pass on the exception to the parent.
+
+    ensure
+      Rack::StreamingProxy::Proxy.log :debug, "Child process #{Process.pid} closing connection."
+      @piper.close
+
+      Rack::StreamingProxy::Proxy.log :info, "Child process #{Process.pid} exiting."
+      exit!(0) # child needs to exit, always.
+    end
+  end
+
+  def parent
+    Rack::StreamingProxy::Proxy.log :info, "Parent process #{Process.pid} forked a child process #{@piper.pid}."
+
+    response = Rack::StreamingProxy::Response.new(@piper)
+    return response
+  end
+
+  def perform_request
+    http_session = Net::HTTP.new(@request.host, @request.port)
+    http_session.use_ssl = @request.use_ssl?
+
+    http_session.start do |session|
+      # Retry the request up to self.class.num_retries_on_5xx times if a 5xx is experienced.
+      # This is because some 500/503 errors resolve themselves quickly, might as well give it a chance.
+      # do...while loop as suggested by Matz: http://blade.nagaokaut.ac.jp/cgi-bin/scat.rb/ruby/ruby-core/6745
+      retries = 1
+      stop = false
+      loop do
+        session.request(@request.http_request) do |response|
+          # At this point the headers and status are available, but the body has not yet been read.
+          Rack::StreamingProxy::Proxy.log :debug, "Child got response: #{response.class.name}"
+
+          if response.class <= Net::HTTPServerError # Includes Net::HTTPServiceUnavailable, Net::HTTPInternalServerError
+            if retries <= Rack::StreamingProxy::Proxy.num_retries_on_5xx
+              Rack::StreamingProxy::Proxy.log :info, "Child got #{response.code}, retrying (Retry ##{retries})"
+              sleep 1
+              retries += 1
+              next
+            end
+          end
+          stop = true
+
+          Rack::StreamingProxy::Proxy.log :debug, "Child process #{Process.pid} returning Status = #{response.code}."
+
+          process_response(response)
+        end
+
+        break if stop
+      end
+    end
+  end
+
+  def process_response(response)
+
+    # Raise an exception if the raise_on_5xx config is set, and the response is a 5xx.
+    # Otherwise continue and put the error body in the pipe. (e.g. Apache error page, for example)
+    if response.class <= Net::HTTPServerError && Rack::StreamingProxy::Proxy.raise_on_5xx
+      raise Rack::StreamingProxy::HttpServerError.new "Got a #{response.class.name} (#{response.code}) response while proxying to #{@request.uri}"
+    end
+
+    # Put the response in the parent's pipe.
+    @piper.puts response.code
+    @piper.puts response.class.body_permitted?
+
+    # Could potentially use a one-liner here:
+    # @piper.puts Hash[response.to_hash.map { |key, value| [key, value.join(', ')] } ]
+    # But the following three lines seem to be more readable.
+    # Watch out: response.to_hash and response.each_header returns in different formats!
+    # to_hash requires the values to be joined with a comma.
+    headers = {}
+    response.each_header { |key, value| headers[key] = value }
+    log_headers :debug, 'Proxy Response Headers:', headers
+    @piper.puts headers
+
+    response.read_body { |chunk| @piper.puts chunk }
+    @piper.puts :done
+  end
+
+  def log_headers(level, title, headers)
+    Rack::StreamingProxy::Proxy.log level, "+-------------------------------------------------------------"
+    Rack::StreamingProxy::Proxy.log level, "| #{title}"
+    Rack::StreamingProxy::Proxy.log level, "+-------------------------------------------------------------"
+    headers.each { |key, value| Rack::StreamingProxy::Proxy.log level, "| #{key} = #{value.to_s}" }
+    Rack::StreamingProxy::Proxy.log level, "+-------------------------------------------------------------"
+  end
+
+end

data/lib/rack/streaming_proxy/version.rb

@@ -0,0 +1,5 @@
+module Rack
+  module StreamingProxy
+    VERSION = "2.0.1"
+  end
+end

data/rack-streaming-proxy.gemspec

@@ -0,0 +1,34 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'rack/streaming_proxy/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'bowtie-io-rack-streaming-proxy'
+  spec.version       = Rack::StreamingProxy::VERSION
+  spec.authors       = ['Fred Ngo', 'Nathan Witmer', 'Naotoshi Seo', 'James Kassemi']
+  spec.email         = ['fredngo@gmail.com', 'nwitmer@gmail.com', 'sonots@gmail.com', 'jkassemi@gmail.com']
+  spec.description   = %q{Streaming proxy for Rack, the rainbows to Rack::Proxy's unicorn.}
+  spec.summary       = %q{Streaming proxy for Rack, the rainbows to Rack::Proxy's unicorn.}
+  spec.homepage      = 'http://github.com/bowtie-io/rack-streaming-proxy'
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ['lib']
+
+  spec.add_runtime_dependency     'rack',     '>= 1.4'
+  spec.add_runtime_dependency     'servolux', '~> 0.10'
+
+  # test
+  spec.add_development_dependency 'bundler',  '>= 1.3'
+  spec.add_development_dependency 'rake',     '>= 10.0'
+  spec.add_development_dependency 'rspec'
+  spec.add_development_dependency 'rack-test'
+  spec.add_development_dependency 'unicorn'
+
+  # debug
+  spec.add_development_dependency 'pry'
+  spec.add_development_dependency 'pry-nav'
+end

data/spec/app.ru

@@ -0,0 +1,55 @@
+require "yaml"
+
+class Streamer
+  include Rack::Utils
+
+  def initialize(sleep=0.05)
+    @sleep = sleep
+    @strings = 5.times.collect {|n| "~~~~~ #{n} ~~~~~\n" }
+  end
+
+  def call(env)
+    req = Rack::Request.new(env)
+    headers = {"Content-Type" => "text/plain"}
+    headers["Transfer-Encoding"] = "chunked"
+    [200, headers, self.dup]
+  end
+
+  def each
+    term = "\r\n"
+    @strings.each do |chunk|
+      size = bytesize(chunk)
+      yield [size.to_s(16), term, chunk, term].join
+      sleep @sleep
+    end
+    yield ["0", term, "", term].join
+  end
+end
+
+# if no content-length is provided and the response isn't streamed,
+# make sure the headers get a content length.
+use Rack::ContentLength
+
+map "/" do
+  run lambda { |env| [200, {"Content-Type" => "text/plain"}, ["ALL GOOD"]] }
+end
+
+map "/stream" do
+  run Streamer.new
+end
+
+map "/slow_stream" do
+  run Streamer.new(0.5)
+end
+
+map "/env" do
+  run lambda { |env|
+    req = Rack::Request.new(env)
+    req.POST # modifies env inplace to include "rack.request.form_vars" key
+    [200, {"Content-Type" => "application/x-yaml"}, [env.to_yaml]] }
+end
+
+map "/boom" do
+  run lambda { |env| [500, {"Content-Type" => "text/plain"}, ["kaboom!"]] }
+end
+

data/spec/proxy.ru

@@ -0,0 +1,9 @@
+require File.expand_path(
+  File.join(File.dirname(__FILE__), %w[.. lib rack streaming_proxy]))
+
+ENV['RACK_ENV'] = 'none' # 'development' automatically use Rack::Lint and results in errors with unicorn
+# use Rack::CommonLogger
+use Rack::StreamingProxy::Proxy do |req|
+  "http://localhost:4321#{req.path}"
+end
+run lambda { |env| [200, {}, ["should never get here..."]]}

data/spec/spec_helper.rb

@@ -0,0 +1,9 @@
+require File.expand_path( File.join(File.dirname(__FILE__), %w[.. lib rack streaming_proxy]))
+
+require "rack/test"
+
+RSpec.configure do |config|
+  config.treat_symbols_as_metadata_keys_with_true_values = true
+  config.run_all_when_everything_filtered = true
+  config.filter_run :focus
+end

data/spec/streaming_proxy_spec.rb

@@ -0,0 +1,230 @@
+require 'yaml'
+require File.join(File.dirname(__FILE__), %w[spec_helper])
+
+APP_PORT = 4321 # hardcoded in proxy.ru as well!
+PROXY_PORT = 4322
+
+shared_examples "rack-streaming-proxy" do
+  it "passes through to the rest of the stack if block returns false" do
+    get "/not_proxied"
+    last_response.should be_ok
+    last_response.body.should == "not proxied"
+  end
+
+  it "proxies a request back to the app server" do
+    get "/", {}, rack_env
+    last_response.should be_ok
+    last_response.body.should == "ALL GOOD"
+    # Expect a Content-Length header field which the origin server sent is 
+    # not deleted by streaming-proxy.
+    last_response.headers["Content-Length"].should eq '8'
+  end
+
+  it "handles POST, PUT, and DELETE methods" do
+    post "/env", {}, rack_env
+    last_response.should be_ok
+    last_response.body.should =~ /REQUEST_METHOD: POST/
+    put "/env", {}, rack_env
+    last_response.should be_ok
+    last_response.body.should =~ /REQUEST_METHOD: PUT/
+    delete "/env", {}, rack_env
+    last_response.should be_ok
+    last_response.body.should =~ /REQUEST_METHOD: DELETE/
+  end
+
+  it "sets a X-Forwarded-For header" do
+    post "/env", {}, rack_env
+    last_response.should =~ /HTTP_X_FORWARDED_FOR: 127.0.0.1/
+  end
+
+  it "preserves the post body" do
+    post "/env", {"foo" => "bar"}, rack_env
+    last_response.body.should =~ /rack.request.form_vars: foo=bar/
+  end
+
+  it "raises a Rack::Proxy::StreamingProxy error when something goes wrong" do
+    Rack::StreamingProxy::Request.should_receive(:new).and_raise(RuntimeError.new("kaboom"))
+    lambda { get "/" }.should raise_error(RuntimeError, /kaboom/i)
+  end
+
+  it "does not raise a Rack::Proxy error if the app itself raises something" do
+    lambda { get "/not_proxied/boom" }.should raise_error(RuntimeError, /app error/)
+  end
+
+  it "preserves cookies" do
+    set_cookie "foo"
+    post "/env", {}, rack_env
+    YAML::load(last_response.body)["HTTP_COOKIE"].should == "foo"
+  end
+
+  it "preserves authentication info" do
+    basic_authorize "admin", "secret"
+    post "/env", {}, rack_env
+    YAML::load(last_response.body)["HTTP_AUTHORIZATION"].should == "Basic YWRtaW46c2VjcmV0"
+  end
+
+  it "preserves arbitrary headers" do
+    get "/env", {}, rack_env.merge("HTTP_X_FOOHEADER" => "Bar")
+    YAML::load(last_response.body)["HTTP_X_FOOHEADER"].should == "Bar"
+  end
+end
+
+describe Rack::StreamingProxy::Proxy do
+  include Rack::Test::Methods
+
+  def app
+    @app ||= Rack::Builder.new do
+      use Rack::Lint
+      use Rack::StreamingProxy::Proxy do |req|
+        # STDERR.puts "== incoming request env =="
+        # STDERR.puts req.env
+        # STDERR.puts "=^ incoming request env ^="
+        # STDERR.puts
+        unless req.path.start_with?("/not_proxied")
+          url = "http://localhost:#{APP_PORT}#{req.path}"
+          url << "?#{req.query_string}" unless req.query_string.empty?
+          # STDERR.puts "PROXYING to #{url}"
+          url
+        end
+      end
+      run lambda { |env|
+        raise "app error" if env["PATH_INFO"] =~ /boom/
+        [200, {"Content-Type" => "text/plain"}, ["not proxied"]]
+      }
+    end
+  end
+
+  before(:all) do
+    app_path = File.join(File.dirname(__FILE__), %w[app.ru])
+    @app_server = Servolux::Child.new(
+      # :command => "thin -R #{app_path} -p #{APP_PORT} start", # buffers!
+      :command => "rackup #{app_path} -p #{APP_PORT}",
+      :timeout => 30, # all specs should take <30 sec to run
+      :suspend => 0.25
+    )
+    puts "----- starting app server -----"
+    @app_server.start
+    sleep 2 # give it a sec
+    puts "----- started app server -----"
+  end
+
+  after(:all) do
+    puts "----- shutting down app server -----"
+    @app_server.stop
+    @app_server.wait
+    puts "----- app server is stopped -----"
+  end
+
+  context 'client requests with HTTP/1.0' do
+    let(:rack_env) { {'HTTP_VERSION' => 'HTTP/1.0'} }
+    it_behaves_like 'rack-streaming-proxy'
+    it "does not use chunked encoding when the app server send chunked body" do
+      get "/stream", {}, rack_env
+      last_response.should be_ok
+      # Expect a Transfer-Encoding header is deleted by rack-streaming-proxy
+      last_response.headers["Transfer-Encoding"].should be_nil
+      # I expected a Content-Length header which the origin server sent was deleted,
+      # But the following test failed against my expectation. The reason is 
+      # that a Content-Length header was added in creating Rack::MockResponse 
+      # instance. So I gave up writing this test right now.
+      #
+      # last_response.headers["Content-Length"].should be_nil
+      #
+      last_response.body.should == <<-EOS
+~~~~~ 0 ~~~~~
+~~~~~ 1 ~~~~~
+~~~~~ 2 ~~~~~
+~~~~~ 3 ~~~~~
+~~~~~ 4 ~~~~~
+      EOS
+    end
+  end
+
+  context 'client requests with HTTP/1.1' do
+    let(:rack_env) { {'HTTP_VERSION' => 'HTTP/1.1'} }
+    it_behaves_like 'rack-streaming-proxy'
+    it "uses chunked encoding when the app server send chunked body" do
+      get "/stream", {}, rack_env
+      last_response.should be_ok
+      last_response.headers["Transfer-Encoding"].should == 'chunked'
+      last_response.headers["Content-Length"].should be_nil
+      last_response.body.should =~ /^e\r\n~~~~~ 0 ~~~~~\n\r\n/
+    end
+  end
+
+end
+
+describe Rack::StreamingProxy::Proxy do
+  include Rack::Test::Methods
+
+  attr_reader :app
+
+  before(:all) do
+    app_path = File.join(File.dirname(__FILE__), %w[app.ru])
+    @app_server = Servolux::Child.new(
+      # :command => "thin -R #{app_path} -p #{APP_PORT} start", # buffers!
+      # :command => "rackup #{app_path} -p #{APP_PORT}", # webrick adds content-length, it should be wrong
+      :command => "unicorn #{app_path} -p #{APP_PORT} -E none",
+      :timeout => 30, # all specs should take <30 sec to run
+      :suspend => 0.25
+    )
+    puts "----- starting app server -----"
+    @app_server.start
+    sleep 2 # give it a sec
+    puts "----- started app server -----"
+  end
+
+  after(:all) do
+    puts "----- shutting down app server -----"
+    @app_server.stop
+    @app_server.wait
+    puts "----- app server is stopped -----"
+  end
+
+  def with_proxy_server
+    proxy_path = File.join(File.dirname(__FILE__), %w[proxy.ru])
+    @proxy_server = Servolux::Child.new(
+      :command => "unicorn #{proxy_path} -p #{PROXY_PORT} -E none",
+      :timeout => 10,
+      :suspend => 0.25
+    )
+    puts "----- starting proxy server -----"
+    @proxy_server.start
+    sleep 2
+    puts "----- started proxy server -----"
+    yield
+  ensure
+    puts "----- shutting down proxy server -----"
+    @proxy_server.stop
+    @proxy_server.wait
+    puts "----- proxy server is stopped -----"
+  end
+
+  # this is the most critical spec: it makes sure things are actually streamed, not buffered
+  # MEMO: only unicorn worked. webrick, thin, and puma did not progressively stream
+  it "streams data from the app server to the client" do
+    @app = Rack::Builder.new do
+      use Rack::Lint
+      run lambda { |env|
+        body = []
+        Net::HTTP.start("localhost", PROXY_PORT) do |http|
+          http.request_get("/slow_stream") do |response|
+            response.read_body do |chunk|
+              body << "#{Time.now.to_i}\n"
+            end
+          end
+        end
+        [200, {"Content-Type" => "text/plain"}, body]
+      }
+    end
+
+    with_proxy_server do
+      get "/"
+      last_response.should be_ok
+      times = last_response.body.split("\n").map {|l| l.to_i}
+      unless (times.last - times.first) >= 2
+        fail "expected receive time of first chunk to be at least two seconds before the last chunk, but the times were: #{times.join(', ')}"
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,201 @@
+--- !ruby/object:Gem::Specification
+name: bowtie-io-rack-streaming-proxy
+version: !ruby/object:Gem::Version
+  version: 2.0.1
+platform: ruby
+authors:
+- Fred Ngo
+- Nathan Witmer
+- Naotoshi Seo
+- James Kassemi
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-01-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.4'
+- !ruby/object:Gem::Dependency
+  name: servolux
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.10'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rack-test
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: unicorn
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry-nav
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Streaming proxy for Rack, the rainbows to Rack::Proxy's unicorn.
+email:
+- fredngo@gmail.com
+- nwitmer@gmail.com
+- sonots@gmail.com
+- jkassemi@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- README.txt
+- Rakefile
+- dev/client.rb
+- dev/proxy.ru
+- dev/streamer.ru
+- lib/rack/streaming_proxy.rb
+- lib/rack/streaming_proxy/errors.rb
+- lib/rack/streaming_proxy/proxy.rb
+- lib/rack/streaming_proxy/request.rb
+- lib/rack/streaming_proxy/response.rb
+- lib/rack/streaming_proxy/session.rb
+- lib/rack/streaming_proxy/version.rb
+- rack-streaming-proxy.gemspec
+- spec/app.ru
+- spec/proxy.ru
+- spec/spec_helper.rb
+- spec/streaming_proxy_spec.rb
+homepage: http://github.com/bowtie-io/rack-streaming-proxy
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Streaming proxy for Rack, the rainbows to Rack::Proxy's unicorn.
+test_files:
+- spec/app.ru
+- spec/proxy.ru
+- spec/spec_helper.rb
+- spec/streaming_proxy_spec.rb