rack-streaming-proxy2 1.0.3

data/History.txt

@@ -0,0 +1,4 @@
+== 1.0.0 / 2009-11-14
+
+* 1 major enhancement
+  * Birthday!

data/README.txt

@@ -0,0 +1,78 @@
+rack-streaming-proxy
+    by Nathan Witmer <nwitmer@gmail.com>
+    http://github.com/aniero/rack-streaming-proxy
+
+== DESCRIPTION:
+
+Streaming proxy for Rack, the rainbows to Rack::Proxy's unicorn.
+
+== FEATURES/PROBLEMS:
+
+Provides a transparent streaming proxy to be used as rack middleware.
+
+* Streams the response from the downstream server to minimize memory usage
+* Handles chunked encoding if used
+* Proxies GET/PUT/POST/DELETE, XHR, and cookies
+
+Use this when you need to have the response streamed back to the client,
+for example when handling large file requests that could be proxied
+directly but need to be authenticated against the rest of your middleware
+stack.
+
+Note that this will not work well with EventMachine. EM buffers the entire
+rack response before sending it to the client. When testing, try
+mongrel (via rackup) or passenger, rather than the EM-based thin. See
+http://groups.google.com/group/thin-ruby/browse_thread/thread/4762f8f851b965f6
+for more discussion.
+
+I've included a simple streamer app for testing and development.
+
+Thanks to:
+
+* Tom Lea (cwninja) for Rack::Proxy (http://gist.github.com/207938)
+* Tim Pease for bones, servolux, &c
+
+== SYNOPSIS:
+
+  require "rack/streaming_proxy"
+
+  use Rack::StreamingProxy do |request|
+    # inside the request block, return the full URI to redirect the request to,
+    # or nil/false if the request should continue on down the middleware stack.
+    if request.path.start_with?("/proxy")
+      "http://another_server#{request.path}"
+    end
+  end
+
+== REQUIREMENTS:
+
+* servolux (gem install servolux)
+
+== INSTALL:
+
+* sudo gem install rack-streaming-proxy --source http://gemcutter.org
+
+== LICENSE:
+
+(The MIT License)
+
+Copyright (c) 2009 Nathan Witmer
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,28 @@
+begin
+  require 'bones'
+rescue LoadError
+  abort '### Please install the "bones" gem ###'
+end
+
+ensure_in_path 'lib'
+require 'rack/streaming_proxy'
+
+task :default => 'spec:specdoc'
+task 'gem:release' => 'spec:specdoc'
+
+Bones {
+  name  'rack-streaming-proxy'
+  authors  'Nathan Witmer'
+  email  'nwitmer@gmail.com'
+  url  'http://github.com/aniero/rack-streaming-proxy'
+  version  Rack::StreamingProxy::VERSION
+  ignore_file  '.gitignore'
+  depend_on "rack", :version => ">= 1.0"
+  depend_on "servolux", :version => "~> 0.8.1"
+  depend_on "rack-test", :version => "~> 0.5.1", :development => true
+  spec {
+    opts ["--colour",  "--loadby mtime", "--reverse", "--diff unified"]
+  }
+  enable_sudo
+}
+

data/dev/proxy.ru

@@ -0,0 +1,13 @@
+require File.expand_path(
+    File.join(File.dirname(__FILE__), %w[.. lib rack streaming_proxy]))
+
+use Rack::Reloader, 1
+# use Rack::CommonLogger # rackup already has commonlogger loaded
+use Rack::Lint
+use Rack::StreamingProxy do |req|
+  url = "http://localhost:4000#{req.path}"
+  url << "?#{req.query_string}" unless req.query_string.empty?
+  url
+end
+
+run lambda { |env| [200, {"Content-Type" => "text/plain"}, ""] }

data/dev/streamer.ru

@@ -0,0 +1,54 @@
+class Streamer
+  include Rack::Utils
+
+  def call(env)
+    req = Rack::Request.new(env)
+    headers = {"Content-Type" => "text/plain"}
+
+    @chunked = req.path.start_with?("/chunked")
+
+    if count = req.path.match(/(\d+)$/)
+      count = count[0].to_i
+    else
+      count = 100
+    end
+    @strings = count.times.collect {|n| "~~~~~ #{n} ~~~~~\n" }
+
+    if chunked?
+      headers["Transfer-Encoding"] = "chunked"
+    else
+      headers["Content-Length"] = @strings.inject(0) {|sum, s| sum += bytesize(s)}.to_s
+    end
+
+    [200, headers, self.dup]
+  end
+
+  def each
+    term = "\r\n"
+    @strings.each do |chunk|
+      if chunked?
+        size = bytesize(chunk)
+        yield [size.to_s(16), term, chunk, term].join
+      else
+        yield chunk
+      end
+      sleep 0.05
+    end
+    yield ["0", term, "", term].join if chunked?
+  end
+
+  protected
+
+  def chunked?
+    @chunked
+  end
+end
+
+# use Rack::CommonLogger # rackup already has commonlogger loaded
+use Rack::Lint
+
+# GET /
+# GET /10
+# GET /chunked
+# GET /chunked/10
+run Streamer.new

data/lib/rack/streaming_proxy.rb

@@ -0,0 +1,107 @@
+module Rack
+  class StreamingProxy
+
+    class Error < StandardError; end
+
+    # :stopdoc:
+    VERSION = '1.0.3'
+    LIBPATH = ::File.expand_path(::File.dirname(__FILE__)) + ::File::SEPARATOR
+    PATH = ::File.expand_path(::File.join(::File.dirname(__FILE__), "..", "..")) + ::File::SEPARATOR
+    # :startdoc:
+
+    # Returns the version string for the library.
+    #
+    def self.version
+      VERSION
+    end
+
+    # Returns the library path for the module. If any arguments are given,
+    # they will be joined to the end of the libray path using
+    # <tt>File.join</tt>.
+    #
+    def self.libpath( *args )
+      args.empty? ? LIBPATH : ::File.join(LIBPATH, args.flatten)
+    end
+
+    # Returns the lpath for the module. If any arguments are given,
+    # they will be joined to the end of the path using
+    # <tt>File.join</tt>.
+    #
+    def self.path( *args )
+      args.empty? ? PATH : ::File.join(PATH, args.flatten)
+    end
+
+    # Utility method used to require all files ending in .rb that lie in the
+    # directory below this file that has the same name as the filename passed
+    # in. Optionally, a specific _directory_ name can be passed in such that
+    # the _filename_ does not have to be equivalent to the directory.
+    #
+    def self.require_all_libs_relative_to( fname, dir = nil )
+      dir ||= ::File.basename(fname, '.*')
+      search_me = ::File.expand_path(
+        ::File.join(::File.dirname(fname), dir, '**', '*.rb'))
+
+        Dir.glob(search_me).sort.each {|rb| require rb}
+    end
+
+    # The block provided to the initializer is given a Rack::Request
+    # and should return:
+    #
+    #   * nil/false to skip the proxy and continue down the stack
+    #   * a complete uri (with query string if applicable) to proxy to
+    #
+    # E.g.
+    #
+    #   use Rack::StreamingProxy do |req|
+    #     if req.path.start_with?("/search")
+    #       "http://some_other_service/search?#{req.query}"
+    #     end
+    #   end
+    #
+    # Most headers, request body, and HTTP method are preserved.
+    #
+    def initialize(app, &block)
+      @request_uri = block
+      @app = app
+    end
+
+    def call(env)
+      req = Rack::Request.new(env)
+      unless uri = request_uri.call(req)
+        code, headers, body = app.call(env)
+        unless headers['X-Accel-Redirect']
+          return [code, headers, body]
+        else
+          proxy_env = env.merge("PATH_INFO" => headers['X-Accel-Redirect'])
+          unless uri = request_uri.call(Rack::Request.new(proxy_env))
+            raise "Could not proxy #{headers['X-Accel-Redirect']}: Path does not map to any uri"
+          end
+        end
+      end
+      begin # only want to catch proxy errors, not app errors
+        proxy = ProxyRequest.new(req, uri)
+        [proxy.status, proxy.headers, proxy]
+      rescue => e
+        msg = "Proxy error when proxying to #{uri}: #{e.class}: #{e.message}"
+        env["rack.errors"].puts msg
+        env["rack.errors"].puts e.backtrace.map { |l| "\t" + l }
+        env["rack.errors"].flush
+        raise Error, msg
+      end
+    end
+
+    protected
+
+    attr_reader :request_uri, :app
+
+  end
+
+end
+
+require "rack"
+require "servolux"
+require "net/https"
+require "uri"
+
+Rack::StreamingProxy.require_all_libs_relative_to(__FILE__)
+

data/lib/rack/streaming_proxy/proxy_request.rb

@@ -0,0 +1,98 @@
+class Rack::StreamingProxy
+  class ProxyRequest
+    include Rack::Utils
+
+    attr_reader :status, :headers
+
+    def initialize(request, uri)
+      uri = URI.parse(uri)
+
+      method = request.request_method.downcase
+      method[0..0] = method[0..0].upcase
+      
+      proxy_request = Net::HTTP.const_get(method).new("#{uri.path}#{"?" if uri.query}#{uri.query}")
+
+      if proxy_request.request_body_permitted? and request.body
+        proxy_request.body_stream = request.body
+        proxy_request.content_length = request.content_length if request.content_length
+        proxy_request.content_type = request.content_type if request.content_type
+      end
+
+      %w(Accept Accept-Encoding Accept-Charset
+        X-Requested-With Referer User-Agent Cookie
+        Authorization
+        ).each do |header|
+        key = "HTTP_#{header.upcase.gsub('-', '_')}"
+        proxy_request[header] = request.env[key] if request.env[key]
+      end
+      proxy_request["X-Forwarded-For"] =
+        (request.env["X-Forwarded-For"].to_s.split(/, +/) + [request.env["REMOTE_ADDR"]]).join(", ")
+
+      @piper = Servolux::Piper.new 'r', :timeout => 30
+
+      @piper.child do
+        http_req = Net::HTTP.new(uri.host, uri.port)
+        http_req.use_ssl = uri.is_a?(URI::HTTPS)
+        http_req.start do |http|
+          http.request(proxy_request) do |response|
+            # at this point the headers and status are available, but the body
+            # has not yet been read. start reading it and putting it in the parent's pipe.
+            response_headers = {}
+            response.each_header {|k,v| response_headers[k] = v}
+            @piper.puts response.code.to_i
+            @piper.puts response_headers
+
+            response.read_body do |chunk|
+              @piper.puts chunk
+            end
+            @piper.puts :done
+          end
+        end
+      end
+
+      @piper.parent do
+        # wait for the status and headers to come back from the child
+        @status = read_from_child
+        @headers = HeaderHash.new(read_from_child)
+      end
+    rescue => e
+      if @piper
+        @piper.parent { raise }
+        @piper.child { @piper.puts e }
+      else
+        raise
+      end
+    ensure
+      # child needs to exit, always.
+      @piper.child { exit!(0) } if @piper
+    end
+
+    def each
+      chunked = @headers["Transfer-Encoding"] == "chunked"
+      term = "\r\n"
+
+      while chunk = read_from_child
+        break if chunk == :done
+        if chunked
+          size = bytesize(chunk)
+          next if size == 0
+          yield [size.to_s(16), term, chunk, term].join
+        else
+          yield chunk
+        end
+      end
+
+      yield ["0", term, "", term].join if chunked
+    end
+
+
+    protected
+
+    def read_from_child
+      val = @piper.gets
+      raise val if val.kind_of?(Exception)
+      val
+    end
+
+  end
+end

data/spec/app.ru

@@ -0,0 +1,55 @@
+require "yaml"
+
+class Streamer
+  include Rack::Utils
+
+  def initialize(sleep=0.05)
+    @sleep = sleep
+    @strings = 5.times.collect {|n| "~~~~~ #{n} ~~~~~\n" }
+  end
+
+  def call(env)
+    req = Rack::Request.new(env)
+    headers = {"Content-Type" => "text/plain"}
+    headers["Transfer-Encoding"] = "chunked"
+    [200, headers, self.dup]
+  end
+
+  def each
+    term = "\r\n"
+    @strings.each do |chunk|
+      size = bytesize(chunk)
+      yield [size.to_s(16), term, chunk, term].join
+      sleep @sleep
+    end
+    yield ["0", term, "", term].join
+  end
+end
+
+# if no content-length is provided and the response isn't streamed,
+# make sure the headers get a content length.
+use Rack::ContentLength
+
+map "/" do
+  run lambda { |env| [200, {"Content-Type" => "text/plain"}, "ALL GOOD"] }
+end
+
+map "/stream" do
+  run Streamer.new
+end
+
+map "/slow_stream" do
+  run Streamer.new(0.5)
+end
+
+map "/env" do
+  run lambda { |env|
+    req = Rack::Request.new(env)
+    req.POST # modifies env inplace to include "rack.request.form_vars" key
+    [200, {"Content-Type" => "application/x-yaml"}, [env.to_yaml]] }
+end
+
+map "/boom" do
+  run lambda { |env| [500, {"Content-Type" => "text/plain"}, "kaboom!"] }
+end
+

data/spec/proxy.ru

@@ -0,0 +1,9 @@
+require File.expand_path(
+  File.join(File.dirname(__FILE__), %w[.. lib rack streaming_proxy]))
+
+use Rack::Lint
+# use Rack::CommonLogger
+use Rack::StreamingProxy do |req|
+  "http://localhost:4321#{req.path}"
+end
+run lambda { |env| [200, {}, "should never get here..."]}

data/spec/spec_helper.rb

@@ -0,0 +1,8 @@
+require File.expand_path(
+  File.join(File.dirname(__FILE__), %w[.. lib rack streaming_proxy]))
+
+require "rack/test"
+
+Spec::Runner.configure do |config|
+end
+

data/spec/streaming_proxy_spec.rb

@@ -0,0 +1,159 @@
+require File.join(File.dirname(__FILE__), %w[spec_helper])
+
+describe Rack::StreamingProxy do
+  include Rack::Test::Methods
+
+  APP_PORT = 4321 # hardcoded in proxy.ru as well!
+  PROXY_PORT = 4322
+
+  def app
+    @app ||= Rack::Builder.new do
+      use Rack::Lint
+      use Rack::StreamingProxy do |req|
+        unless req.path.start_with?("/not_proxied")
+          url = "http://localhost:#{APP_PORT}#{req.path}"
+          url << "?#{req.query_string}" unless req.query_string.empty?
+          # STDERR.puts "PROXYING to #{url}"
+          url
+        end
+      end
+      run lambda { |env|
+        raise "app error" if env["PATH_INFO"] =~ /boom/
+        [200, {"Content-Type" => "text/plain"}, "not proxied"]
+      }
+    end
+  end
+
+  before(:all) do
+    app_path = Rack::StreamingProxy.path("spec", "app.ru")
+    @app_server = Servolux::Child.new(
+      # :command => "thin -R #{app_path} -p #{APP_PORT} start", # buffers!
+      :command => "rackup #{app_path} -p #{APP_PORT}",
+      :timeout => 30, # all specs should take <30 sec to run
+      :suspend => 0.25
+    )
+    puts "----- starting app server -----"
+    @app_server.start
+    sleep 2 # give it a sec
+    puts "----- started app server -----"
+  end
+
+  after(:all) do
+    puts "----- shutting down app server -----"
+    @app_server.stop
+    @app_server.wait
+    puts "----- app server is stopped -----"
+  end
+
+  def with_proxy_server
+    proxy_path = Rack::StreamingProxy.path("spec", "proxy.ru")
+    @proxy_server = Servolux::Child.new(
+      :command => "rackup #{proxy_path} -p #{PROXY_PORT}",
+      :timeout => 10,
+      :suspend => 0.25
+    )
+    puts "----- starting proxy server -----"
+    @proxy_server.start
+    sleep 2
+    puts "----- started proxy server -----"
+    yield
+  ensure
+    puts "----- shutting down proxy server -----"
+    @proxy_server.stop
+    @proxy_server.wait
+    puts "----- proxy server is stopped -----"
+  end
+
+  it "passes through to the rest of the stack if block returns false" do
+    get "/not_proxied"
+    last_response.should be_ok
+    last_response.body.should == "not proxied"
+  end
+
+  it "proxies a request back to the app server" do
+    get "/"
+    last_response.should be_ok
+    last_response.body.should == "ALL GOOD"
+  end
+
+  it "uses chunked encoding when the app server send data that way" do
+    get "/stream"
+    last_response.should be_ok
+    last_response.headers["Transfer-Encoding"].should == "chunked"
+    last_response.body.should =~ /^e\r\n~~~~~ 0 ~~~~~\n\r\n/
+  end
+
+  # this is the most critical spec: it makes sure things are actually streamed, not buffered
+  it "streams data from the app server to the client" do
+    @app = Rack::Builder.new do
+      use Rack::Lint
+      run lambda { |env|
+        body = []
+        Net::HTTP.start("localhost", PROXY_PORT) do |http|
+          http.request_get("/slow_stream") do |response|
+            response.read_body do |chunk|
+              body << "#{Time.now.to_i}\n"
+            end
+          end
+        end
+        [200, {"Content-Type" => "text/plain"}, body]
+      }
+    end
+
+    with_proxy_server do
+      get "/"
+      last_response.should be_ok
+      times = last_response.body.split("\n").map {|l| l.to_i}
+      unless (times.last - times.first) >= 2
+        violated "expected receive time of first chunk to be at least " +
+          "two seconds before the last chunk"
+      end
+    end
+
+  end
+
+  it "handles POST, PUT, and DELETE methods" do
+    post "/env"
+    last_response.should be_ok
+    last_response.body.should =~ /REQUEST_METHOD: POST/
+    put "/env"
+    last_response.should be_ok
+    last_response.body.should =~ /REQUEST_METHOD: PUT/
+    delete "/env"
+    last_response.should be_ok
+    last_response.body.should =~ /REQUEST_METHOD: DELETE/
+  end
+
+  it "sets a X-Forwarded-For header" do
+    post "/env"
+    last_response.should =~ /HTTP_X_FORWARDED_FOR: 127.0.0.1/
+  end
+
+  it "preserves the post body" do
+    post "/env", "foo" => "bar"
+    last_response.body.should =~ /rack.request.form_vars: foo=bar/
+  end
+
+  it "raises a Rack::Proxy::StreamingProxy error when something goes wrong" do
+    Rack::StreamingProxy::ProxyRequest.should_receive(:new).and_raise(RuntimeError.new("kaboom"))
+    lambda { get "/" }.should raise_error(Rack::StreamingProxy::Error, /proxy error.*kaboom/i)
+  end
+
+  it "does not raise a Rack::Proxy error if the app itself raises something" do
+    lambda { get "/not_proxied/boom" }.should raise_error(RuntimeError, /app error/)
+  end
+
+  it "preserves cookies" do
+    set_cookie "foo"
+    post "/env"
+    YAML.load(last_response.body)["HTTP_COOKIE"].should == "foo"
+  end
+
+  it "preserves authentication info" do
+    basic_authorize "admin", "secret"
+    post "/env"
+    YAML.load(last_response.body)["HTTP_AUTHORIZATION"].should == "Basic YWRtaW46c2VjcmV0\n"
+  end
+
+end
+

metadata

@@ -0,0 +1,141 @@
+--- !ruby/object:Gem::Specification 
+name: rack-streaming-proxy2
+version: !ruby/object:Gem::Version 
+  hash: 17
+  prerelease: false
+  segments: 
+  - 1
+  - 0
+  - 3
+  version: 1.0.3
+platform: ruby
+authors: 
+- Nathan Witmer, gr2m
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-11-15 00:00:00 -08:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rack
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 15
+        segments: 
+        - 1
+        - 0
+        version: "1.0"
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: servolux
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 61
+        segments: 
+        - 0
+        - 8
+        - 1
+        version: 0.8.1
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: rack-test
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 9
+        segments: 
+        - 0
+        - 5
+        - 1
+        version: 0.5.1
+  type: :development
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: bones
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 17
+        segments: 
+        - 3
+        - 5
+        - 1
+        version: 3.5.1
+  type: :development
+  version_requirements: *id004
+description: Streaming proxy for Rack, the rainbows to Rack::Proxy's unicorn. With SSL Support
+email: nwitmer@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- History.txt
+- README.txt
+files: 
+- History.txt
+- README.txt
+- Rakefile
+- dev/proxy.ru
+- dev/streamer.ru
+- lib/rack/streaming_proxy.rb
+- lib/rack/streaming_proxy/proxy_request.rb
+- spec/app.ru
+- spec/proxy.ru
+- spec/spec_helper.rb
+- spec/streaming_proxy_spec.rb
+has_rdoc: true
+homepage: https://github.com/gr2m/rack-streaming-proxy
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --main
+- README.txt
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: rack-streaming-proxy2
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: Streaming proxy for Rack, the rainbows to Rack::Proxy's unicorn With SSL Support
+test_files: []
+