em-proxy 0.1.4 → 0.1.5

data/.gitignore

@@ -0,0 +1 @@
+.bundle

data/Gemfile

@@ -0,0 +1,3 @@
+source "http://rubygems.org"
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,36 @@
+PATH
+  remote: .
+  specs:
+    em-proxy (0.1.5)
+      eventmachine
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    addressable (2.2.2)
+    ansi (1.2.2)
+    diff-lcs (1.1.2)
+    em-http-request (0.3.0)
+      addressable (>= 2.0.0)
+      escape_utils
+      eventmachine (>= 0.12.9)
+    escape_utils (0.1.9)
+    eventmachine (0.12.11)
+    rspec (2.4.0)
+      rspec-core (~> 2.4.0)
+      rspec-expectations (~> 2.4.0)
+      rspec-mocks (~> 2.4.0)
+    rspec-core (2.4.0)
+    rspec-expectations (2.4.0)
+      diff-lcs (~> 1.1.2)
+    rspec-mocks (2.4.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  ansi
+  em-http-request
+  em-proxy!
+  eventmachine
+  rspec

data/README.rdoc

@@ -7,14 +7,17 @@ EventMachine Proxy DSL for writing high-performance transparent / intercepting p
 
 == Getting started
 
-  # install & configure gemcutter repos
-  gem update --system
-  gem install gemcutter
-  gem tumble
+  $> gem install em-proxy
+  $> em-proxy
+  Usage: em-proxy [options]
+      -l, --listen [PORT]              Port to listen on
+      -d, --duplex [host:port, ...]    List of backends to duplex data to
+      -r, --relay [hostname:port]      Relay endpoint: hostname:port
+      -v, --verbose                    Run in debug mode
 
-  gem install em-proxy
+  $> em-proxy -l 8080 -r localhost:8081 -d localhost:8082,localhost:8083 -v
 
-  irb:0> require 'em-proxy'
+The above will start em-proxy on port 8080, relay and respond with data from port 8081, and also (optional) duplicate all traffic to ports 8082 and 8083 (and discard their responses).
 
 == Simple port forwarding proxy
 

data/Rakefile

@@ -1,19 +1,4 @@
-require 'rake'
+require 'bundler'
 
-begin
-  require 'jeweler'
-  Jeweler::Tasks.new do |gemspec|
-    gemspec.name = "em-proxy"
-    gemspec.summary = "EventMachine Proxy DSL"
-    gemspec.description = gemspec.summary
-    gemspec.email = "ilya@igvita.com"
-    gemspec.homepage = "http://github.com/igrigorik/em-proxy"
-    gemspec.authors = ["Ilya Grigorik"]
-    gemspec.add_dependency('eventmachine', '>= 0.12.9')
-    gemspec.rubyforge_project = "em-proxy"
-  end
-
-  Jeweler::GemcutterTasks.new
-rescue LoadError
-  puts "Jeweler not available. Install it with: sudo gem install technicalpickles-jeweler -s http://gems.github.com"
-end
+Bundler.setup
+Bundler::GemHelper.install_tasks

data/bin/em-proxy

@@ -0,0 +1,48 @@
+#!/usr/bin/env ruby
+
+lib = File.expand_path(File.dirname(__FILE__) + '/../lib')
+$LOAD_PATH.unshift(lib) if File.directory?(lib) && !$LOAD_PATH.include?(lib)
+
+require 'em-proxy'
+require 'optparse'
+
+ARGV << '--help' if ARGV.empty?
+
+options = {}
+OptionParser.new do |opts|
+  opts.banner = "Usage: em-proxy [options]"
+
+  opts.on("-l", "--listen [PORT]", Integer, "Port to listen on") do |v|
+    options[:listen] = v
+  end
+
+  opts.on("-d", "--duplex [host:port, ...]", Array, "List of backends to duplex data to") do |v|
+    options[:duplex] = v
+  end
+
+  opts.on("-r", "--relay [hostname:port]", String, "Relay endpoint: hostname:port") do |v|
+    options[:relay] = v.split(":")
+  end
+
+  opts.on("-v", "--verbose", "Run in debug mode") do |v|
+    options[:verbose] = v
+  end
+end.parse!
+
+
+Proxy.start(:host => "0.0.0.0", :port => options[:listen] , :debug => options[:verbose]) do |conn|
+  conn.server :relay, :host => options[:relay].first, :port => options[:relay].last.to_i
+
+  options[:duplex].each_with_index do |backend,i|
+    hostname, port = backend.split(":")
+    conn.server "backend_#{i}".intern, :host => hostname, :port => port.to_i
+  end if options[:duplex]
+
+  conn.on_data do |data|
+    data
+  end
+
+  conn.on_response do |server, resp|
+    resp if server == :relay
+  end
+end

data/em-proxy.gemspec

@@ -1,76 +1,25 @@
-# Generated by jeweler
-# DO NOT EDIT THIS FILE DIRECTLY
-# Instead, edit Jeweler::Tasks in Rakefile, and run the gemspec command
 # -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |s|
-  s.name = %q{em-proxy}
-  s.version = "0.1.4"
+  s.name        = "em-proxy"
+  s.version     = "0.1.5"
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Ilya Grigorik"]
+  s.email       = ["ilya@igvita.com"]
+  s.homepage    = "http://github.com/igrigorik/em-proxy"
+  s.summary     = %q{EventMachine Proxy DSL}
+  s.description = s.summary
 
-  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
-  s.authors = ["Ilya Grigorik"]
-  s.date = %q{2010-10-03}
-  s.description = %q{EventMachine Proxy DSL}
-  s.email = %q{ilya@igvita.com}
-  s.extra_rdoc_files = [
-    "README.rdoc"
-  ]
-  s.files = [
-    "README.rdoc",
-     "Rakefile",
-     "VERSION",
-     "em-proxy.gemspec",
-     "examples/appserver.rb",
-     "examples/beanstalkd_interceptor.rb",
-     "examples/duplex.rb",
-     "examples/line_interceptor.rb",
-     "examples/port_forward.rb",
-     "examples/relay_port_forward.rb",
-     "examples/schemaless-mysql/mysql_interceptor.rb",
-     "examples/selective_forward.rb",
-     "examples/simple.rb",
-     "examples/smtp_spam_filter.rb",
-     "examples/smtp_whitelist.rb",
-     "lib/em-proxy.rb",
-     "lib/em-proxy/backend.rb",
-     "lib/em-proxy/connection.rb",
-     "lib/em-proxy/proxy.rb",
-     "spec/helper.rb",
-     "spec/proxy_spec.rb"
-  ]
-  s.homepage = %q{http://github.com/igrigorik/em-proxy}
-  s.rdoc_options = ["--charset=UTF-8"]
-  s.require_paths = ["lib"]
-  s.rubyforge_project = %q{em-proxy}
-  s.rubygems_version = %q{1.3.7}
-  s.summary = %q{EventMachine Proxy DSL}
-  s.test_files = [
-    "spec/helper.rb",
-     "spec/proxy_spec.rb",
-     "examples/appserver.rb",
-     "examples/beanstalkd_interceptor.rb",
-     "examples/duplex.rb",
-     "examples/line_interceptor.rb",
-     "examples/port_forward.rb",
-     "examples/relay_port_forward.rb",
-     "examples/schemaless-mysql/mysql_interceptor.rb",
-     "examples/selective_forward.rb",
-     "examples/simple.rb",
-     "examples/smtp_spam_filter.rb",
-     "examples/smtp_whitelist.rb"
-  ]
-
-  if s.respond_to? :specification_version then
-    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
-    s.specification_version = 3
+  s.rubyforge_project = "em-proxy"
 
-    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
-      s.add_runtime_dependency(%q<eventmachine>, [">= 0.12.9"])
-    else
-      s.add_dependency(%q<eventmachine>, [">= 0.12.9"])
-    end
-  else
-    s.add_dependency(%q<eventmachine>, [">= 0.12.9"])
-  end
-end
+  s.add_dependency "eventmachine"
+  s.add_development_dependency "rspec"
+  s.add_development_dependency "em-http-request"
+  s.add_development_dependency "ansi"
 
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+end

data/examples/balancing-client.rb

@@ -0,0 +1,20 @@
+# A simple HTTP client, which sends multiple requests to the proxy server
+
+require 'net/http'
+
+proxy = Net::HTTP::Proxy('0.0.0.0', '9999')
+
+count = ENV['COUNT'] || 5
+
+threads = []
+count.to_i.times do |i|
+  threads << Thread.new do
+    proxy.start('www.example.com') do |http|
+      puts http.get('/').body
+      puts "^^^ #{i+1} " + '-'*80 + "\n\n"
+    end
+    sleep 0.1
+  end
+end
+
+threads.each { |t| t.join }

data/examples/balancing.rb

@@ -0,0 +1,195 @@
+require 'lib/em-proxy'
+require 'ansi/code'
+require 'uri'
+
+# = Balancing Proxy
+#
+# A simple example of a balancing, reverse/forward proxy such as Nginx or HAProxy.
+#
+# Given a list of backends, it's able to distribute requests to backends
+# via different strategies (_random_, _roundrobin_, _balanced_), see <tt>Backend.select</tt>.
+#
+# This example is provided for didactic purposes. Nevertheless, based on some preliminary benchmarks
+# and live tests, it performs well in production usage.
+#
+# You can customize the behaviour of the proxy by changing the <tt>BalancingProxy::Callbacks</tt>
+# callbacks. To give you some ideas:
+#
+# * Store statistics for the proxy load in Redis (eg.: <tt>$redis.incr "proxy>backends>#{backend}>total"</tt>)
+# * Use Redis' _SortedSet_ instead of updating the <tt>Backend.list</tt> hash to allow for polling from external process
+# * Use <b>em-websocket</b>[https://github.com/igrigorik/em-websocket] to build a web frontend for monitoring
+#
+module BalancingProxy
+  extend self
+
+  BACKENDS = [
+    {:url => "http://0.0.0.0:3000"},
+    {:url => "http://0.0.0.0:3001"},
+    {:url => "http://0.0.0.0:3002"}
+  ]
+
+  # Represents a "backend", ie. the endpoint for the proxy.
+  #
+  # This could be eg. a WEBrick webserver (see below), so the proxy server works as a _reverse_ proxy.
+  # But it could also be a proxy server, so the proxy server works as a _forward_ proxy.
+  #
+  class Backend
+
+    attr_reader   :url, :host, :port
+    attr_accessor :load
+    alias         :to_s :url
+
+    def initialize(options={})
+      raise ArgumentError, "Please provide a :url and :load" unless options[:url]
+      @url   = options[:url]
+      @load  = options[:load] || 0
+      parsed = URI.parse(@url)
+      @host, @port = parsed.host, parsed.port
+    end
+
+    # Select backend: balanced, round-robin or random
+    #
+    def self.select(strategy = :balanced)
+      @strategy = strategy.to_sym
+      case @strategy
+        when :balanced
+          backend = list.sort_by { |b| b.load }.first
+        when :roundrobin
+          @pool   = list.clone if @pool.nil? || @pool.empty?
+          backend = @pool.shift
+        when :random
+          backend = list[ rand(list.size-1) ]
+        else
+          raise ArgumentError, "Unknown strategy: #{@strategy}"
+      end
+
+      Callbacks.on_select.call(backend)
+      yield backend if block_given?
+      backend
+    end
+
+    # List of backends
+    #
+    def self.list
+      @list ||= BACKENDS.map { |backend| new backend }
+    end
+
+    # Return balancing strategy
+    #
+    def self.strategy
+      @strategy
+    end
+
+    # Increment "currently serving requests" counter
+    #
+    def increment_counter
+      self.load += 1
+    end
+
+    # Decrement "currently serving requests" counter
+    #
+    def decrement_counter
+      self.load -= 1
+    end
+
+  end
+
+  # Callbacks for em-proxy events
+  #
+  module Callbacks
+    include ANSI::Code
+    extend  self
+
+    def on_select
+      lambda do |backend|
+        puts black_on_white { 'on_select'.ljust(12) } + " #{backend.inspect}"
+        backend.increment_counter if Backend.strategy == :balanced
+      end
+    end
+
+    def on_connect
+      lambda do |backend|
+        puts black_on_magenta { 'on_connect'.ljust(12) } + ' ' + bold { backend }
+      end
+    end
+
+    def on_data
+      lambda do |data|
+        puts black_on_yellow { 'on_data'.ljust(12) }, data
+        data
+      end
+    end
+
+    def on_response
+      lambda do |backend, resp|
+        puts black_on_green { 'on_response'.ljust(12) } + " from #{backend}", resp
+        resp
+      end
+    end
+
+    def on_finish
+      lambda do |backend|
+        puts black_on_cyan { 'on_finish'.ljust(12) } + " for #{backend}", ''
+        backend.decrement_counter if Backend.strategy == :balanced
+      end
+    end
+
+  end
+
+  # Wrapping the proxy server
+  #
+  module Server
+    def run(host='0.0.0.0', port=9999)
+
+      puts ANSI::Code.bold { "Launching proxy at #{host}:#{port}...\n" }
+
+      Proxy.start(:host => host, :port => port, :debug => false) do |conn|
+
+        Backend.select do |backend|
+
+          conn.server backend, :host => backend.host, :port => backend.port
+
+          conn.on_connect  &Callbacks.on_connect
+          conn.on_data     &Callbacks.on_data
+          conn.on_response &Callbacks.on_response
+          conn.on_finish   &Callbacks.on_finish
+        end
+
+      end
+    end
+
+    module_function :run
+  end
+
+end
+
+if __FILE__ == $0
+
+  require 'rack'
+
+  class Proxy
+    def self.stop
+      puts "Terminating ProxyServer"
+      EventMachine.stop
+      $servers.each do |pid|
+        puts "Terminating webserver #{pid}"
+        Process.kill('KILL', pid)
+      end
+    end
+  end
+
+  # Simple Rack app to run
+  app = proc { |env| [ 200, {'Content-Type' => 'text/plain'}, ["Hello World!"] ] }
+
+  # Run app on ports 3000-3002
+  $servers = []
+  3.times do |i|
+    $servers << Process.fork { Rack::Handler::WEBrick.run(app, {:Host => "0.0.0.0", :Port => "300#{i}"}) }
+  end
+
+  puts ANSI::Code::green_on_black { "\n=> Send multiple requests to the proxy by running `ruby balancing-client.rb`\n" }
+
+  # Start proxy
+  BalancingProxy::Server.run
+
+end

data/spec/balancing_spec.rb

@@ -0,0 +1,103 @@
+require 'spec/helper'
+require 'examples/balancing'
+
+describe BalancingProxy do
+
+  before(:each) do
+    class BalancingProxy::Backend
+      @list = nil; @pool = nil
+    end
+  end
+
+  before(:all) do
+    @original_stdout = $stdout
+    # Silence the noisy STDOUT output
+    $stdout = File.new('/dev/null', 'w')
+  end
+
+  after(:all) do
+    $stdout = @original_stdout
+  end
+
+  context "generally" do
+
+    it "should raise error for unknown strategy" do
+      lambda { BalancingProxy::Backend.select(:asdf)}.should raise_error(ArgumentError)
+    end
+
+  end
+
+  context "when using the 'random' strategy" do
+
+    it "should select random backend" do
+      class BalancingProxy::Backend
+        def self.list
+          @list ||= [
+            {:url => "http://127.0.0.1:3000"},
+            {:url => "http://127.0.0.2:3000"},
+            {:url => "http://127.0.0.3:3000"}
+          ].map { |backend| new backend }
+        end
+      end
+
+      srand(0)
+      BalancingProxy::Backend.select(:random).host.should == '127.0.0.1'
+    end
+
+  end
+
+  context "when using the 'roundrobin' strategy" do
+    it "should select backends in rotating order" do
+      class BalancingProxy::Backend
+        def self.list
+          @list ||= [
+            {:url => "http://127.0.0.1:3000"},
+            {:url => "http://127.0.0.2:3000"},
+            {:url => "http://127.0.0.3:3000"}
+          ].map { |backend| new backend }
+        end
+      end
+
+      BalancingProxy::Backend.select(:roundrobin).host.should == '127.0.0.1'
+      BalancingProxy::Backend.select(:roundrobin).host.should == '127.0.0.2'
+      BalancingProxy::Backend.select(:roundrobin).host.should == '127.0.0.3'
+      BalancingProxy::Backend.select(:roundrobin).host.should == '127.0.0.1'
+    end
+  end
+
+  context "when using the 'balanced' strategy" do
+
+    it "should select the first backend when all backends have the same load" do
+      class BalancingProxy::Backend
+        def self.list
+          @list ||= [
+            {:url => "http://127.0.0.3:3000", :load => 0},
+            {:url => "http://127.0.0.2:3000", :load => 0},
+            {:url => "http://127.0.0.1:3000", :load => 0}
+          ].map { |backend| new backend }
+        end
+      end
+
+      BalancingProxy::Backend.select.host.should == '127.0.0.3'
+    end
+
+    it "should select the least loaded backend" do
+      class BalancingProxy::Backend
+        def self.list
+          @list ||= [
+            {:url => "http://127.0.0.3:3000", :load => 2},
+            {:url => "http://127.0.0.2:3000", :load => 1},
+            {:url => "http://127.0.0.1:3000", :load => 0}
+          ].map { |backend| new backend }
+        end
+      end
+
+      BalancingProxy::Backend.select.host.should == '127.0.0.1'
+      BalancingProxy::Backend.select.host.should == '127.0.0.1'
+      BalancingProxy::Backend.select.host.should == '127.0.0.2'
+      BalancingProxy::Backend.select.host.should == '127.0.0.3'
+    end
+
+  end
+
+end

data/spec/helper.rb

@@ -1,6 +1,6 @@
 require 'rubygems'
-require 'spec'
+require 'rspec'
 require 'pp'
 require 'em-http'
 
-require 'lib/em-proxy'
+require 'lib/em-proxy'

data/spec/proxy_spec.rb

@@ -54,7 +54,7 @@ describe Proxy do
 
         conn.on_response do |backend, resp|
           backend.should == :goog
-          resp.should =~ /google/
+          resp.size.should >= 0
           EventMachine.stop
         end
       end

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 1
-  - 4
-  version: 0.1.4
+  - 5
+  version: 0.1.5
 platform: ruby
 authors: 
 - Ilya Grigorik
@@ -14,12 +14,11 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-10-03 00:00:00 -04:00
+date: 2011-01-16 00:00:00 -05:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: eventmachine
-  prerelease: false
   requirement: &id001 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
@@ -27,25 +26,70 @@ dependencies:
       - !ruby/object:Gem::Version 
         segments: 
         - 0
-        - 12
-        - 9
-        version: 0.12.9
+        version: "0"
   type: :runtime
+  prerelease: false
   version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  prerelease: false
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: em-http-request
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  prerelease: false
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: ansi
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  prerelease: false
+  version_requirements: *id004
 description: EventMachine Proxy DSL
-email: ilya@igvita.com
-executables: []
-
+email: 
+- ilya@igvita.com
+executables: 
+- em-proxy
 extensions: []
 
-extra_rdoc_files: 
-- README.rdoc
+extra_rdoc_files: []
+
 files: 
+- .gitignore
+- .rspec
+- Gemfile
+- Gemfile.lock
 - README.rdoc
 - Rakefile
-- VERSION
+- bin/em-proxy
 - em-proxy.gemspec
 - examples/appserver.rb
+- examples/balancing-client.rb
+- examples/balancing.rb
 - examples/beanstalkd_interceptor.rb
 - examples/duplex.rb
 - examples/line_interceptor.rb
@@ -60,6 +104,7 @@ files:
 - lib/em-proxy/backend.rb
 - lib/em-proxy/connection.rb
 - lib/em-proxy/proxy.rb
+- spec/balancing_spec.rb
 - spec/helper.rb
 - spec/proxy_spec.rb
 has_rdoc: true
@@ -67,8 +112,8 @@ homepage: http://github.com/igrigorik/em-proxy
 licenses: []
 
 post_install_message: 
-rdoc_options: 
-- --charset=UTF-8
+rdoc_options: []
+
 require_paths: 
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement 
@@ -76,6 +121,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: -2051780992673318158
       segments: 
       - 0
       version: "0"
@@ -84,6 +130,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      hash: -2051780992673318158
       segments: 
       - 0
       version: "0"
@@ -95,16 +142,6 @@ signing_key:
 specification_version: 3
 summary: EventMachine Proxy DSL
 test_files: 
+- spec/balancing_spec.rb
 - spec/helper.rb
 - spec/proxy_spec.rb
-- examples/appserver.rb
-- examples/beanstalkd_interceptor.rb
-- examples/duplex.rb
-- examples/line_interceptor.rb
-- examples/port_forward.rb
-- examples/relay_port_forward.rb
-- examples/schemaless-mysql/mysql_interceptor.rb
-- examples/selective_forward.rb
-- examples/simple.rb
-- examples/smtp_spam_filter.rb
-- examples/smtp_whitelist.rb

data/VERSION

@@ -1 +0,0 @@
-0.1.4