em-proxy 0.1.1

data/README.rdoc

@@ -0,0 +1,50 @@
+= EM-Proxy
+
+EventMachine Proxy DSL for writing high-performance transparent / intercepting proxies in Ruby.
+
+- Slides from RailsConf 2009: http://bit.ly/D7oWB
+- GoGaRuCo notes & Slides: http://www.igvita.com/2009/04/20/ruby-proxies-for-scale-and-monitoring/
+
+== Getting started
+
+  # install & configure gemcutter repos
+  gem update --system
+  gem install gemcutter
+  gem tumble
+
+  gem install em-proxy
+
+  irb:0> require 'em-proxy'
+
+== Simple port forwarding proxy
+
+    Proxy.start(:host => "0.0.0.0", :port => 80, :debug => true) do |conn|
+      conn.server :srv, :host => "127.0.0.1", :port => 81
+
+      # modify / process request stream
+      conn.on_data do |data|
+        p [:on_data, data]
+        data
+      end
+
+      # modify / process response stream
+      conn.on_response do |backend, resp|
+        p [:on_response, backend, resp]
+        resp
+      end
+
+      # termination logic
+      conn.on_finish do |backend|
+        p [:on_finish, name]
+
+        # terminate connection (in duplex mode, you can terminate when prod is done)
+        unbind if backend == :srv
+      end
+    end
+
+For more examples see the /examples directory.
+- SMTP Spam Filtering
+- Duplicating traffic
+- Selective forwarding
+- Beanstalkd interceptor
+- etc.

data/VERSION

@@ -0,0 +1 @@
+0.1.1

data/examples/appserver.rb

@@ -0,0 +1,11 @@
+require "rubygems"
+require "rack"
+
+app = lambda {
+  p  [:serving, ARGV[0]]
+
+	sleep(ARGV[1].to_i)
+  [200, {"Content-Type" => "text/plain"}, ["hello world: #{ARGV[1]}"]]
+}
+
+Rack::Handler::Mongrel.run(app, {:Host => "0.0.0.0", :Port => ARGV[0]})

data/examples/beanstalkd_interceptor.rb

@@ -0,0 +1,43 @@
+require 'lib/em-proxy'
+
+Proxy.start(:host => "0.0.0.0", :port => 11300) do |conn|
+  conn.server :srv, :host => "127.0.0.1", :port => 11301
+
+  # put <pri> <delay> <ttr> <bytes>\r\n
+  PUT_CMD = /put (\d+) (\d+) (\d+) (\d+)\r\n/
+
+  conn.on_data do |data|
+    if put = data.match(PUT_CMD)
+
+      # archive any job > 10 minutes away
+      if put[2].to_i > 600
+        p [:put, :archive]
+        # INSERT INTO ....
+
+        conn.send_data "INSERTED 9999\r\n"
+        data = nil
+      end
+    end
+
+    data
+  end
+ 
+  conn.on_response do |backend, resp|
+    p [:resp, resp]
+    resp
+  end
+end
+
+#
+# beanstalkd -p 11301 -d
+# ruby examples/beanstalkd_interceptor.rb
+#
+# irb
+# >> require 'beanstalk-client'
+# >> beanstalk = Beanstalk::Pool.new(['127.0.0.1'])
+# >> beanstalk.put("job1")
+# => 1
+# >> beanstalk.put("job2")
+# => 2
+# >> beanstalk.put("job3", 0, 1000)
+# => 9999

data/examples/duplex.rb

@@ -0,0 +1,36 @@
+require 'lib/em-proxy'
+
+Proxy.start(:host => "0.0.0.0", :port => 80, :debug => true) do |conn|
+  @start = Time.now
+  @data = Hash.new("")
+
+  conn.server :test, :host => "127.0.0.1", :port => 81    # production, will render resposne
+  conn.server :prod, :host => "127.0.0.1", :port => 82     # testing, internal only
+
+  conn.on_data do |data|
+    # rewrite User-Agent
+    data.gsub(/User-Agent: .*?\r\n/, "User-Agent: em-proxy/0.1\r\n")
+  end
+ 
+  conn.on_response do |server, resp|
+    # only render response from production
+    @data[server] += resp
+    resp if server == :prod
+  end
+
+  conn.on_finish do |name|
+    p [:on_finish, name, Time.now - @start]
+    p @data
+  end
+end
+
+#
+# ruby examples/appserver.rb 81
+# ruby examples/appserver.rb 82
+# ruby examples/line_interceptor.rb
+# curl localhost
+#
+# > [:on_finish, 1.008561]
+# > {:prod=>"HTTP/1.1 200 OK\r\nConnection: close\r\nDate: Fri, 01 May 2009 04:20:00 GMT\r\nContent-Type: text/plain\r\n\r\nhello world: 0",
+#       :test=>"HTTP/1.1 200 OK\r\nConnection: close\r\nDate: Fri, 01 May 2009 04:20:00 GMT\r\nContent-Type: text/plain\r\n\r\nhello world: 1"}
+#

data/examples/line_interceptor.rb

@@ -0,0 +1,22 @@
+require 'lib/em-proxy'
+
+Proxy.start(:host => "0.0.0.0", :port => 80) do |conn|
+  conn.server :srv, :host => "127.0.0.1", :port => 81
+
+  conn.on_data do |data|
+    data
+  end
+ 
+  conn.on_response do |backend, resp|
+    # substitute all mentions of hello to 'good bye', aka intercepting proxy
+    resp.gsub(/hello/, 'good bye')
+  end  
+end
+
+#
+# ruby examples/appserver.rb 81
+# ruby examples/line_interceptor.rb
+# curl localhost
+#
+# > good bye world: 0
+#

data/examples/port_forward.rb

@@ -0,0 +1,18 @@
+require 'lib/em-proxy'
+
+Proxy.start(:host => "0.0.0.0", :port => 80, :debug => true) do |conn|
+  conn.server :srv, :host => "127.0.0.1", :port => 81
+
+  # modify / process request stream
+  conn.on_data do |data|
+    p [:on_data, data]
+    data
+  end
+ 
+  # modify / process response stream
+  conn.on_response do |backend, resp|
+    p [:on_response, backend, resp]
+    # resp = "HTTP/1.1 200 OK\r\nConnection: close\r\nDate: Thu, 30 Apr 2009 03:53:28 GMT\r\nContent-Type: text/plain\r\n\r\nHar!"
+    resp
+  end  
+end

data/examples/selective_forward.rb

@@ -0,0 +1,38 @@
+require 'lib/em-proxy'
+
+Proxy.start(:host => "0.0.0.0", :port => 80) do |conn|
+  @start = Time.now
+  @data = Hash.new("")
+
+  conn.server :prod, :host => "127.0.0.1", :port => 81    # production, will render resposne
+  conn.server :test, :host => "127.0.0.1", :port => 82     # testing, internal only
+
+  conn.on_data do |data|
+    # rewrite User-Agent
+    [data.gsub(/User-Agent: .*?\r\n/, 'User-Agent: em-proxy/0.1\r\n'), [:prod]]
+  end
+ 
+  conn.on_response do |server, resp|
+    # only render response from production
+    @data[server] += resp
+    resp if server == :prod
+  end
+
+  conn.on_finish do |name|
+    p [:on_finish, name, Time.now - @start]
+    unbind if name == :prod # terminate connection once prod is done
+   
+    p @data if name == :done
+  end
+end
+
+#
+# ruby examples/appserver.rb 81
+# ruby examples/appserver.rb 82
+# ruby examples/line_interceptor.rb
+# curl localhost
+#
+# > [:on_finish, 1.008561]
+# > {:prod=>"HTTP/1.1 200 OK\r\nConnection: close\r\nDate: Fri, 01 May 2009 04:20:00 GMT\r\nContent-Type: text/plain\r\n\r\nhello world: 0",
+#       :test=>"HTTP/1.1 200 OK\r\nConnection: close\r\nDate: Fri, 01 May 2009 04:20:00 GMT\r\nContent-Type: text/plain\r\n\r\nhello world: 1"}
+#

data/examples/smtp_spam_filter.rb

@@ -0,0 +1,107 @@
+require 'lib/em-proxy'
+require 'em-http'
+require 'yaml'
+require 'net/http'
+
+Proxy.start(:host => "0.0.0.0", :port => 2524) do |conn|
+  conn.server :srv, :host => "127.0.0.1", :port => 2525
+
+  RCPT_CMD = /RCPT TO:<(.*)?>\r\n/        # RCPT TO:<name@address.com>\r\n
+  FROM_CMD = /MAIL FROM:<(.*)?>\r\n/    # MAIL FROM:<ilya@aiderss.com>\r\n
+  MSG_CMD = /354 Start your message/   # 354 Start your message
+  MSGEND_CMD = /^.\r\n/
+
+  conn.on_data do |data|
+    @from = data.match(FROM_CMD)[1] if data.match(FROM_CMD)
+    @rcpt = data.match(RCPT_CMD)[1] if data.match(RCPT_CMD)
+    @done = true if data.match(MSGEND_CMD)
+
+    if @buffer
+      @msg += data
+      data = nil
+    end
+
+    if @done
+      @buffer = false
+      res = Net::HTTP.post_form(URI.parse('http://api.defensio.com/app/1.2/audit-comment/77ca297d7546705ee2b5136fad0dcaf8.yaml'), {
+          "owner-url" => "http://www.github.com/igrigorik/em-http-request",
+          "user-ip" => "216.16.254.254",
+          "article-date" => "2009/04/01",
+          "comment-author" => @from,
+          "comment-type" => "comment",
+          "comment-content" => @msg})
+
+      defensio = YAML.load(res.body)['defensio-result']
+      p [:defensio, "SPAM: #{defensio['spam']}, Spaminess: #{defensio['spaminess']}"]
+
+      if defensio['spam']
+        conn.send_data "550 No such user here\n"
+      else
+        data = @msg
+      end
+    end
+
+    data
+  end
+ 
+  conn.on_response do |server, resp|
+    p [:resp, resp]
+
+    if resp.match(MSG_CMD)
+      @buffer = true
+      @msg = ""
+    end
+
+    resp
+  end
+end
+
+# mailtrap run -p 2525 -f /tmp/mailtrap.log
+# ruby examples/smtp_spam_filter.rb
+#
+# >> require 'net/smtp'
+# >> smtp = Net::SMTP.start("localhost", 2524)
+# >> smtp.send_message "Hello World!", "ilya@aiderss.com", "ilya@igvita.com"
+
+
+# Protocol trace
+#
+# [:srv, :conn_complete]
+# [:srv, "220 localhost MailTrap ready ESTMP\n"]
+# [:relay_from_backend, :srv, "220 localhost MailTrap ready ESTMP\n"]
+# [:resp, "220 localhost MailTrap ready ESTMP\n"]
+# [:connection, "EHLO localhost.localdomain\r\n"]
+# [:srv, "250-localhost offers just ONE extension my pretty"]
+# [:relay_from_backend, :srv, "250-localhost offers just ONE extension my pretty"]
+# [:resp, "250-localhost offers just ONE extension my pretty"]
+# [:srv, "\n250 HELP\n"]
+# [:relay_from_backend, :srv, "\n250 HELP\n"]
+# [:resp, "\n250 HELP\n"]
+# [:connection, "MAIL FROM:<ilya@aiderss.com>\r\n"]
+# [:srv, "250 OK\n"]
+# [:relay_from_backend, :srv, "250 OK\n"]
+# [:resp, "250 OK\n"]
+# [:connection, "RCPT TO:<ilya@igvita.com>\r\n"]
+# [:srv, "250 OK"]
+# [:relay_from_backend, :srv, "250 OK"]
+# [:resp, "250 OK"]
+# [:srv, "\n"]
+# [:relay_from_backend, :srv, "\n"]
+# [:resp, "\n"]
+# [:connection, "DATA\r\n"]
+# [:srv, "354 Start your message"]
+# [:relay_from_backend, :srv, "354 Start your message"]
+# [:resp, "354 Start your message"]
+# [:srv, "\n"]
+# [:relay_from_backend, :srv, "\n"]
+# [:resp, "\n"]
+# [:connection, "Hello World\r\n"]
+# [:connection, ".\r\n"]
+#
+# [:defensio, "SPAM: false, Spaminess: 0.4"]
+#
+# [:srv, "250 OK\n"]
+# [:relay_from_backend, :srv, "250 OK\n"]
+# [:resp, "250 OK\n"]
+#
+

data/examples/smtp_whitelist.rb

@@ -0,0 +1,39 @@
+require 'lib/em-proxy'
+
+Proxy.start(:host => "0.0.0.0", :port => 2524) do |conn|
+  conn.server :srv, :host => "127.0.0.1", :port => 2525
+
+  # RCPT TO:<name@address.com>\r\n
+  RCPT_CMD = /RCPT TO:<(.*)?>\r\n/
+
+  conn.on_data do |data|
+    
+    if rcpt = data.match(RCPT_CMD)
+      if rcpt[1] != "ilya@igvita.com"
+       conn.send_data "550 No such user here\n"
+       data = nil
+      end
+    end
+
+    data
+  end
+ 
+  conn.on_response do |backend, resp|
+    resp
+  end
+end
+
+
+# mailtrap run -p 2525 -f /tmp/mailtrap.log
+# ruby examples/smtp_whitelist.rb
+#
+# >> require 'net/smtp'
+# >> smtp = Net::SMTP.start("localhost", 2524)
+# >> smtp.send_message "Hello World!", "ilya@aiderss.com", "ilya@igvita.com"
+# => #<Net::SMTP::Response:0xb7dcff5c @status="250", @string="250 OK\n">
+# >> smtp.finish
+# => #<Net::SMTP::Response:0xb7dcc8d4 @status="221", @string="221 Seeya\n">
+#
+# >> smtp.send_message "Hello World!", "ilya@aiderss.com", "missing_user@igvita.com"
+# => Net::SMTPFatalError: 550 No such user here
+#

data/lib/em-proxy.rb

@@ -0,0 +1,8 @@
+$:.unshift(File.dirname(__FILE__) + '/../lib')
+
+require "rubygems"
+require "eventmachine"
+
+%w[ backend proxy connection ].each do |file|
+  require "em-proxy/#{file}"
+end

data/lib/em-proxy/backend.rb

@@ -0,0 +1,46 @@
+module EventMachine
+  module ProxyServer
+    class Backend < EventMachine::Connection
+      attr_accessor :plexer, :data, :name, :debug
+
+      def initialize(debug = false)
+        @debug = debug
+        @connected = EM::DefaultDeferrable.new
+        @data = []
+      end
+
+      def connection_completed
+        debug [@name, :conn_complete]
+        @connected.succeed
+      end
+
+      def receive_data(data)
+        debug [@name, data]
+        @data.push data
+        @plexer.relay_from_backend(@name, data)
+      end
+
+      # Buffer data until the connection to the backend server
+      # is established and is ready for use
+      def send(data)
+        @connected.callback { send_data data }
+      end
+
+      # Notify upstream plexer that the backend server is done
+      # processing the request
+      def unbind
+        debug [@name, :unbind]
+        @plexer.unbind_backend(@name)
+      end
+
+      private
+
+      def debug(*data)
+        return unless @debug
+        require 'pp'
+        pp data
+        puts
+      end
+    end
+  end
+end

data/lib/em-proxy/connection.rb

@@ -0,0 +1,90 @@
+module EventMachine
+  module ProxyServer
+    class Connection < EventMachine::Connection
+      attr_accessor :debug
+      
+      ##### Proxy Methods
+      def on_data(&blk); @on_data = blk; end
+      def on_response(&blk); @on_response = blk; end
+      def on_finish(&blk); @on_finish = blk; end
+
+      ##### EventMachine
+      def initialize(options)
+        @debug = options[:debug] || false
+        @servers = {}
+      end
+
+      def receive_data(data)
+        debug [:connection, data]
+        processed = @on_data.call(data)
+
+        if processed.is_a? Array
+          data, servers = *processed
+
+          # guard for "unbound" servers
+          servers = servers.collect {|s| @servers[s]}.compact
+        else
+          data = processed
+          servers ||= @servers.values.compact
+        end
+
+        servers.each do |s|
+          s.send_data data unless data.nil?
+        end
+      end
+
+      #
+      # initialize connections to backend servers
+      #
+      def server(name, opts)
+        srv = EventMachine::connect(opts[:host], opts[:port], EventMachine::ProxyServer::Backend, @debug) do |c|
+          c.name = name
+          c.plexer = self
+        end
+
+        @servers[name] = srv
+      end
+
+      #
+      # relay data from backend server to client
+      #
+      def relay_from_backend(name, data)
+        debug [:relay_from_backend, name, data]
+
+        data = @on_response.call(name, data)
+        send_data data unless data.nil?
+      end
+
+      def unbind
+        debug [:unbind, :connection]
+      
+        # terminate any unfinished connections
+        @servers.values.compact.each do |s|
+          s.close_connection
+        end
+      end
+
+      def unbind_backend(name)
+        debug [:unbind_backend, name]
+        @servers[name] = nil
+
+        # if all connections are terminated downstream, then notify client
+        close_connection if @servers.values.compact.size.zero?
+        
+        if @on_finish
+          @on_finish.call(name)
+          @on_finish.call(:done) if @servers.values.compact.size.zero?
+        end
+      end
+
+      private
+      
+      def debug(*data)
+        return unless @debug
+        require 'pp'
+        pp data
+        puts
+      end
+    end
+  end
+end

data/lib/em-proxy/proxy.rb

@@ -0,0 +1,21 @@
+class Proxy
+
+  def self.start(options, &blk)
+    EM.epoll
+    EM.run do
+
+      trap("TERM") { stop }
+      trap("INT")  { stop }
+
+      EventMachine::start_server(options[:host], options[:port],
+                                 EventMachine::ProxyServer::Connection, options) do |c|
+        c.instance_eval(&blk)
+      end
+    end
+  end
+
+  def self.stop
+    puts "Terminating ProxyServer"
+    EventMachine.stop
+  end
+end

data/spec/helper.rb

@@ -0,0 +1,6 @@
+require 'rubygems'
+require 'spec'
+require 'pp'
+require 'em-http'
+
+require 'lib/em-proxy'

data/spec/proxy_spec.rb

@@ -0,0 +1,111 @@
+require 'spec/helper'
+
+describe Proxy do
+
+  def failed
+    EventMachine.stop
+    fail
+  end
+
+  it "should recieve data on port 8080" do
+    EM.run do
+      EventMachine.add_timer(2) do
+        EventMachine::HttpRequest.new('http://127.0.0.1:8080/test').get({:timeout => 1})
+      end
+
+      Proxy.start(:host => "0.0.0.0", :port => 8080) do |conn|
+        conn.on_data do |data|
+          data.should =~ /GET \/test/
+          EventMachine.stop
+        end
+      end
+    end
+  end
+
+  it "should transparently redirect TCP traffic to google" do
+    EM.run do
+      EventMachine.add_timer(2) do
+        EventMachine::HttpRequest.new('http://127.0.0.1:8080/').get({:timeout => 1})
+      end
+
+      Proxy.start(:host => "0.0.0.0", :port => 8080) do |conn|
+        conn.server :goog, :host => "google.com", :port => 80
+        conn.on_data { |data| data }
+
+        conn.on_response do |backend, resp|
+          backend.should == :goog
+          resp.should =~ /google/
+          EventMachine.stop
+        end
+      end
+    end
+  end
+
+  it "should duplex TCP traffic to two backends google & yahoo" do
+    EM.run do
+      EventMachine.add_timer(2) do
+        EventMachine::HttpRequest.new('http://127.0.0.1:8080/').get({:timeout => 1})
+      end
+
+      Proxy.start(:host => "0.0.0.0", :port => 8080) do |conn|
+        conn.server :goog, :host => "google.com", :port => 80
+        conn.server :yhoo, :host => "yahoo.com", :port => 80
+        conn.on_data { |data| data }
+
+        seen = []
+        conn.on_response do |backend, resp|
+          case backend
+          when :goog then
+            resp.should =~ /google/
+            seen.push backend
+          when :yhoo
+            resp.should =~ /yahoo|yimg/
+            seen.push backend
+          end
+          seen.uniq!
+
+          EventMachine.stop if seen.size == 2
+        end
+      end
+    end
+  end
+
+  it "should intercept & alter response from Google" do
+    EM.run do
+      EventMachine.add_timer(2) do
+        http = EventMachine::HttpRequest.new('http://127.0.0.1:8080/').get({:timeout => 1})
+        http.errback { failed }
+        http.callback {
+          http.response_header.status.should == 404
+          EventMachine.stop
+        }
+      end
+
+      Proxy.start(:host => "0.0.0.0", :port => 8080) do |conn|
+        conn.server :goog, :host => "google.com", :port => 80
+        conn.on_data { |data| data }
+        conn.on_response do |backend, data|
+          data.gsub(/^HTTP\/1.1 200/, 'HTTP/1.1 404')
+        end
+      end
+    end
+  end
+
+  it "should invoke on_finish callback when connection is terminated" do
+    EM.run do
+      EventMachine.add_timer(2) do
+        EventMachine::HttpRequest.new('http://127.0.0.1:8080/').get({:timeout => 1})
+      end
+
+      Proxy.start(:host => "0.0.0.0", :port => 8080) do |conn|
+        conn.server :goog, :host => "google.com", :port => 80
+        conn.on_data { |data| data }
+        conn.on_response { |backend, resp| resp }
+        conn.on_finish do |backend|
+          backend.should == :goog
+          EventMachine.stop
+        end
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,88 @@
+--- !ruby/object:Gem::Specification 
+name: em-proxy
+version: !ruby/object:Gem::Version 
+  version: 0.1.1
+platform: ruby
+authors: 
+- Ilya Grigorik
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-10-25 00:00:00 -04:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: eventmachine
+  type: :runtime
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 0.12.9
+    version: 
+description: EventMachine Proxy DSL
+email: ilya@igvita.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README.rdoc
+files: 
+- README.rdoc
+- VERSION
+- examples/appserver.rb
+- examples/beanstalkd_interceptor.rb
+- examples/duplex.rb
+- examples/line_interceptor.rb
+- examples/port_forward.rb
+- examples/selective_forward.rb
+- examples/smtp_spam_filter.rb
+- examples/smtp_whitelist.rb
+- lib/em-proxy.rb
+- lib/em-proxy/backend.rb
+- lib/em-proxy/connection.rb
+- lib/em-proxy/proxy.rb
+- spec/helper.rb
+- spec/proxy_spec.rb
+has_rdoc: true
+homepage: http://github.com/igrigorik/em-proxy
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: em-proxy
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: EventMachine Proxy DSL
+test_files: 
+- spec/helper.rb
+- spec/proxy_spec.rb
+- examples/appserver.rb
+- examples/beanstalkd_interceptor.rb
+- examples/duplex.rb
+- examples/line_interceptor.rb
+- examples/port_forward.rb
+- examples/selective_forward.rb
+- examples/smtp_spam_filter.rb
+- examples/smtp_whitelist.rb