em-proxy 0.1.1 → 0.1.2

data/README.rdoc

@@ -34,7 +34,7 @@ EventMachine Proxy DSL for writing high-performance transparent / intercepting p
       end
 
       # termination logic
-      conn.on_finish do |backend|
+      conn.on_finish do |backend, name|
         p [:on_finish, name]
 
         # terminate connection (in duplex mode, you can terminate when prod is done)
@@ -47,4 +47,33 @@ For more examples see the /examples directory.
 - Duplicating traffic
 - Selective forwarding
 - Beanstalkd interceptor
-- etc.
+- etc.
+
+A schema-free MySQL proof of concept, via an EM-Proxy server:
+- http://www.igvita.com/2010/03/01/schema-free-mysql-vs-nosql/
+- Code in examples/schemaless-mysql
+
+== License
+
+(The MIT License)
+
+Copyright (c) 2010 Ilya Grigorik
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,19 @@
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gemspec|
+    gemspec.name = "em-proxy"
+    gemspec.summary = "EventMachine Proxy DSL"
+    gemspec.description = gemspec.summary
+    gemspec.email = "ilya@igvita.com"
+    gemspec.homepage = "http://github.com/igrigorik/em-proxy"
+    gemspec.authors = ["Ilya Grigorik"]
+    gemspec.add_dependency('eventmachine', '>= 0.12.9')
+    gemspec.rubyforge_project = "em-proxy"
+  end
+
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler not available. Install it with: sudo gem install technicalpickles-jeweler -s http://gems.github.com"
+end

data/VERSION

@@ -1 +1 @@
-0.1.1
+0.1.2

data/examples/relay_port_forward.rb

@@ -0,0 +1,26 @@
+require 'lib/em-proxy'
+
+Proxy.start(:host => "0.0.0.0", :port => 80, :debug => false) do |conn|
+  # Specifying :relay_server or :relay_client is useful if only requests or responses 
+  # need to be processed. The proxy throughput will roughly double.
+  conn.server :srv, :host => "127.0.0.1", :port => 81, :relay_client => true, :relay_server => true
+
+  conn.on_connect do
+    p [:on_connect, "#{conn.peer.join(':')} connected"]
+  end
+
+  # modify / process request stream
+  # on_data will not be called when :relay_server => true is passed as server option
+  conn.on_data do |data|
+    p [:on_data, data]
+    data
+  end
+ 
+  # modify / process response stream
+  # on_response will not be called when :relay_client => true is passed as server option
+  conn.on_response do |backend, resp|
+    p [:on_response, backend, resp]
+    # resp = "HTTP/1.1 200 OK\r\nConnection: close\r\nDate: Thu, 30 Apr 2009 03:53:28 GMT\r\nContent-Type: text/plain\r\n\r\nHar!"
+    resp
+  end  
+end

data/examples/schemaless-mysql/client.rb

@@ -0,0 +1,17 @@
+require "rubygems"
+require "mysql"    # gem install ruby-mysql -v 2.9.2
+
+my = Mysql.connect('127.0.0.1', 'root', '', 'noschema', 3307)
+
+p my.list_tables
+__END__
+
+# look ma, no schema! ooh yeah.
+my.query("create table posts")
+
+# insert a few records into our schemaless MySQL store... :-)
+my.query("insert into posts values('first post'('author', 'Ilya'),('nick', 'igrigorik'))")
+my.query("insert into posts values('author:Ilya','location:Waterloo')")
+
+my.query("select author from posts") { |r| puts r }
+my.query("select nick from posts") { |r| puts r }

data/examples/schemaless-mysql/mysql_interceptor.rb

@@ -0,0 +1,181 @@
+require "lib/em-proxy"
+require "em-mysql"
+require "stringio"
+require "fiber"
+
+Proxy.start(:host => "0.0.0.0", :port => 3307) do |conn|
+  conn.server :mysql, :host => "127.0.0.1", :port => 3306, :relay_server => true
+
+  QUERY_CMD = 3
+  MAX_PACKET_LENGTH = 2**24-1
+
+  # open a direct connection to MySQL for the schema-free coordination logic
+  @mysql = EventMachine::MySQL.new(:host => 'localhost', :database => 'noschema')
+
+  conn.on_data do |data|
+    fiber = Fiber.new {
+      p [:original_request, data]
+
+      overhead, chunks, seq = data[0,4].unpack("CvC")
+      type, sql = data[4, data.size].unpack("Ca*")
+
+      p [:request, [overhead, chunks, seq], [type, sql]]
+
+      if type == QUERY_CMD
+        query = sql.downcase.split
+        p [:query, query]
+
+        # TODO: can probably switch to http://github.com/omghax/sql
+        #       for AST query parsing & mods.
+
+        case query.first
+        when "create" then
+          # Allow schemaless table creation, ex: 'create table posts'
+          # By creating a table with a single id for key storage, aka
+          # rewrite to: 'create table posts (id varchar(255))'. All
+          # future attribute tables will be created on demand at
+          # insert time of a new record
+          overload = "(id varchar(255), UNIQUE(id));"
+          query += [overload]
+          overhead += overload.size + 1
+
+          p [:create_new_schema_free_table, query, data]
+
+        when "insert" then
+          # Overload the INSERT syntax to allow for nested parameters
+          # inside the statement. ex:
+          #   INSERT INTO posts (id, author, nickname, ...) VALUES (
+          #     'ilya', 'Ilya Grigorik', 'igrigorik'
+          #   )
+          #
+          # The following query will be mapped into 3 distinct tables:
+          # => 'posts' table will store the key
+          # => 'posts_author' will store key, value
+          # => 'posts_nickname' will store key, value
+          #
+          #  or, in SQL..
+          #
+          # =>  insert into posts values("ilya");
+          # =>  create table posts_author (id varchar(40), value varchar(255), UNIQUE(id));
+          # =>  insert into posts_author values("ilya", "Ilya Grigorik");
+          # =>  ... repeat for every attribute
+          #
+          # If the table post_value has not been seen before, it will
+          # be created on the fly. Hence allowing us to add and remove
+          # keys and values at will. :-)
+          #
+          # P.S. There is probably cleaner syntax for this, but hey...
+            
+            
+          if insert = sql.match(/\((.*?)\).*?\((.*?)\)/)
+            data = {}
+            table = query[2]
+            keys = insert[1].split(',').map!{|s| s.strip}
+            values = insert[2].scan(/([^\'|\"]+)/).flatten.reject {|s| s.strip == ','}
+            keys.each_with_index {|k,i| data[k] = values[i]}
+
+            data.each do |key, value|
+              next if key == 'id'
+              attr_sql = "insert into #{table}_#{key} values('#{data['id']}', '#{value}')"
+
+              q = @mysql.query(attr_sql)
+              q.errback  { |res|
+                # if the attribute table for this model does not yet exist then create it!
+                # - yes, there is a race condition here, add fiber logic later
+                if res.is_a?(Mysql::Error) and res.message =~ /Table.*doesn\'t exist/
+
+                  table_sql = "create table #{table}_#{key} (id varchar(255), value varchar(255), UNIQUE(id))"
+                  tc = @mysql.query(table_sql)
+                  tc.callback { @mysql.query(attr_sql) }
+                end
+              }
+
+              p [:inserted_attr, table, key, value]
+            end
+
+            # override the query to insert the key into posts table
+            query = query[0,3] + ["VALUES('#{data['id']}')"]
+            overhead = query.join(" ").size + 1
+
+            p [:insert, query]
+          end
+
+        when "select" then
+          # Overload the select call to perform a multi-join in the background
+          # and rewrite the attribute names to fool the client into thinking it
+          # all came from the same table.
+          #
+          # To figure out which tables we need to join on, do the simple / dumb
+          # approach and issue a 'show tables like key_%' to do 'runtime
+          # introspection'. Could easily cache this, but that's for later.
+          #
+          # Ex, a 'select * from posts' query with one value (author) would be
+          # rewritten into the following query:
+          #
+          #  SELECT posts.id as id, posts_author.value as author FROM posts
+          #   LEFT OUTER JOIN posts_author ON posts_author.id = posts.id
+          #   WHERE posts.id = "ilya";
+
+          select = sql.match(/select(.*?)from\s([^\s]+)/)
+          where  = sql.match(/where\s([^=]+)\s?=\s?'?"?([^\s'"]+)'?"?/)
+          attrs, table = select[1].strip.split(','), select[2] if select
+          key = where[2] if where
+
+          if select
+            p [:select, select, attrs, where]
+
+            tables = @mysql.query("show tables like '#{table}_%'")
+            tables.callback { |res|
+              fiber.resume(res.all_hashes.collect(&:values).flatten.collect{ |c|
+                  c.split('_').last
+                })
+            }
+            tables = Fiber.yield
+
+            p [:select_tables, tables]
+
+            # build the select statements, hide the tables behind each attribute
+            join =  "select #{table}.id as id "
+            tables.each do |column|
+              join += " , #{table}_#{column}.value as #{column} "
+            end
+
+            # add the joins to stich it all together
+            join += " FROM #{table} "
+            tables.each do |column|
+              join += " LEFT OUTER JOIN #{table}_#{column} ON #{table}_#{column}.id = #{table}.id "
+            end
+
+            join += " WHERE #{table}.id = '#{key}' " if key
+
+            query = [join]
+            overhead = join.size + 1
+
+            p [:join_query, join]
+          end
+        end
+
+        # repack the query data and forward to server
+        # - have to split message on packet boundaries
+
+        seq, data = 0, []
+        query = StringIO.new([type, query.join(" ")].pack("Ca*"))
+        while q = query.read(MAX_PACKET_LENGTH)
+          data.push [q.length % 256, q.length / 256, seq].pack("CvC") + q
+          seq = (seq + 1) % 256
+        end
+
+        p [:final_query, data, chunks, overhead]
+        puts "-" * 100
+      end
+
+      [data].flatten.each do |chunk|
+        conn.relay_to_servers(chunk)
+      end
+
+      :async # we will render results later
+    }
+
+    fiber.resume
+  end
+end

data/lib/em-proxy.rb

@@ -1,7 +1,9 @@
 $:.unshift(File.dirname(__FILE__) + '/../lib')
 
+
 require "rubygems"
 require "eventmachine"
+require "socket"
 
 %w[ backend proxy connection ].each do |file|
   require "em-proxy/#{file}"

data/lib/em-proxy/connection.rb

@@ -7,6 +7,7 @@ module EventMachine
       def on_data(&blk); @on_data = blk; end
       def on_response(&blk); @on_response = blk; end
       def on_finish(&blk); @on_finish = blk; end
+      def on_connect(&blk); blk.call; end
 
       ##### EventMachine
       def initialize(options)
@@ -18,6 +19,11 @@ module EventMachine
         debug [:connection, data]
         processed = @on_data.call(data)
 
+        return if processed == :async or processed.nil?
+        relay_to_servers(processed)
+      end
+
+      def relay_to_servers(processed)
         if processed.is_a? Array
           data, servers = *processed
 
@@ -32,7 +38,7 @@ module EventMachine
           s.send_data data unless data.nil?
         end
       end
-
+      
       #
       # initialize connections to backend servers
       #
@@ -40,11 +46,20 @@ module EventMachine
         srv = EventMachine::connect(opts[:host], opts[:port], EventMachine::ProxyServer::Backend, @debug) do |c|
           c.name = name
           c.plexer = self
+          c.proxy_incoming_to(self, 10240) if opts[:relay_server]
         end
+        self.proxy_incoming_to(srv, 10240) if opts[:relay_client]
 
         @servers[name] = srv
       end
 
+      #
+      # [ip, port] of the connected client
+      #
+      def peer
+        @peer ||= Socket.unpack_sockaddr_in(get_peername).reverse
+      end
+
       #
       # relay data from backend server to client
       #
@@ -69,21 +84,24 @@ module EventMachine
         @servers[name] = nil
 
         # if all connections are terminated downstream, then notify client
-        close_connection if @servers.values.compact.size.zero?
+        close_connection_after_writing if @servers.values.compact.size.zero?
         
         if @on_finish
           @on_finish.call(name)
-          @on_finish.call(:done) if @servers.values.compact.size.zero?
+
+          # not sure if this is required
+          # @on_finish.call(:done) if @servers.values.compact.size.zero?
         end
       end
 
       private
       
       def debug(*data)
-        return unless @debug
-        require 'pp'
-        pp data
-        puts
+        if @debug
+          require 'pp'
+          pp data
+          puts
+        end
       end
     end
   end

data/lib/em-proxy/proxy.rb

@@ -9,7 +9,7 @@ class Proxy
 
       EventMachine::start_server(options[:host], options[:port],
                                  EventMachine::ProxyServer::Connection, options) do |c|
-        c.instance_eval(&blk)
+        blk.call(c)
       end
     end
   end

data/spec/proxy_spec.rb

@@ -9,7 +9,7 @@ describe Proxy do
 
   it "should recieve data on port 8080" do
     EM.run do
-      EventMachine.add_timer(2) do
+      EventMachine.add_timer(0.1) do
         EventMachine::HttpRequest.new('http://127.0.0.1:8080/test').get({:timeout => 1})
       end
 
@@ -22,9 +22,27 @@ describe Proxy do
     end
   end
 
+  it "should call the on_connect callback" do
+    connected = false
+    EM.run do
+      EventMachine.add_timer(0.1) do
+        EventMachine::HttpRequest.new('http://127.0.0.1:8080/test').get({:timeout => 1})
+      end
+
+      Proxy.start(:host => "0.0.0.0", :port => 8080) do |conn|
+        conn.on_connect do 
+          connected = true
+          EventMachine.stop
+        end
+      end
+    end
+    connected.should == true
+  end
+
+
   it "should transparently redirect TCP traffic to google" do
     EM.run do
-      EventMachine.add_timer(2) do
+      EventMachine.add_timer(0.1) do
         EventMachine::HttpRequest.new('http://127.0.0.1:8080/').get({:timeout => 1})
       end
 
@@ -43,7 +61,7 @@ describe Proxy do
 
   it "should duplex TCP traffic to two backends google & yahoo" do
     EM.run do
-      EventMachine.add_timer(2) do
+      EventMachine.add_timer(0.1) do
         EventMachine::HttpRequest.new('http://127.0.0.1:8080/').get({:timeout => 1})
       end
 
@@ -72,7 +90,7 @@ describe Proxy do
 
   it "should intercept & alter response from Google" do
     EM.run do
-      EventMachine.add_timer(2) do
+      EventMachine.add_timer(0.1) do
         http = EventMachine::HttpRequest.new('http://127.0.0.1:8080/').get({:timeout => 1})
         http.errback { failed }
         http.callback {
@@ -93,7 +111,7 @@ describe Proxy do
 
   it "should invoke on_finish callback when connection is terminated" do
     EM.run do
-      EventMachine.add_timer(2) do
+      EventMachine.add_timer(0.1) do
         EventMachine::HttpRequest.new('http://127.0.0.1:8080/').get({:timeout => 1})
       end
 
@@ -108,4 +126,40 @@ describe Proxy do
       end
     end
   end
+
+  it "should not invoke on_data when :relay_client is passed as server option" do
+    lambda {
+      EM.run do
+        EventMachine.add_timer(0.1) do
+          http =EventMachine::HttpRequest.new('http://127.0.0.1:8080/').get({:timeout => 1})
+          http.callback { EventMachine.stop }
+        end
+  
+        Proxy.start(:host => "0.0.0.0", :port => 8080) do |conn|
+          conn.server :goog, :host => "google.com", :port => 80, :relay_client => true
+          conn.on_data { |data| raise "Should not be here"; data }
+          conn.on_response { |backend, resp| resp }
+          
+        end
+      end
+    }.should_not raise_error
+  end
+
+  it "should not invoke on_response when :relay_server is passed as server option" do
+    lambda {
+      EM.run do
+        EventMachine.add_timer(0.1) do
+          http =EventMachine::HttpRequest.new('http://127.0.0.1:8080/').get({:timeout => 1})
+          http.callback { EventMachine.stop }
+        end
+  
+        Proxy.start(:host => "0.0.0.0", :port => 8080) do |conn|
+          conn.server :goog, :host => "google.com", :port => 80, :relay_server => true
+          conn.on_data { |data| data }
+          conn.on_response { |backend, resp| raise "Should not be here"; }
+          
+        end
+      end
+    }.should_not raise_error
+  end
 end

metadata

@@ -1,7 +1,12 @@
 --- !ruby/object:Gem::Specification 
 name: em-proxy
 version: !ruby/object:Gem::Version 
-  version: 0.1.1
+  prerelease: false
+  segments: 
+  - 0
+  - 1
+  - 2
+  version: 0.1.2
 platform: ruby
 authors: 
 - Ilya Grigorik
@@ -9,19 +14,23 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-10-25 00:00:00 -04:00
+date: 2010-03-26 00:00:00 -04:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: eventmachine
-  type: :runtime
-  version_requirement: 
-  version_requirements: !ruby/object:Gem::Requirement 
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
     requirements: 
     - - ">="
       - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        - 12
+        - 9
         version: 0.12.9
-    version: 
+  type: :runtime
+  version_requirements: *id001
 description: EventMachine Proxy DSL
 email: ilya@igvita.com
 executables: []
@@ -32,12 +41,15 @@ extra_rdoc_files:
 - README.rdoc
 files: 
 - README.rdoc
+- Rakefile
 - VERSION
 - examples/appserver.rb
 - examples/beanstalkd_interceptor.rb
 - examples/duplex.rb
 - examples/line_interceptor.rb
 - examples/port_forward.rb
+- examples/relay_port_forward.rb
+- examples/schemaless-mysql/mysql_interceptor.rb
 - examples/selective_forward.rb
 - examples/smtp_spam_filter.rb
 - examples/smtp_whitelist.rb
@@ -60,18 +72,20 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      segments: 
+      - 0
       version: "0"
-  version: 
 required_rubygems_version: !ruby/object:Gem::Requirement 
   requirements: 
   - - ">="
     - !ruby/object:Gem::Version 
+      segments: 
+      - 0
       version: "0"
-  version: 
 requirements: []
 
 rubyforge_project: em-proxy
-rubygems_version: 1.3.5
+rubygems_version: 1.3.6
 signing_key: 
 specification_version: 3
 summary: EventMachine Proxy DSL
@@ -83,6 +97,9 @@ test_files:
 - examples/duplex.rb
 - examples/line_interceptor.rb
 - examples/port_forward.rb
+- examples/relay_port_forward.rb
+- examples/schemaless-mysql/client.rb
+- examples/schemaless-mysql/mysql_interceptor.rb
 - examples/selective_forward.rb
 - examples/smtp_spam_filter.rb
 - examples/smtp_whitelist.rb