proxi 0.1 → 1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 125abf0d6cecec0a987df416390f7d903f896ea2
-  data.tar.gz: 524c9c84a6494d9838b448fa1723d0e7b838ebae
+  metadata.gz: 32e73fc0b66c9d84b526d7b819498c971141804b
+  data.tar.gz: 2708a278216649879f072473ebd6e16375ad0d5e
 SHA512:
-  metadata.gz: dd8bf72dfb3dabd7088fbb5fc73d8cb02d6be8013659fda142e9f9ea439a5f4b876b434d22676c0b7616630bba6887768342720b602c0ed7b452647b4bb0acbd
-  data.tar.gz: 4b10253015c9ab46428ae0494fa5a2d4188e5f5087e8f40e8d45355fdf26709d705c78f082b749055948f6f0c8dc31a2ae6816f934e0f75f1076fa274fc46f75
+  metadata.gz: f7ed6e53d8984ef3127443daf857d68aedc5a70f05c8548be9d3d7fdb58504c84ae37f4d73b9934e72aa3dd8d3cf79ebb8d2ad3f2e93abfeb2f3a63a2aef93e3
+  data.tar.gz: 0bf15ffa0343656882bf45df6c69e7b66fcf346d72db149c60ed0669e132a5bdd88bdf4a6958e1d6281ce4b8d10094d43485e27bec6f415bcf41206021b44809

data/.gitignore

@@ -1 +1,4 @@
-scratch.rb
+scratch.rb
+.yardoc
+proxi-0.1.gem
+doc

data/README.md

@@ -6,6 +6,7 @@ This is intended as a development tool, for when you want to see exactly what
 goes over the wire, want to log or save specific requests, responses, simulate
 timeouts or slow connections, etc.
 
+For documentation see the [annotated source code](http://plexus.github.io/proxi/).
 
 ## License
 

data/generate_docs

@@ -0,0 +1,7 @@
+cat lib/proxi.rb lib/proxi/server.rb lib/proxi/connection.rb lib/proxi/socket_factory.rb lib/proxi/reporting.rb lib/proxi/listeners.rb > doc/proxi
+
+# Use https://github.com/Billiam/rocco , seems to at least mostly still work
+
+RUBYOPT=-I~/github/rocco/lib ~/github/rocco/bin/rocco --docblocks doc/proxi
+
+firefox-dev doc/proxi.html

data/lib/proxi.rb

@@ -1,13 +1,86 @@
 require 'socket'
 require 'thread'
-require 'optparse'
-require 'tmpdir'
 require 'openssl'
 
 require 'wisper'
 
+# [Proxi](https://github.com/plexus/proxi) gives you flexible TCP/HTTP proxy
+# servers for use during development.
+#
+# This can be useful when developing against external services, to see what goes
+# over the wire, capture responses, or simulate timeouts.
+#
+# There are three main concepts in Proxi, `Server`, `Connection`, and `Socket`.
+# A server listens locally for incoming requests. When it receives a request, it
+# establishes a `Connection`, which opens a socket to the remote host, and then
+# acts as a bidirectional pipe between the incoming and outgoing network
+# sockets.
+#
+# To allow for multiple ways to handle the proxying, and multiple strategies for
+# making remote connections, the `Server` does not create `Connections`
+# directly, but instead delegates this to a "connection factory".
+#
+# A `Connection` in turn delegates how to open a remote socket to a "socket
+# factory".
+#
+# Both Servers and Connections are observable, they emit events that objects can
+# subscribe to.
+#
+# To use Proxi, hook up these factories, and register event listeners, and then
+# start the server.
 module Proxi
+
+  # ## Basic examples
+  #
+  # These are provided for basic use cases, and as a starting point for more
+  # complex uses. They return the server instance, call `#call` to start the
+  # server, or use `on` or `subscribe` to listen to events.
+
+  # With `Proxy.tcp_proxy` you get basic proxying from a local port to a remote
+  # host and port, all bytes are simply forwarded without caring about their
+  # contents.
+  #
+  # For example:
+  #
+  #     Proxi.tcp_proxy(3000, 'foo.example.com', 4000).call
+  def self.tcp_proxy(local_port, remote_host, remote_port)
+    reporter = ConsoleReporter.new
+
+    connection_factory = -> in_socket do
+      socket_factory = TCPSocketFactory.new(remote_host, remote_port)
+      Connection.new(in_socket, socket_factory).subscribe(reporter)
+    end
+
+    Server.new(local_port, connection_factory)
+  end
+
+  # `Proxi.http_host_proxy` allows proxying to multiple remote hosts, based on
+  # the HTTP `Host:` header. To use it, gather the IP addresses that correspond
+  # to each domain name, and provide this name-to-ip mapping to
+  # `http_host_proxy`. Now configure these domain names in `/etc/hosts` to point
+  # to the local host, so Proxi can intercept traffic intended for these
+  # domains.
+  #
+  # For example
+  #
+  #     Proxi.http_host_proxy(80, {'foo.example.org' => '10.10.0.1'}).call
+  def self.http_host_proxy(local_port, host_mapping)
+    reporter = ConsoleReporter.new
+
+    connection_factory = -> in_socket do
+      socket_factory = HTTPHostSocketFactory.new(host_mapping)
+      Connection
+        .new(in_socket, socket_factory)
+        .subscribe(socket_factory, on: :data_in)
+        .subscribe(reporter)
+    end
+
+    Server.new(local_port, connection_factory)
+  end
 end
 
 require_relative 'proxi/server'
 require_relative 'proxi/connection'
+require_relative 'proxi/socket_factory'
+require_relative 'proxi/reporting'
+require_relative 'proxi/listeners'

data/lib/proxi/connection.rb

@@ -1,18 +1,45 @@
+# ## Proxi::Connection
+#
 module Proxi
+
+  # A `Connection` is a bidirectional pipe between two sockets.
+  #
+  # The proxy server hands it the socket for the incoming request from, and
+  # `Connection` then initiates an outgoing request, after which it forwards all
+  # traffic in both directions.
+  #
+  # Creating the outgoing request is delegated to a `Proxi::SocketFactory`. The
+  # reason being that the type of socket can vary (`TCPSocket`, `SSLSocket`), or
+  # there might be some logic involved to dispatch to the correct host, e.g.
+  # based on the HTTP Host header (cfr. `Proxi::HTTPHostSocketFactory`).
+  #
+  # A socket factory can subscribe to events to make informed decision, e.g. to
+  # inspect incoming data for HTTP headers.
+  #
+  # Proxi::Connection broadcasts the following events:
+  #
+  # - `start_connection(Proxi::Connection)`
+  # - `end_connection(Proxi::Connection)`
+  # - `main_loop_error(Proxi::Connection, Exception)`
+  # - `data_in(Proxi::Connection, Array)`
+  # - `data_out(Proxi::Connection, Array)`
   class Connection
     include Wisper::Publisher
 
-    attr_reader :in_socket, :thread, :remote_host, :remote_port
+    attr_reader :in_socket, :thread
 
-    def initialize(in_socket, remote_host, remote_port)
+    def initialize(in_socket, socket_factory, max_block_size: 4096)
       @in_socket = in_socket
-      @remote_host, @remote_port = remote_host, remote_port
-      @state = :new
+      @socket_factory = socket_factory
+      @max_block_size = max_block_size
     end
 
+    # `Connection#call` starts the connection handler thread. This is called by
+    # the server, and spawns a new Thread that handles the forwarding of data.
     def call
       broadcast(:start_connection, self)
       @thread = Thread.new { proxy_loop }
+      self
     end
 
     def alive?
@@ -26,11 +53,12 @@ module Proxi
     private
 
     def out_socket
-      @out_socket ||= TCPSocket.new(remote_host, remote_port)
+      @out_socket ||= @socket_factory.call
     end
 
     def proxy_loop
       begin
+        handle_socket(in_socket)
         loop do
           begin
             ready_sockets.each do |socket|
@@ -55,7 +83,7 @@ module Proxi
     end
 
     def handle_socket(socket)
-      data = socket.readpartial(4096)
+      data = socket.readpartial(@max_block_size)
 
       if socket == in_socket
         broadcast(:data_in, self, data)
@@ -69,67 +97,4 @@ module Proxi
     end
   end
 
-  module SSLConnection
-    def connect
-      @out_socket = OpenSSL::SSL::SSLSocket.new(super)
-      @out_socket.connect
-    end
-  end
-
-  class HTTPConnection < Connection
-    def initialize(in_socket, host_to_ip)
-      @in_socket, @host_to_ip = in_socket, host_to_ip
-      @new = true
-    end
-
-    def handle_socket(socket)
-      data = socket.readpartial(4096)
-
-      if socket == in_socket
-        broadcast(:data_in, self, data)
-
-        if @new
-          @first_packet = data
-          @new = false
-        end
-
-        out_socket.write data
-        out_socket.flush
-      else
-        broadcast(:data_out, self, data)
-        in_socket.write data
-        in_socket.flush
-      end
-    end
-
-    def ready_sockets
-      if @new
-        sockets = [in_socket]
-      else
-        sockets = [in_socket, out_socket]
-      end
-      IO.select(sockets).first
-    end
-
-    def out_socket
-      host, port = @host_to_ip.fetch(headers["host"]).split(':')
-      port ||= 80
-      @out_socket ||= TCPSocket.new(host, port.to_i)
-    end
-
-    def headers
-      Hash[
-        @first_packet
-        .sub(/\r\n\r\n.*/m, '')
-        .each_line
-        .drop(1) # GET / HTTP/1.1
-        .map do |line|
-          k,v = line.split(':', 2)
-          [k.downcase, v.strip].tap do |header|
-            broadcast(:header, self, *header)
-          end
-        end
-      ]
-    end
-  end
 end

data/lib/proxi/listeners.rb

@@ -0,0 +1,50 @@
+# ## Server Listeners
+#
+# These can be attached to a server to add extra behavior
+#
+module Proxi
+
+  # Log all incoming and outgoing traffic to files under `/tmp`
+  #
+  # For example:
+  #
+  #      Proxi.tcp_server(...).subscribe(RequestResponseLogging.new).call
+  #
+  # The in and outgoing traffic will be captured per connection in
+  # `/tmp/proxi.1.in` and `/tmp/proxi.1.out`; `/tmp/proxi.2.in`, etc.
+  class RequestResponseLogging
+    def initialize(dir: Dir.tmpdir, name: "proxi")
+      @dir = dir
+      @name = name
+      @count = 0
+      @mutex = Mutex.new
+    end
+
+    def new_connection(connection)
+      count = @mutex.synchronize { @count += 1 }
+      in_fd = File.open(log_name(count, "in"), 'w')
+      out_fd = File.open(log_name(count, "out"), 'w')
+
+      connection
+        .on(:data_in) { |_, data| in_fd.write(data) ; in_fd.flush }
+        .on(:data_out) { |_, data| out_fd.write(data) ; out_fd.flush }
+        .on(:end_connection) { in_fd.close ; out_fd.close }
+    end
+
+    def log_name(num, suffix)
+      '%s/%s.%d.%s' % [@dir, @name, num, suffix]
+    end
+  end
+
+  # Wait before handing back data coming from the remote, this simulates a slow
+  # connection, and can be used to test timeouts.
+  class SlowDown
+    def initialize(wait_seconds: 5)
+      @wait_seconds = wait_seconds
+    end
+
+    def new_connection(connection)
+      connection.on(:data_out) { sleep @wait_seconds }
+    end
+  end
+end

data/lib/proxi/reporting.rb

@@ -0,0 +1,57 @@
+# ### Reporting
+#
+# Proxi's server and connection classes don't have any logging or UI
+# capabilities built in, but they broadcast events that we can listen to to perform these tasks.
+
+module Proxi
+  # This is a very basic console reporter to see what's happening
+  #
+  # Subscribe to connection events, and you will see output that looks like this
+  #
+  #     1. +++
+  #     1. < 91
+  #     2. +++
+  #     3. +++
+  #     2. < 91
+  #     3. < 91
+  #     1. > 4096
+  #     1. > 3422
+  #     1. ---
+  #
+  # Each connection gets a unique incremental number assigned, followed by:
+  #
+  # - '+++' new connection
+  # - '---' connection closed
+  # - '< 1234' number of bytes proxied to the remote
+  # - '> 1234' number of bytes proxied back from the remote
+  class ConsoleReporter
+    def initialize
+      @count = 0
+      @mutex = Mutex.new
+      @connections = {}
+    end
+
+    def start_connection(conn)
+      @mutex.synchronize { @connections[conn] = (@count += 1) }
+      puts "#{@connections[conn]}. +++"
+    end
+
+    def end_connection(conn)
+      puts "#{@connections[conn]}. ---"
+      @connections.delete(conn)
+    end
+
+    def data_in(conn, data)
+      puts "#{@connections[conn]}. < #{data.size}"
+    end
+
+    def data_out(conn, data)
+      puts "#{@connections[conn]}. > #{data.size}"
+    end
+
+    def main_loop_error(conn, exc)
+      STDERR.puts "#{@connections[conn]}. #{exc.class} #{exc.message}"
+      STDERR.puts exc.backtrace
+    end
+  end
+end

data/lib/proxi/server.rb

@@ -1,19 +1,45 @@
+# ## Proxi::Server
+#
 module Proxi
+
+  # `Proxi::Server` accepts TCP requests, and forwards them, by creating an
+  # outbound connection and forwarding traffic in both directions.
+  #
+  # The destination of the outbound connection, and the forwarding of data, is
+  # handled by a `Proxi::Connection`, created by a factory object, which can be a
+  # lambda.
+  #
+  # Start listening for connections by calling #call.
+  #
+  # `Proxi::Server` broadcasts the following events:
+  #
+  # - `new_connection(Proxi::Connection)`
+  # - `dead_connection(Proxi::Connection)`
   class Server
     include Wisper::Publisher
 
-    MAX_CONNECTIONS = 5
-
-    def initialize(listen_port, connection_factory)
+    # Public: Initialize a Server
+    #
+    # listen_port        - The String or Integer of the port to listen to for
+    #                      incoming connections
+    # connection_factory - Implements #call(in_socket) and returns a
+    #                      Proxi::Connection
+    # max_connections    - The maximum amount of parallel connections to handle
+    #                      at once
+    def initialize(listen_port, connection_factory, max_connections: 5)
+      @listen_port = listen_port
       @connection_factory = connection_factory
-
-      @server = TCPServer.new(nil, listen_port)
-
+      @max_connections = 5
       @connections = []
     end
 
+    # Public: Start the server
+    #
+    # Start accepting and forwarding requests
     def call
-      loop do
+      @server = TCPServer.new('localhost', @listen_port)
+
+      until @server.closed?
         in_socket = @server.accept
         connection = @connection_factory.call(in_socket)
 
@@ -21,22 +47,36 @@ module Proxi
 
         @connections.push(connection)
 
-        connection.call
+        connection.call # spawns a new thread that handles proxying
 
         reap_connections
-        while @connections.size >= MAX_CONNECTIONS
+        while @connections.size >= @max_connections
           sleep 1
           reap_connections
         end
       end
+    ensure
+      close unless @server.closed?
+    end
+
+    # Public: close the TCP server socket
+    #
+    # Included for completeness, note that if the proxy server is active it will
+    # likely be blocking on TCPServer#accept, and the server port will stay open
+    # until it has accepted one final request.
+    def close
+      @server.close
     end
 
+    private
+
     def reap_connections
-      @connections = @connections.select do |t|
-        if t.alive?
+      @connections = @connections.select do |conn|
+        if conn.alive?
           true
         else
-          t.join_thread
+          broadcast(:dead_connection, conn)
+          conn.join_thread
           false
         end
       end

data/lib/proxi/socket_factory.rb

@@ -0,0 +1,92 @@
+# ## Socket factories
+#
+module Proxi
+  # ### TCPSocketFactory
+  #
+  # This is the most vanilla type of socket factory.
+  #
+  # Suitable when all requests need to be forwarded to the same host and port.
+  class TCPSocketFactory
+    def initialize(remote_host, remote_port)
+      @remote_host, @remote_port = remote_host, remote_port
+    end
+
+    def call
+      TCPSocket.new(@remote_host, @remote_port)
+    end
+  end
+
+  # ### SSLSocketFactory
+  #
+  # This will set up an encrypted (SSL, https) connection to the target host.
+  # This way the proxy server communicates *unencrypted* locally, but
+  # encrypts/decrypts communication with the remote host.
+  #
+  # If you want to forward SSL connections as-is, use a `TCPSocketFactory`, in
+  # that case however you won't be able to inspect any data passing through,
+  # since it will be encrypted.
+  class SSLSocketFactory < TCPSocketFactory
+    def call
+      OpenSSL::SSL::SSLSocket.new(super).tap(&:connect)
+    end
+  end
+
+  # ### HTTPHostSocketFactory
+  #
+  # Dispatches HTTP traffic to multiple hosts, based on the HTTP `Host:` header.
+  #
+  # HTTPHostSocketFactory expects to receive data events from the connection, so
+  # make sure you subscribe it to connection events. (see `Proxi.http_proxy` for
+  # an example).
+  #
+  # To use this effectively, configure your local `/etc/hosts` so the relevant
+  # domains point to localhost. That way the proxy will be able to intercept
+  # them.
+  #
+  # This class is single use only! Create a new instance for each `Proxi::Connection`.
+  class HTTPHostSocketFactory
+
+    # Initialize a HTTPHostSocketFactory
+    #
+    # `host_mapping` - A Hash mapping hostnames to IP addresses, and, optionally, ports
+    #
+    # For example:
+    #
+    #     HTTPHostSocketFactory.new(
+    #       'foo.example.com' => '10.10.10.1:8080',
+    #       'bar.example.com' => '10.10.10.2:8080'
+    #     )
+    def initialize(host_mapping)
+      @host_mapping = host_mapping
+    end
+
+    # This is an event listener, it will be broadcast by the `Connection` whenever
+    # it gets new request data. We capture the first packet, assuming it
+    # contains the HTTP headers.
+    #
+    # `Connection` will only request an outgoing socket from us (call `#call`)
+    # after it received the initial request payload.
+    def data_in(connection, data)
+      @first_packet ||= data
+    end
+
+    def call
+      host, port = @host_to_ip.fetch(headers["host"]).split(':')
+      port ||= 80
+      TCPSocket.new(host, port.to_i)
+    end
+
+    def headers
+      Hash[
+        @first_packet
+        .sub(/\r\n\r\n.*/m, '')
+        .each_line
+        .drop(1) # GET / HTTP/1.1
+        .map do |line|
+          k,v = line.split(':', 2)
+          [k.downcase, v.strip]
+        end
+      ]
+    end
+  end
+end

data/proxi.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |gem|
   gem.name        = 'proxi'
-  gem.version     = '0.1'
+  gem.version     = '1.0'
   gem.authors     = [ 'Arne Brasseur' ]
   gem.email       = [ 'arne@arnebrasseur.net' ]
   gem.description = 'TCP and HTTP proxy scripts'

data/proxy.rb

@@ -1,5 +1,12 @@
 #!/usr/bin/ruby
 
+# This is my original proxy.rb script that served me well for many years, before
+# I decided to clean it up and turn it into a gem.
+#
+# Have a look under lib/ for the much improved version. Leaving it here just a
+# little longer for reference. There are still a few features that haven't been
+# ported yet.
+
 require 'socket'
 require 'thread'
 require 'optparse'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: proxi
 version: !ruby/object:Gem::Version
-  version: '0.1'
+  version: '1.0'
 platform: ruby
 authors:
 - Arne Brasseur
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-10-24 00:00:00.000000000 Z
+date: 2015-10-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: wisper
@@ -51,9 +51,13 @@ files:
 - Gemfile.lock
 - LICENSE
 - README.md
+- generate_docs
 - lib/proxi.rb
 - lib/proxi/connection.rb
+- lib/proxi/listeners.rb
+- lib/proxi/reporting.rb
 - lib/proxi/server.rb
+- lib/proxi/socket_factory.rb
 - proxi.gemspec
 - proxy.rb
 homepage: https://github.com/plexus/proxi