http_spew 0.1.0

data/ChangeLog

@@ -0,0 +1,170 @@
+ChangeLog from http://bogomips.org/http_spew.git
+
+   commit 32223af63dc5296cd09f2472ad60263d98f379a2
+   Author: Eric Wong <normalperson@yhbt.net>
+   Date:   Thu Feb 24 08:28:34 2011 +0000
+   
+       HTTP spew 0.1.0 - initial release
+       
+       Might as well...
+   
+   commit e11489549eefff5ae031f2127bc72efe8187f938
+   Author: Eric Wong <normalperson@yhbt.net>
+   Date:   Thu Feb 24 08:48:14 2011 +0000
+   
+       fixes for Ruby 1.9.3dev
+       
+       1.9.3dev got louder about warnings and more careful about
+       shared strings.
+   
+   commit aefc248decd16db8098fef6148ed825e83bebfc1
+   Author: Eric Wong <normalperson@yhbt.net>
+   Date:   Thu Feb 10 13:48:15 2011 -0800
+   
+       test/helper: cleanup fifo after using it
+       
+       No need to flood our $TMPDIR :x
+   
+   commit 5d23e362ee382d57672bf4f21563c94d3b0bca03
+   Author: Eric Wong <normalperson@yhbt.net>
+   Date:   Thu Feb 10 10:44:21 2011 +0000
+   
+       hit_n_run: an even more terrible HTTP requester
+       
+       It just requests and never cares for a response!
+   
+   commit cf5b6c3b0664471f3fbd384c50d940d2f6c2781a
+   Author: Eric Wong <normalperson@yhbt.net>
+   Date:   Thu Feb 10 10:25:39 2011 +0000
+   
+       add Content-MD5 input filter support
+       
+       This means we can make requests with Content-MD5 trailers
+   
+   commit 20f5a767769aabee3b26ae05c9d8c9532fa84b16
+   Author: Eric Wong <normalperson@yhbt.net>
+   Date:   Thu Feb 10 10:23:35 2011 +0000
+   
+       chunky_pipe: error transfer between threads
+       
+       Calling IO#close is racy with threads, so we transfer
+       the error over to the reader if the writer has to error
+       out.
+   
+   commit 9f97b6012ab0e777adfc342474bb95b5a55f9643
+   Author: Eric Wong <normalperson@yhbt.net>
+   Date:   Thu Feb 10 10:21:27 2011 +0000
+   
+       wait: do not poll if pollset is empty
+       
+       No point :P
+   
+   commit a9f76e7e7d78d4d2313819e43c486ecd33cc998a
+   Author: Eric Wong <normalperson@yhbt.net>
+   Date:   Thu Feb 10 10:21:00 2011 +0000
+   
+       do not clobber existing error when assigning
+   
+   commit 711b3503dbcbe08850595796824b84eaf32ba4c4
+   Author: Eric Wong <normalperson@yhbt.net>
+   Date:   Thu Feb 10 08:54:29 2011 +0000
+   
+       split out chunky_pipe into it's own module
+       
+       It should be top-level and it's a good name, too :D
+   
+   commit bbb83625d3197d321c4fe4526e0fcdb80ddcc1b9
+   Author: Eric Wong <normalperson@yhbt.net>
+   Date:   Thu Feb 10 08:37:43 2011 +0000
+   
+       input_splitter -> input_spray, add tests
+       
+       It seems to work alright in a unit test environment
+   
+   commit fb0d48b2ccfced833f879e592d43700d40473c77
+   Author: Eric Wong <normalperson@yhbt.net>
+   Date:   Tue Feb 8 19:46:31 2011 -0800
+   
+       add untested input_splitter
+       
+       It'll be useful for splitting out inputs into different
+       requests and processing them in parallel.
+   
+   commit 89e2b2b90136525d7e7905b3e1fa0e711398f4b3
+   Author: Eric Wong <normalperson@yhbt.net>
+   Date:   Tue Feb 8 18:59:32 2011 -0800
+   
+       cleanup request handling of input
+       
+       Remove unneeded Fiber dependency.
+   
+   commit 9421276c001adbd8904366e92fe6181ea925a2db
+   Author: Eric Wong <normalperson@yhbt.net>
+   Date:   Tue Feb 8 18:59:01 2011 -0800
+   
+       test/helper: properly handle worker_processes != 4
+       
+       Oops, we're about to start using it
+   
+   commit 8a5b2fac7f448c35507cb081178ad9027843b335
+   Author: Eric Wong <normalperson@yhbt.net>
+   Date:   Tue Feb 8 18:16:01 2011 -0800
+   
+       HTTP_Spew.wait retries requests on EINTR
+       
+       We may not have rv set yet.
+   
+   commit 4809a6b7a97a0087899107820a93d70283c9502a
+   Author: Eric Wong <normalperson@yhbt.net>
+   Date:   Tue Feb 8 18:13:40 2011 -0800
+   
+       add wait_nonblock! and fix wait return values
+       
+       Like Process.waitall, we want to wait on and return
+       all running requests possible.
+   
+   commit a898603166528d6eaebcf151008ccd99b01ea115
+   Author: Eric Wong <normalperson@yhbt.net>
+   Date:   Tue Feb 8 18:13:17 2011 -0800
+   
+       request: document close method
+       
+       We don't want to think it's unused.
+   
+   commit 27696c3947f2339a8f708377ba5618044207340a
+   Author: Eric Wong <normalperson@yhbt.net>
+   Date:   Tue Feb 8 18:12:07 2011 -0800
+   
+       request: optimize string requests by avoiding Fiber
+       
+       No need for Fiber in some cases...
+   
+   commit 10283cdff905c4af58e633ef052d3823deba6db5
+   Author: Eric Wong <normalperson@yhbt.net>
+   Date:   Tue Feb 8 18:11:23 2011 -0800
+   
+       test_upload: switch this test to use a temporary file
+       
+       We don't want to use too much memory
+   
+   commit 5312aeb04ae29a6187f39cc655356f1b42402fe0
+   Author: Eric Wong <normalperson@yhbt.net>
+   Date:   Tue Feb 8 18:10:14 2011 -0800
+   
+       test/helper: allow worker_processes to be specified
+       
+       We want to disable parallelization sometimes
+   
+   commit 3a2e1ac5d3679bfa44056a67118034da50c948d1
+   Author: Eric Wong <normalperson@yhbt.net>
+   Date:   Tue Feb 8 15:03:28 2011 -0800
+   
+       switch from IO.select to to Kgio.poll
+       
+       No high descriptor limits anymore
+   
+   commit c25326ff75c1e7da0c8e366ce46c84aea6ad094b
+   Author: Eric Wong <normalperson@yhbt.net>
+   Date:   Mon Feb 7 08:51:15 2011 +0000
+   
+       initial commit

data/GIT-VERSION-FILE

@@ -0,0 +1 @@
+GIT_VERSION = 0.1.0

data/GIT-VERSION-GEN

@@ -0,0 +1,40 @@
+#!/bin/sh
+
+GVF=GIT-VERSION-FILE
+DEF_VER=v0.1.0.GIT
+
+LF='
+'
+
+# First see if there is a version file (included in release tarballs),
+# then try git-describe, then default.
+if test -f version
+then
+	VN=$(cat version) || VN="$DEF_VER"
+elif test -d .git -o -f .git &&
+	VN=$(git describe --abbrev=4 HEAD 2>/dev/null) &&
+	case "$VN" in
+	*$LF*) (exit 1) ;;
+	v[0-9]*)
+		git update-index -q --refresh
+		test -z "$(git diff-index --name-only HEAD --)" ||
+		VN="$VN-dirty" ;;
+	esac
+then
+	VN=$(echo "$VN" | sed -e 's/-/./g');
+else
+	VN="$DEF_VER"
+fi
+
+VN=$(expr "$VN" : v*'\(.*\)')
+
+if test -r $GVF
+then
+	VC=$(sed -e 's/^GIT_VERSION = //' <$GVF)
+else
+	VC=unset
+fi
+test "$VN" = "$VC" || {
+	echo >&2 "GIT_VERSION = $VN"
+	echo "GIT_VERSION = $VN" >$GVF
+}

data/GNUmakefile

@@ -0,0 +1,5 @@
+all::
+RSYNC_DEST := bogomips.org:/srv/bogomips/http_spew
+rfproject := rainbows
+rfpackage := http_spew
+include pkg.mk

data/LATEST

@@ -0,0 +1,4 @@
+=== HTTP spew 0.1.0 - initial release / 2011-02-24 08:50 UTC
+
+  Might as well...
+

data/LICENSE

@@ -0,0 +1,17 @@
+HTTP Spew is copyrighted Free Software by all contributors, see logs in
+revision control for names and email addresses of all of them.
+
+You can redistribute it and/or modify it under the terms of the GNU
+General Public License, version 2 *or* 3 ({GPLv3}[link:COPYING]) as
+published by the Free Software Foundation.  The project leader
+(Eric Wong) reserves the right to relicense HTTP Spew under future versions
+of the GPL.
+
+HTTP Spew is distributed in the hope that it will be useful, but WITHOUT
+ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public
+License for more details.
+
+You should have received a copy of the GNU General Public License
+along with HTTP Spew; if not, write to the Free Software Foundation, Inc.,
+51 Franklin Street, Fifth Floor, Boston, MA 02110-1301

data/NEWS

@@ -0,0 +1,4 @@
+=== HTTP spew 0.1.0 - initial release / 2011-02-24 08:50 UTC
+
+  Might as well...
+

data/README

@@ -0,0 +1,61 @@
+= HTTP Spew - LAN-only HTTP spam^H^H^H^Hclient for Ruby
+
+Do not use HTTP Spew for connecting to servers outside of your LAN.
+Do not use HTTP Spew without the permission of your server admins.
+Use HTTP Spew if you wish you could kinda multicast with HTTP...
+
+== Problems
+
+* No support for bidirectional HTTP streaming
+* No support for "Expect: 100-continue" headers/responses
+* No support for DNS resolution (WONTFIX, ever)
+* No support for HTTPS
+* No support for keepalive (yet?)
+* No support for Ruby 1.8, this is Ruby 1.9-only
+* Not remotely RFC-compliant
+* Messes up analytics/reporting on servers
+* Resets server connections
+* May get you banned from the Internet!
+
+== Still Interested?
+
+HTTP Spew lets you fire off identical (or similar) requests to multiple
+HTTP servers and wait for any number (or all) of them to complete or for
+a timeout to expire.
+
+HTTP Spew may be useful for implementing reverse proxy servers if you
+1) can't decide on a load balancing strategy for them
+2) have much more hardware than you actually use
+
+It's also completely untested and unused anywhere!
+
+== Hacking
+
+You can get the latest source via git from the following locations:
+
+  git://bogomips.org/http_spew.git
+  git://repo.or.cz/http_spew.git (mirror)
+
+You may browse the code from the web and download the latest snapshot
+tarballs here:
+
+* http://bogomips.org/http_spew.git (cgit)
+* http://repo.or.cz/w/http_spew.git (gitweb)
+
+Inline patches (from "git format-patch") to the mailing list are
+preferred because they allow code review and comments in the reply to
+the patch.
+
+We will adhere to mostly the same conventions for patch submissions as
+git itself.  See the Documentation/SubmittingPatches document
+distributed with git on on patch submission guidelines to follow.  Just
+don't email the git mailing list or maintainer with http_spew patches.
+
+== Contact
+
+All feedback (bug reports, user/development discussion, patches, pull
+requests) go to the mailing list: mailto:http.spew@librelist.org
+
+Mailing list archives in mbox format may be downloaded here:
+
+http://bogomips.org/http_spew/archives/

data/http_spew.gemspec

@@ -0,0 +1,24 @@
+ENV["VERSION"] or abort "VERSION= must be specified"
+manifest = File.readlines('.manifest').map! { |x| x.chomp! }
+require 'wrongdoc'
+extend Wrongdoc::Gemspec
+name, summary, title = readme_metadata
+
+Gem::Specification.new do |s|
+  s.name = %q{http_spew}
+  s.version = ENV["VERSION"].dup
+  s.authors = ["HTTP Spew hackers"]
+  s.date = Time.now.utc.strftime('%Y-%m-%d')
+  s.description = readme_description
+  s.email = %q{http.spew@librelist.org}
+  s.extra_rdoc_files = extra_rdoc_files(manifest)
+  s.files = manifest
+  s.homepage = Wrongdoc.config[:rdoc_url]
+  s.summary = summary
+  s.rdoc_options = rdoc_options
+  s.rubyforge_project = %q{rainbows}
+  s.test_files = Dir["test/test_*.rb"]
+  s.add_dependency(%q<kcar>, "~> 0.1.2")
+  s.add_dependency(%q<kgio>, "~> 2.3")
+  s.add_development_dependency(%q<wrongdoc>, "~> 1.5")
+end

data/lib/http_spew/chunky_pipe.rb

@@ -0,0 +1,12 @@
+# -*- encoding: binary -*-
+#
+# HTTP Spew blows chunks (at HTTP servers)!
+class HTTP_Spew::ChunkyPipe < Kgio::Pipe
+  attr_accessor :error
+
+  # makes read behave like readpartial without EOFError
+  def read(*args)
+    defined?(@error) and raise @error
+    kgio_read(*args)
+  end
+end

data/lib/http_spew/content_md5.rb

@@ -0,0 +1,55 @@
+# -*- encoding: binary -*-
+require "digest/md5"
+module HTTP_Spew::ContentMD5
+  class MismatchError < HTTP_Spew::Error
+  end
+  class LengthError < HTTP_Spew::Error
+  end
+
+  def self.input(env)
+    if trailer = env["HTTP_TRAILER"]
+      have_md5 = trailer.split(/\s*,\s*/).grep(/\AContent-MD5\z/i)[0]
+      return env["rack.input"] if have_md5 && env["HTTP_CONTENT_MD5"]
+    end
+    if trailer
+      unless have_md5
+        trailer << (trailer.empty? ? "Content-MD5" : ",Content-MD5")
+      end
+    else
+      env["HTTP_TRAILER"] = "Content-MD5"
+    end
+    env["HTTP_TRANSFER_ENCODING"] = "chunked"
+    rd, wr = HTTP_Spew::ChunkyPipe.new
+    md5 = env.delete("HTTP_CONTENT_MD5")
+    len = env.delete("CONTENT_LENGTH")
+    start_write_driver(env["rack.input"], rd, wr, md5, len)
+    rd
+  end
+
+  def self.start_write_driver(input, rd, wr, expect_md5, expect_len)
+    Thread.new do
+      begin
+        digest, buf, bytes = Digest::MD5.new, "", 0
+        while input.read(0x4000, buf)
+          n = buf.size
+          bytes += n
+          wr.kgio_write("#{n.to_s(16)}\r\n")
+          digest.update(buf)
+          wr.kgio_write(buf << "\r\n")
+        end
+        if expect_len && expect_len.to_i != bytes
+          raise LengthError, "expect=#{expect_len} != got=#{bytes}"
+        end
+        digest = [ digest.digest ].pack("m").strip!
+        if expect_md5 && expect_md5.strip != digest
+          raise MismatchError, "expect=#{expect_md5} != got=#{digest}"
+        end
+        wr.write "0\r\nContent-MD5: #{digest}\r\n\r\n"
+      rescue => e
+        rd.error = e
+      ensure
+        wr.close
+      end
+    end
+  end
+end

data/lib/http_spew/headers.rb

@@ -0,0 +1,43 @@
+# -*- encoding: binary -*-
+module HTTP_Spew::Headers
+  # :stopdoc:
+  TR = %w(- _)
+  REQUEST_METHOD = "REQUEST_METHOD"
+  REQUEST_URI = "REQUEST_URI"
+  CRLF = "\r\n"
+  QUERY_STRING = "QUERY_STRING"
+  PATH_INFO = "PATH_INFO"
+  CONTENT_TYPE = "CONTENT_TYPE" # specified by Rack to be !/^HTTP_/
+  # :startdoc:
+
+  def request_uri(env)
+    qs = env[QUERY_STRING]
+    qs.size == 0 ? env[PATH_INFO] : "#{env[PATH_INFO]}?#{qs}"
+  end
+  module_function :request_uri
+
+  def env_to_headers(env, input)
+    req = "#{env[REQUEST_METHOD]} " \
+          "#{env[REQUEST_URI] || request_uri(env)} HTTP/1.1\r\n" \
+          "Connection: close\r\n"
+    uscore, dash = *TR
+    env.each do |key,value|
+      %r{\AHTTP_(\w+)\z} =~ key or next
+      key = $1
+      %r{\A(?:VERSION|EXPECT|TRANSFER_ENCODING|CONNECTION|KEEP_ALIVE)\z}x =~
+        key and next
+
+      key.tr!(uscore, dash)
+      req << "#{key}: #{value}\r\n"
+    end
+    if input
+      req << (input.respond_to?(:size) ?
+             "Content-Length: #{input.size}\r\n" :
+             "Transfer-Encoding: chunked\r\n")
+      ct = env[CONTENT_TYPE] and req << "Content-Type: #{ct}\r\n"
+    end
+    req << CRLF
+    String === input ? (req << input) : [ req, input ]
+  end
+  module_function :env_to_headers
+end

data/lib/http_spew/hit_n_run.rb

@@ -0,0 +1,19 @@
+# -*- encoding: binary -*-
+
+# An even more horrible way to make HTTP requests, just make them without
+# reading a response!
+class HTTP_Spew::HitNRun < HTTP_Spew::Request
+
+  # frozen response
+  RESPONSE = [ 666,
+    {
+      "Content-Length" => "0".freeze,
+      "Content-Type" => "hit/run".freeze
+    }.freeze,
+    [].freeze ].freeze
+
+  def read_response
+    close
+    RESPONSE
+  end
+end

data/lib/http_spew/input_spray.rb

@@ -0,0 +1,53 @@
+# -*- encoding: binary -*-
+#
+# Use this to wrap and replace your input object for spraying to multiple
+# servers.
+class HTTP_Spew::InputSpray
+  attr_reader :readers
+  class NoWritersError < HTTP_Spew::Error
+  end
+
+  class SizedPipe < HTTP_Spew::ChunkyPipe
+    attr_accessor :size
+  end
+
+  def initialize(env, nr)
+    @input = env["rack.input"]
+    size = @input.respond_to?(:size) ? @input.size : env["CONTENT_LENGTH"]
+    size = size ? size.to_i : nil
+    klass = size ? SizedPipe : HTTP_Spew::ChunkyPipe
+    @readers, @writers = [], []
+    nr.times do
+      r, w = klass.new
+      r.size = size if size
+      @readers << r
+      @writers << w
+    end
+    @wr = start_write_driver
+  end
+
+  def write_fail?(wr, buf)
+    wr.kgio_write(buf)
+    false
+    rescue
+      wr.close
+      true
+  end
+
+  # TODO: splice(2) if @input is an IO
+  def start_write_driver
+    Thread.new do
+      begin
+        buf = ""
+        while buf = @input.read(0x4000, buf)
+          @writers.delete_if { |wr| write_fail?(wr, buf) }.empty? and
+            raise NoWritersError, "all writers have died"
+        end
+      rescue => e
+        @readers.each { |io| io.error = e }
+      ensure
+        @writers.each { |io| io.close unless io.closed? }
+      end
+    end
+  end
+end

data/lib/http_spew/request.rb

@@ -0,0 +1,77 @@
+# -*- encoding: binary -*-
+class HTTP_Spew::Request
+  attr_reader :to_io
+  attr_reader :error
+  attr_reader :response
+  class RequestError < HTTP_Spew::Error
+  end
+
+  include HTTP_Spew::Headers
+
+  def initialize(env, input, sock)
+    @to_io = Kgio::SocketMethods === sock ? sock : Kgio::Socket.start(sock)
+    if Hash === env
+      @buf, @input = env_to_headers(env, input)
+    else
+      @buf, @input = env, input
+    end
+  end
+
+  # returns a 3-element Rack response array on completion
+  # returns :wait_readable or :wait_writable if busy
+  def resume
+    if @buf
+      case rv = @to_io.kgio_trywrite(@buf)
+      when String
+        @buf = rv # loop retry, socket buffer could've expanded
+      when Symbol
+        return rv
+      else # done writing, read more
+        @buf = @input ? @input.read(0x4000, @buf) : nil
+      end while @buf
+      read_response
+    else
+      read_response
+    end
+  end
+
+  def read_response
+    buf = @to_io.kgio_trypeek(0x4000) or eof!
+    String === buf or return buf
+
+    # Kcar::Parser#headers shortens +buf+ for us
+    hdr_len = buf.size
+    r = Kcar::Parser.new.headers({}, buf) or too_big!
+
+    # discard the header data from the socket buffer
+    (hdr_len -= buf.size) > 0 and @to_io.kgio_read(hdr_len, buf)
+    r[2] = self
+    @response = r
+  end
+
+  def to_path
+    "/dev/fd/#{@to_io.fileno}"
+  end
+
+  def too_big!
+    raise RequestError.new(self), "response headers too large", []
+  end
+
+  def each
+    buf = ""
+    while buf = @to_io.kgio_read(0x4000, buf)
+      yield buf
+    end
+  end
+
+  def error=(exception)
+    close
+    @error = exception
+  end
+
+  # this may be called by a Rack web server
+  def close
+    @to_io.close
+    IO === @input and @input.close
+  end
+end

data/lib/http_spew.rb

@@ -0,0 +1,97 @@
+# -*- encoding: binary -*-
+require "kgio"
+require "kcar"
+
+module HTTP_Spew
+  autoload :ChunkyPipe, "http_spew/chunky_pipe"
+  autoload :ContentMD5, "http_spew/content_md5"
+  autoload :HitNRun, "http_spew/hit_n_run"
+  autoload :InputSpray, "http_spew/input_spray"
+
+  class Error < RuntimeError; end
+  class TimeoutError < Error; end
+  class ConnectionReset < Error; end
+
+  def self.error_all(requests, error) # :nodoc:
+    requests.each { |req| req.error ||= error }
+  end
+
+  def self.done_early(ready, failed, requests) # :nodoc:
+    ready.concat(failed)
+    pending = requests - ready
+    unless pending.empty?
+      error = ConnectionReset.new("prematurely terminated")
+      ready.concat(error_all(pending, error))
+    end
+    ready.uniq!
+    ready
+  end
+
+  # Returns an array of requests that are complete, including those
+  # that have errored out.  Incomplete requests remain in +requests+
+  # If +need+ is fullfilled, it closes all incomplete requests and
+  # returns all requests.
+  def self.wait_nonblock!(need, requests)
+    ready, failed = [], []
+    requests.delete_if do |req|
+      begin
+        case req.resume
+        when Symbol # :wait_writable, :wait_readable
+          false
+        else
+          (ready << req).size == need and
+            return done_early(ready, failed, requests)
+          true
+        end
+      rescue => e
+        req.error = e
+        failed << req
+      end
+    end
+    ready.concat(failed).empty? ? nil : ready
+  end
+
+  # Returns an array of requests that are complete, including those
+  # that have errored out.
+  # If +need+ is fullfilled, it closes all incomplete requests.
+  def self.wait(need, requests, timeout)
+    ready, failed = [], []
+    pollset = {}
+    begin
+      requests.each do |req|
+        begin
+          case rv = req.resume
+          when Symbol # :wait_writable, :wait_readable
+            pollset[req] = rv
+          else
+            (ready << req).size == need and
+              return done_early(ready, failed, requests)
+            pollset.delete(req)
+          end
+        rescue => e
+          req.error = e
+          failed << req
+          pollset.delete(req)
+        end
+      end
+      break if pollset.empty?
+
+      t0 = Time.now
+      busy = pollset.keys
+      rv = Kgio.poll(pollset, timeout.to_i) or break
+      timeout -= (Time.now - t0) * 1000
+    rescue Errno::EINTR
+      timeout -= (Time.now - t0) * 1000
+      retry
+    end while timeout > 0.0 && requests = rv.keys.concat(busy).uniq!
+
+    ready.concat(failed)
+    unless requests.empty?
+      ready.concat(error_all(requests, TimeoutError.new("request timed out")))
+      ready.uniq!
+    end
+    ready
+  end
+end
+require "http_spew/headers"
+require "http_spew/request"