qreplay 1.0.0

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+gemspec
+

data/LICENSE.md

@@ -0,0 +1,10 @@
+Copyright (C) 2014 Old School Industries LLC
+
+(MIT License)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+

data/README.md

@@ -0,0 +1,70 @@
+# qreplay
+
+This is a simple tool to drive capturing, saving, and replaying HTTP requests. It depends on `pcap_tools`, `tshark`, `dumpcap`, and `httperf`.
+
+## Use
+
+```shell
+gem install qreplay
+qreplay [capture|replay|transform|capture_only] [options]
+```
+
+Options:
+```
+    --capture-time, -c <f>:   Capture time length in seconds (default: 60.0)
+            --port, -p <i>:   Capture/replay traffic to port (default: 80)
+            --host, -h <s>:   Replay host (default: 0.0.0.0)
+         --req-sec, -r <i>:   Requests per second for replays (default: 20)
+  --total-requests, -t <i>:   Total replay requests to send (default: 10000)
+    --capture-file, -a <s>:   Output file (default: ./qreplay.sesslog)
+   --tshark-binary, -s <s>:   TShark binary file location (default: tshark)
+  --dumpcap-binary, -d <s>:   dumpcap binary file location (default: dumpcap)
+       --pcap-file, -f <s>:   Temporary intermediate pcap file path (default: ./qreplay.pcap)
+  --httperf-binary, -e <s>:   httperf binary file location (default: httperf)
+                --help, -l:   Show this message
+```
+
+## Example
+
+You want to capture traffic on a live web server. On the remote machine run:
+
+```
+qreplay capture --capture-time 60 --port 80
+```
+
+This will use `tshark` to capture TCP traffic to/from port 80 for 60 seconds, stitch together HTTP requests from the TCP traffic, and save requests to `./qreplay.sesslog`. If you observe this file you will notice that each request line contains an HTTP method, path, and body in a format acceptable to `httperf`.
+
+You can then replay with:
+
+```
+qreplay replay --host 127.0.0.1 --port 80 --req-sec 50
+```
+
+To replay these requests to the local server at port 80 at a rate of 50 per second.
+
+## Installing
+
+Mac OS X (homebrew):
+```
+gem install qreplay
+brew install wireshark httperf
+```
+
+Yum package manager:
+```
+gem install qreplay
+yum install wireshark httperf
+```
+
+## httperf
+
+By default, httperf only handles HTTP bodies with 10,000 bytes or less, which is easy to exceed if you're testing an application that POSTs significant amounts of data. We've increased a few of the arbitrary limitations in httperf in our fork, [here](https://github.com/quizlet/httperf), we recommend using it if you're hitting limits in the mainline httperf.
+
+## License
+
+The qreplay copyright is owned by Old School Industries LLC. We've licensed it under the MIT License, which can be found in `LICENCE.md`.
+
+## Thanks
+
+Thanks to Bertrand Paquet for developing pcap tools and licensing it under the Apache 2 license. You can find more about pcap_tools [here](https://github.com/bpaquet/pcap_tools).
+

data/bin/qreplay

@@ -0,0 +1,142 @@
+#!/usr/bin/env ruby
+
+require 'trollop'
+require 'pcap_tools'
+require 'pp'
+require 'qreplay'
+
+p = Trollop::Parser.new do
+  version "qreplay #{QReplay::VERSION} - (c) Old School Industries LLC"
+  banner <<-EOS
+  qreplay is a tool for capturing and replaying HTTP traffic.
+
+  qreplay [capture|replay|transform|capture-only] <args>
+
+  capture - Use tshark to capture http packets.
+  replay - Replay HTTP session file with requests to a host/port.
+  transform - Transform a dumpcap file to a sesslog file. This is executed automatically in capture mode.
+  capture-only - Perform a capture without a transform step.
+
+  Capturing TCP traffic requires root privileges on most systems.
+
+  Examples:
+
+  > sudo qreplay capture --capture-time 60 --port 80
+  > qreplay replay --host 127.0.0.1 --port 80 --req-sec 50
+
+  qreplay depends on tshark, dumpcap, and httperf, by default attempting to find them in the environment, with the option of passing the paths to them directly on the command line.
+
+  More info can be found at the gem website at github.com/quizlet/qreplay
+
+  Command Line Options:
+
+  EOS
+
+  opt :capture_time, 'Capture time length in seconds', :default => 60.0
+  opt :capture_interface, 'Traffic capture interface (uses dumpcap default if not specified)', :type => :string
+  opt :port, 'Capture/replay traffic to port', :default => 80
+  opt :host, 'Replay host', :default => '0.0.0.0'
+  opt :req_sec, 'Requests per second for replays', :default => 20
+  opt :total_requests, 'Total replay requests to send', :default => 10000
+  opt :capture_file, 'Output file', :default => './qreplay.sesslog'
+  opt :pcap_file, 'Temporary intermediate pcap file path', :default => './qreplay.pcap'
+  opt :timeout, 'Timeout for replay requests', :default => 10
+  opt :tshark_binary, 'TShark binary file location', :default => 'tshark'
+  opt :dumpcap_binary, 'dumpcap binary file location', :default => 'dumpcap'
+  opt :httperf_binary, 'httperf binary file location', :default => 'httperf'
+end
+
+COMMANDS = ['capture', 'replay', 'transform', 'capture-only']
+
+OPT = Trollop::with_standard_exception_handling p do
+  opt = p.parse ARGV
+  raise Trollop::HelpNeeded if ARGV.size < 1 || !COMMANDS.include?(ARGV[0])
+  opt
+end
+
+
+def check_binary(name, binary)
+  r = `which #{binary}`
+  raise "Could not find #{name} binary at path #{binary}" unless r and r.size > 0
+end
+
+class Printer
+  def initialize(capture_file)
+    @counter = 0
+	@fhandle = File.open(capture_file, 'w+')
+  end
+  
+  def process_stream stream
+    stream.each do |index, req, resp|
+	  body = req.body
+      @fhandle.puts "#{req.path} method=#{req.method} contents=#{body.inspect}"
+	  @fhandle.puts "\n"
+      @counter += 1
+    end
+  end
+  
+  def finalize
+    puts "Number of HTTP Requests : #{@counter}"
+	@fhandle.close
+  end
+end
+
+def transform_pcap(tshark_binary, pcap_file, capture_file)
+  puts "Writing HTTP requests with #{tshark_binary} from #{pcap_file} to #{capture_file}"
+  check_binary('tshark', tshark_binary)
+  
+  processor = QReplay::TcpProcessor.new
+  processor.add_stream_processor(PcapTools::TcpStreamRebuilder.new)
+  processor.add_stream_processor(QReplay::HttpExtractor.new)
+  processor.add_stream_processor(Printer.new(capture_file))
+
+  begin
+    PcapTools::Loader::load_file(pcap_file, {}) do |index, packet|
+      begin
+        processor.inject index, packet
+      rescue Exception => e
+        puts "Skipping unparseable request:"
+        puts e.message
+        pp e.backtrace
+      end
+    end
+  rescue Exception => e
+    puts "Exception while parsing tshark output, saving current requests and bailing out."
+	puts e.message
+	pp e.backtrace
+  end
+
+  processor.finalize
+end
+
+def capture_pcap(port, pcap_file, capture_time, dumpcap_binary, capture_interface)
+  check_binary('dumpcap', dumpcap_binary)
+
+  puts "Capturing HTTP on port #{port} to file #{pcap_file} for #{capture_time} seconds"
+  cmd = "#{dumpcap_binary} -w #{pcap_file} -f 'tcp port #{port}' -a duration:#{capture_time.to_i}"
+  cmd += " -i #{capture_interface}" if capture_interface
+  puts cmd
+  system(cmd)
+end
+
+def replay_httperf(binary, req_sec, total_requests, capture_file, host, port, timeout)
+  check_binary('httperf', binary)
+  thinktime = 0.1
+  cmd = "#{binary} --server=#{host} --port=#{port} --rate=#{req_sec} --verbose --wsesslog=#{total_requests},#{thinktime},#{capture_file} --hog --timeout=#{timeout}"
+  puts cmd
+  system(cmd)
+end
+
+command = ARGV[0]
+case command
+when 'capture'
+  capture_pcap(OPT[:port], OPT[:pcap_file], OPT[:capture_time], OPT[:dumpcap_binary], OPT[:capture_interface])
+  transform_pcap(OPT[:tshark_binary], OPT[:pcap_file], OPT[:capture_file])
+when 'capture-only'
+  capture_pcap(OPT[:port], OPT[:pcap_file], OPT[:capture_time], OPT[:dumpcap_binary], OPT[:capture_interface])
+when 'transform'
+  transform_pcap(OPT[:tshark_binary], OPT[:pcap_file], OPT[:capture_file])
+when 'replay'
+  replay_httperf(OPT[:httperf_binary], OPT[:req_sec], OPT[:total_requests], OPT[:capture_file], OPT[:host], OPT[:port], OPT[:timeout])
+end
+

data/lib/qreplay.rb

@@ -0,0 +1,4 @@
+require_relative 'qreplay/tcpprocessor'
+require_relative 'qreplay/httpextractor'
+require_relative 'qreplay/version'
+

data/lib/qreplay/httpextractor.rb

@@ -0,0 +1,127 @@
+# This is a modified version of the code found in https://github.com/bpaquet/pcap_tools/blob/master/lib/pcap_tools/stream_processors/http.rb
+# Modified to handle more HTTP traffic cases.
+
+require 'pp'
+
+module QReplay
+
+  class HttpExtractor
+
+    def process_stream stream
+      calls = []
+      i = 0
+      last_req = nil
+      req = nil
+
+      while i + 1 < stream[:data].size
+        if last_req
+          req = parse_request(last_req, stream[:data][i][:data])
+          last_req = nil
+          req['Expect'] = nil
+        else
+          req = parse_request(stream[:data][i])
+        end
+
+        resp = parse_response(stream[:data][i + 1])
+
+        if req && req['Expect'] && req['Expect'].strip == '100-continue'
+          last_req = stream[:data][i]
+        elsif !req.nil? && !resp.nil?
+          calls << [stream[:index], req, resp]
+        end
+
+        i += 2
+      end
+
+      calls
+    end
+
+    def finalize
+    end
+
+    private
+
+    def parse_request stream, separate_body = nil
+      headers, body = split_headers(stream[:data])
+      return nil unless headers
+      body = separate_body if separate_body
+      line0 = headers.shift
+      m = /(\S+)\s+(\S+)\s+(\S+)/.match(line0) or raise "Unable to parse first line of http request #{line0}"
+      clazz = {
+        'POST' => Net::HTTP::Post,
+        'HEAD' => Net::HTTP::Head,
+        'GET' => Net::HTTP::Get,
+        'PUT' => Net::HTTP::Put,
+        'DELETE' => Net::HTTP::Delete
+      }[m[1]] or raise "Unknown http request type [#{m[1]}]"
+      req = clazz.new m[2]
+      req['Pcap-Src'] = stream[:from]
+      req['Pcap-Src-Port'] = stream[:from_port]
+      req['Pcap-Dst'] = stream[:to]
+      req['Pcap-Dst-Port'] = stream[:to_port]
+      req.time = stream[:time]
+      req.body = body
+      req['user-agent'] = nil
+      req['accept'] = nil
+      add_headers req, headers
+      if req['Content-Length']
+        if req.body.bytesize != req['Content-Length'].to_i && (req['Expect'].nil? || req['Expect'].strip != '100-continue')
+          puts "Wrong content-length for http request, header say [#{req['Content-Length'].chomp}], found #{req.body.size}"
+          return nil
+        end
+      end
+      req
+    end
+
+    def parse_response stream
+      headers, body = split_headers(stream[:data])
+      line0 = headers.shift
+      m = /^(\S+)\s+(\S+)\s+(.*)$/.match(line0) or raise "Unable to parse first line of http response [#{line0}]"
+      resp = Net::HTTPResponse.send(:response_class, m[2]).new(m[1], m[2], m[3])
+      resp.time = stream[:time]
+      add_headers resp, headers
+      if resp.chunked?
+        resp.body = read_chunked("\r\n" + body)
+      else
+        resp.body = body
+        if resp['Content-Length']
+          unless resp.body.size == resp['Content-Length'].to_i
+            puts "Wrong content-length for http response, header say [#{resp['Content-Length'].chomp}], found #{resp.body.size}"
+            return nil
+          end
+        end
+      end
+      begin
+        resp.body = Zlib::GzipReader.new(StringIO.new(resp.body)).read if resp['Content-Encoding'] == 'gzip'
+      rescue Zlib::GzipFile::Error
+        warn "Response body is not in gzip: [#{resp.body}]"
+      end
+      resp
+    end
+
+    def add_headers o, headers
+      headers.each do |line|
+        m = /\A([^:]+):\s*/.match(line) or raise "Unable to parse header line [#{line}]"
+        o[m[1]] = m.post_match
+      end
+    end
+
+    def split_headers str
+      index = str.index("\r\n\r\n")
+      return nil, nil if index.nil?
+      return str[0 .. index].split("\r\n"), str[index + 4 .. -1]
+    end
+
+    def read_chunked str
+      if str.nil? || (str == "\r\n")
+        return ''
+      end
+      m = /\r\n([0-9a-fA-F]+)\r\n/.match(str) or raise "Unable to read chunked body in #{str.split("\r\n")[0]}"
+      len = m[1].hex
+      return '' if len == 0
+      m.post_match[0..len - 1] + read_chunked(m.post_match[len .. -1])
+    end
+
+  end
+
+end

data/lib/qreplay/tcpprocessor.rb

@@ -0,0 +1,68 @@
+# This is a modified version of the code found in https://github.com/bpaquet/pcap_tools/blob/master/lib/pcap_tools/packet_processors/tcp.rb.
+
+require 'time'
+
+module QReplay
+
+  class TcpProcessor
+
+    def initialize
+      @streams = {}
+      @stream_processors = []
+    end
+
+    def add_stream_processor processor
+      @stream_processors << processor
+    end
+
+    def inject index, packet
+      stream_index = packet[:stream]
+      if stream_index
+        if packet[:tcp_flags][:syn] && packet[:tcp_flags][:ack] === false
+          @streams[stream_index] = {
+            :first => packet,
+            :data => [],
+          }
+        elsif packet[:tcp_flags][:fin] || packet[:tcp_flags][:rst]
+          if @streams[stream_index]
+            current = {:index => stream_index, :data => @streams[stream_index][:data]}
+            @stream_processors.each do |p|
+              current = p.process_stream current
+              break unless current
+            end
+            @streams.delete stream_index
+          end
+        else
+          unless @streams[stream_index]
+            @streams[stream_index] = {
+              :first => packet,
+              :data => [],
+            }
+          end
+
+          packet[:type] = (packet[:from] == @streams[stream_index][:first][:from] && packet[:from_port] == @streams[stream_index][:first][:from_port]) ? :out : :in
+          packet.delete :tcp_flags
+          @streams[stream_index][:data] << packet if packet[:size] > 0
+        end
+      end
+    end
+
+    def finalize
+      @streams.each do |k, stream|
+        current = {:index => k, :data => stream[:data]}
+        @stream_processors.each do |p|
+          current = p.process_stream current
+          break unless current
+        end
+      end
+
+      @stream_processors.each do |p|
+        p.finalize
+      end
+    end
+
+  end
+
+end
+
+

data/lib/qreplay/version.rb

@@ -0,0 +1,4 @@
+module QReplay
+  VERSION = '1.0.0'
+end
+

data/qreplay.gemspec

@@ -0,0 +1,24 @@
+# encoding: utf-8
+require File.expand_path("../lib/qreplay.rb", __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.add_runtime_dependency 'pcap_tools'
+  gem.add_runtime_dependency 'trollop'
+
+  gem.authors = ["Peter Bakkum"]
+  gem.bindir = 'bin'
+  gem.description = %q{Capture and replay HTTP traffic}
+  gem.email = ['peter@quizlet.com']
+  gem.executables = ['qreplay']
+  gem.extra_rdoc_files = ['LICENSE.md', 'README.md']
+  gem.files = Dir['LICENSE.md', 'README.md', 'qreplay.gemspec', 'Gemfile', 'bin/*', 'lib/**/*']
+  gem.homepage = 'http://github.com/quizlet/qreplay'
+  gem.name = 'qreplay'
+  gem.rdoc_options = ["--charset=UTF-8"]
+  gem.require_paths = ['lib']
+  gem.required_rubygems_version = Gem::Requirement.new(">= 1.3.6")
+  gem.summary = %q{Capture and replay HTTP traffic}
+  gem.version = QReplay::VERSION
+  gem.license = 'MIT'
+end
+

metadata

@@ -0,0 +1,91 @@
+--- !ruby/object:Gem::Specification
+name: qreplay
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+  prerelease: 
+platform: ruby
+authors:
+- Peter Bakkum
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-03-10 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: pcap_tools
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: trollop
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Capture and replay HTTP traffic
+email:
+- peter@quizlet.com
+executables:
+- qreplay
+extensions: []
+extra_rdoc_files:
+- LICENSE.md
+- README.md
+files:
+- LICENSE.md
+- README.md
+- qreplay.gemspec
+- Gemfile
+- bin/qreplay
+- lib/qreplay/httpextractor.rb
+- lib/qreplay/tcpprocessor.rb
+- lib/qreplay/version.rb
+- lib/qreplay.rb
+homepage: http://github.com/quizlet/qreplay
+licenses:
+- MIT
+post_install_message: 
+rdoc_options:
+- --charset=UTF-8
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: 1.3.6
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23.2
+signing_key: 
+specification_version: 3
+summary: Capture and replay HTTP traffic
+test_files: []