kronk 1.8.7 → 1.9.0

data/History.rdoc

@@ -1,3 +1,25 @@
+=== 1.9.0 / 2012-05-22
+
+* Enhancements:
+
+  * Support for Keep-Alive connections, used by default.
+
+  * Support for multipart form file uploads.
+
+  * Benchmark player is more interactive.
+
+  * Download player added to write output to files.
+
+  * Command API change to allow for player-specific options.
+
+  * Better Mime-Type recognition from file extensions.
+
+* Bugfixes:
+
+  * Exit immediately when sending SIGINT to the player.
+
+  * Fix the benchmark player error assignment.
+
 === 1.8.7 / 2012-04-30
 
 * Enhancements:

data/Manifest.txt

@@ -5,6 +5,7 @@ README.rdoc
 Rakefile
 bin/kronk
 lib/kronk.rb
+lib/kronk/buffered_io.rb
 lib/kronk/cmd.rb
 lib/kronk/constants.rb
 lib/kronk/data_string.rb
@@ -13,9 +14,11 @@ lib/kronk/diff/ascii_format.rb
 lib/kronk/diff/color_format.rb
 lib/kronk/diff/output.rb
 lib/kronk/http.rb
-lib/kronk/buffered_io.rb
+lib/kronk/multipart.rb
+lib/kronk/multipart_io.rb
 lib/kronk/player.rb
 lib/kronk/player/benchmark.rb
+lib/kronk/player/download.rb
 lib/kronk/player/input_reader.rb
 lib/kronk/player/request_parser.rb
 lib/kronk/player/suite.rb
@@ -49,6 +52,8 @@ test/test_helper.rb
 test/test_helper_methods.rb
 test/test_input_reader.rb
 test/test_kronk.rb
+test/test_multipart.rb
+test/test_multipart_io.rb
 test/test_player.rb
 test/test_request.rb
 test/test_request_parser.rb

data/README.rdoc

@@ -10,7 +10,7 @@ With Kronk, you easily parse and segregate data, run diffs between
 the parsed data from different queries, and easily replay logs and loadtest
 your HTTP applications.
 
-Kronk was made possible by the sponsoring of AT&T Interactive.
+Kronk was made possible by the sponsoring of YP.com.
 
 == FEATURES:
 
@@ -175,7 +175,10 @@ Bash completion is available by sourcing the file returned by:
 == DATA MANIPULATION:
 
 One of Kronk's most powerful features is its ability to segregate data.
-From the command line, this is done by passing data paths after '--'
+From the command line, this is done by passing data paths after '--'.
+
+All data manipulation is handled by the
+{ruby-path gem}[http://github.com/yaksnrainbows/ruby-path].
 
 === Selecting and Deleting:
 
@@ -335,6 +338,8 @@ single quotes around some of your paths.
 
 * cookiejar gem
 
+* ruby-path gem
+
 == INSTALL:
 
   $ gem install kronk

data/Rakefile

@@ -11,8 +11,8 @@ else
   Hoe.plugin :isolate
 end
 
-require 'kronk'
 
+require 'kronk'
 puts "kronk-#{Kronk::VERSION}"
 
 
@@ -22,9 +22,10 @@ Hoe.spec 'kronk' do
   self.history_file     = "History.rdoc"
   self.extra_rdoc_files = FileList['*.rdoc']
 
-  self.extra_deps << ['json',      '~>1.5']
-  self.extra_deps << ['cookiejar', '~>0.3.0']
-  self.extra_deps << ['ruby-path', '~>1.0.0']
+  self.extra_deps << ['json',       '~>1.5']
+  self.extra_deps << ['cookiejar',  '~>0.3.0']
+  self.extra_deps << ['ruby-path',  '~>1.0.0']
+  self.extra_deps << ['mime-types', '~>1.18.0']
 
   self.extra_dev_deps << ['plist',    '~>3.1.0']
   self.extra_dev_deps << ['nokogiri', '~>1.4']

data/TODO.rdoc

@@ -6,15 +6,13 @@
 
 * Consider getting off of net/http.
 
-* Consider allowing Output-specific cmd options.
-
 * Investigate Kronk console.
 
-== Not Viable
+== Done
 
-* Use persistent connection pools.
+* Consider allowing Output-specific cmd options.
 
-== Done
+* Use persistent connection pools.
 
 * Color-coded data output.
 

data/lib/kronk.rb

@@ -3,6 +3,7 @@ require 'rubygems' if RUBY_VERSION =~ /1.8/
 require 'json'
 require 'cookiejar'
 require 'path'
+require 'mime/types'
 
 require 'thread'
 require 'stringio'
@@ -13,7 +14,7 @@ require 'yaml'
 class Kronk
 
   # This gem's version.
-  VERSION = '1.8.7'
+  VERSION = '1.9.0'
 
   require 'kronk/constants'
   require 'kronk/queue_runner'
@@ -21,6 +22,7 @@ class Kronk
   require 'kronk/player/suite'
   require 'kronk/player/stream'
   require 'kronk/player/benchmark'
+  require 'kronk/player/download'
   require 'kronk/player/tsv'
   require 'kronk/player/request_parser'
   require 'kronk/player/input_reader'
@@ -31,6 +33,8 @@ class Kronk
   require 'kronk/diff'
   require 'kronk/http'
   require 'kronk/buffered_io'
+  require 'kronk/multipart'
+  require 'kronk/multipart_io'
   require 'kronk/request'
   require 'kronk/response'
   require 'kronk/plist_parser'
@@ -259,9 +263,6 @@ class Kronk
     new(opts).request uri
   end
 
-  class << self
-    alias retrieve request
-  end
 
   attr_accessor :diff, :options, :response, :responses
 
@@ -345,7 +346,7 @@ class Kronk
     rdir = options[:follow_redirects]
     while resp.redirect? && (rdir == true || rdir.to_s.to_i > 0)
       uri = resp.location
-      Cmd.verbose "Following redirect to #{resp.location}"
+      Cmd.verbose "Following redirect to #{resp.location}" if defined?(Cmd)
       resp = resp.follow_redirect options_for_uri(resp.location)
       rdir = rdir - 1 if Fixnum === rdir
     end
@@ -359,15 +360,16 @@ class Kronk
 
     resp
 
-  rescue SocketError, SystemCallError => e
+  rescue Errno::ENOENT => e
+    raise NotFoundError, e.message
+
+  rescue SocketError, Errno::ECONNREFUSED => e
     raise NotFoundError, "#{uri} could not be found (#{e.class})"
 
   rescue Timeout::Error
     raise TimeoutError, "#{uri} took too long to respond"
   end
 
-  alias retrieve request
-
 
   ##
   # Request without autofilling options.
@@ -376,16 +378,16 @@ class Kronk
     uri = opts.delete(:uri)
 
     if IO === uri || StringIO === uri || BufferedIO === uri
-      Cmd.verbose "Reading IO #{uri}"
+      Cmd.verbose "Reading IO #{uri}" if defined?(Cmd)
       Response.new uri, options
 
     elsif File.file? uri.to_s
-      Cmd.verbose "Reading file:  #{uri}\n"
+      Cmd.verbose "Reading file:  #{uri}\n" if defined?(Cmd)
       Response.read_file uri, options
 
     else
       req = Request.new uri, options
-      Cmd.verbose "Retrieving URL:  #{req.uri}\n"
+      Cmd.verbose "Retrieving URL:  #{req.uri}\n" if defined?(Cmd)
       resp = req.retrieve options
 
       hist_uri = req.uri.to_s[0..-req.uri.request_uri.length]
@@ -417,7 +419,7 @@ class Kronk
         case key
 
         # Hash or uri query String
-        when :data, :query
+        when :data, :query, :form, :form_upload
           val = Request.parse_nested_query val if String === val
 
           out_opts[key] = Request.parse_nested_query out_opts[key] if
@@ -447,11 +449,15 @@ class Kronk
           next if out_opts.has_key?(key) &&
                   (out_opts[key].class != Array || val == true || val == false)
           out_opts[key] = (val == true || val == false) ? val :
-                                      [*out_opts[key]] | [*val]
+                                      Array(out_opts[key]) | Array(val)
 
         # String or Array
         when :only_data, :ignore_data
-          out_opts[key] = [*out_opts[key]] | [*val]
+          out_opts[key] = Array(out_opts[key]) | Array(val)
+
+        # Array concatination
+        when :transform
+          out_opts[key] = Array(val).concat Array(out_opts[key])
         end
       end
     end

data/lib/kronk/buffered_io.rb

@@ -20,6 +20,13 @@ class Kronk
     end
 
 
+    def clear
+      @rbuf = ""
+      @raw_output = nil
+      @response   = nil
+    end
+
+
     private
 
     def rbuf_fill

data/lib/kronk/cmd.rb

@@ -350,23 +350,27 @@ Parse and run diffs against data from live and cached http responses.
         end
 
 
-        opt.on('--benchmark [FILE]',
-               'Print benchmark data; same as -p [FILE] -o benchmark') do |file|
-          options[:player][:io]   = File.open(file, "r") if file
+        opt.on('--benchmark',
+               'Print benchmark data; same as -p -o benchmark') do
           options[:player][:type] = :benchmark
         end
 
 
-        opt.on('--stream [FILE]',
-               'Print response stream; same as -p [FILE] -o stream') do |file|
-          options[:player][:io]   = File.open(file, "r") if file
+        opt.on('--download [DIR]',
+               'Write responses to files; same as -p -o download') do |dir|
+          options[:player][:type] = :download
+          options[:player][:dir]  = dir
+        end
+
+
+        opt.on('--stream',
+               'Print response stream; same as -p -o stream') do
           options[:player][:type] = :stream
         end
 
 
-        opt.on('--tsv [FILE]',
-               'Print TSV metrics; same as -p [FILE] -o tsv') do |file|
-          options[:player][:io]   = File.open(file, "r") if file
+        opt.on('--tsv',
+               'Print TSV metrics; same as -p -o tsv') do
           options[:player][:type] = :tsv
         end
 
@@ -406,6 +410,12 @@ Parse and run diffs against data from live and cached http responses.
         end
 
 
+        opt.on('-M', '--form-upload STR', String,
+               'Multipart file upload <foo=path.ext&bar=path2.ext>') do |value|
+          options[:form_upload] = value
+        end
+
+
         opt.on('-H', '--header STR', String,
                'Header to pass to the server request') do |value|
           options[:headers] ||= {}
@@ -433,6 +443,12 @@ Parse and run diffs against data from live and cached http responses.
         end
 
 
+        opt.on('--no-keepalive', 'Don\'t use persistent connections') do
+          options[:headers] ||= {}
+          options[:headers]['Connection'] = 'close'
+        end
+
+
         opt.on('-x', '--proxy STR', String,
                'Use HTTP proxy on given port: host[:port]') do |value|
           options[:proxy][:host], options[:proxy][:port] = value.split ":", 2

data/lib/kronk/constants.rb

@@ -103,4 +103,12 @@ class Kronk
     Kronk::Error, Timeout::Error,
     SocketError, SystemCallError, URI::InvalidURIError
   ]
+
+
+  # Add Plist to MIME types
+  %w{application/plist application/x-plist text/plist text/x-plist}.
+    each do |mime|
+      MIME::Types.add \
+        MIME::Type.new(mime){|t| t.extensions.concat %w{plist xml}}
+    end
 end

data/lib/kronk/http.rb

@@ -7,6 +7,128 @@ class Kronk
 
   class HTTP < Net::HTTP
 
+    class << self
+      # Total number of http connections ever created.
+      attr_accessor :total_conn
+    end
+
+    self.total_conn = 0
+
+    # Pool of open connections.
+    CONN_POOL = Hash.new{|h,k| h[k] = []}
+
+    # Connections currently in use
+    CONN_USED = {}
+
+    # Max time a pool should hold onto an open connection.
+    MAX_CONN_AGE = 10
+
+    C_MUTEX    = Mutex.new
+    M_MUTEX    = Mutex.new
+    POOL_MUTEX = Hash.new{|h,k| M_MUTEX.synchronize{ h[k] = Mutex.new} }
+
+
+    # Last time this http connection was used
+    attr_accessor :last_used
+
+
+    ##
+    # Create a new http connection or get an existing, unused keep-alive
+    # connection. Supports the following options:
+    # :poxy:: String or Hash with proxy settings (see Kronk::Request)
+    # :ssl::  Boolean specifying whether to use SSL or not
+
+    def self.new(address, port=nil, opts={})
+      port ||= HTTP.default_port
+      proxy  = opts[:proxy] || {}
+
+      conn_id = [address, port, !!opts[:ssl],
+                  proxy[:host], proxy[:port], proxy[:username]]
+
+      conn = get_conn(conn_id)
+
+      if !conn
+        conn  = super(address, port, proxy[:host], proxy[:port],
+                        proxy[:username], proxy[:password])
+
+        if opts[:ssl]
+          require 'net/https'
+          conn.use_ssl = true
+        end
+
+        C_MUTEX.synchronize{ @total_conn += 1 }
+      end
+
+      CONN_USED[conn] = true
+
+      conn
+    end
+
+
+    ##
+    # Total number of currently active connections.
+
+    def self.conn_count
+      M_MUTEX.synchronize do
+        CONN_USED.length + CONN_POOL.values.flatten.length
+      end
+    end
+
+
+    ##
+    # Get a connection from the pool based on a connection id.
+    # Connection ids are an Array with the following values:
+    #   [addr, port, ssl, proxy_addr, proxy_port, proxy_username]
+
+    def self.get_conn(conn_id)
+      conn = nil
+      pool = CONN_POOL[conn_id]
+
+      POOL_MUTEX[conn_id].synchronize do
+        while !pool.empty? && (!conn || conn.closed? || conn.outdated?)
+          conn = pool.shift
+        end
+      end
+
+      conn
+    end
+
+
+    ##
+    # Put this http connection in the pool for use by another request.
+
+    def add_to_pool
+      return if closed? || outdated?
+      conn_id = [@address, @port, @use_ssl,
+                  proxy_address, proxy_port, proxy_user]
+
+      POOL_MUTEX[conn_id].synchronize do
+        CONN_POOL[conn_id] << self
+      end
+    end
+
+
+    ##
+    # Returns true if this connection was last used more than
+    # MAX_CONN_AGE seconds ago.
+
+    def outdated?
+      Time.now - @last_used > MAX_CONN_AGE
+    end
+
+
+    ##
+    # Check if the socket for this http connection can be read and written to.
+
+    def closed?
+      !@socket || @socket.closed?
+    end
+
+
+    ##
+    # Make an http request on the connection. Takes a Net::HTTP request instance
+    # for the `req' argument.
+
     def request(req, body=nil, opts={}, &block)  # :yield: +response+
       unless started?
         start {
@@ -32,8 +154,9 @@ class Kronk
     def transport_request(req, allow_retry=true, opts={})
       # Check if previous request was made on same socket and needs
       # to be completed before we can read the new response.
-      if Kronk::BufferedIO === @socket && !@socket.response.read?
-        @socket.response.body
+      if Kronk::BufferedIO === @socket
+        @socket.response.send(:read_body, "") if @socket.response.body.nil?
+        @socket.clear
       end
 
       begin_transport req
@@ -67,11 +190,15 @@ class Kronk
 
 
     def end_transport(req, res)
+      CONN_USED.delete self
+      @last_used = Time.now
       @curr_http_version = res.http_version
+
       if @socket.closed?
         D 'Conn socket closed'
       elsif keep_alive?(req, res)
         D 'Conn keep-alive'
+        add_to_pool
       elsif not res.body and @close_on_empty_response
         D 'Conn close'
         @socket.close