kronk 1.8.7 → 1.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/History.rdoc +22 -0
- data/Manifest.txt +6 -1
- data/README.rdoc +7 -2
- data/Rakefile +5 -4
- data/TODO.rdoc +3 -5
- data/lib/kronk.rb +20 -14
- data/lib/kronk/buffered_io.rb +7 -0
- data/lib/kronk/cmd.rb +25 -9
- data/lib/kronk/constants.rb +8 -0
- data/lib/kronk/http.rb +129 -2
- data/lib/kronk/multipart.rb +82 -0
- data/lib/kronk/multipart_io.rb +88 -0
- data/lib/kronk/player.rb +1 -1
- data/lib/kronk/player/benchmark.rb +69 -24
- data/lib/kronk/player/download.rb +87 -0
- data/lib/kronk/player/suite.rb +23 -11
- data/lib/kronk/request.rb +144 -77
- data/lib/kronk/response.rb +55 -18
- data/test/mocks/200_response.plist +1 -1
- data/test/test_cmd.rb +5 -3
- data/test/test_helper.rb +20 -10
- data/test/test_multipart.rb +144 -0
- data/test/test_multipart_io.rb +92 -0
- data/test/test_player.rb +7 -2
- data/test/test_request.rb +160 -43
- data/test/test_response.rb +34 -2
- metadata +27 -4
data/History.rdoc
CHANGED
@@ -1,3 +1,25 @@
|
|
1
|
+
=== 1.9.0 / 2012-05-22
|
2
|
+
|
3
|
+
* Enhancements:
|
4
|
+
|
5
|
+
* Support for Keep-Alive connections, used by default.
|
6
|
+
|
7
|
+
* Support for multipart form file uploads.
|
8
|
+
|
9
|
+
* Benchmark player is more interactive.
|
10
|
+
|
11
|
+
* Download player added to write output to files.
|
12
|
+
|
13
|
+
* Command API change to allow for player-specific options.
|
14
|
+
|
15
|
+
* Better Mime-Type recognition from file extensions.
|
16
|
+
|
17
|
+
* Bugfixes:
|
18
|
+
|
19
|
+
* Exit immediately when sending SIGINT to the player.
|
20
|
+
|
21
|
+
* Fix the benchmark player error assignment.
|
22
|
+
|
1
23
|
=== 1.8.7 / 2012-04-30
|
2
24
|
|
3
25
|
* Enhancements:
|
data/Manifest.txt
CHANGED
@@ -5,6 +5,7 @@ README.rdoc
|
|
5
5
|
Rakefile
|
6
6
|
bin/kronk
|
7
7
|
lib/kronk.rb
|
8
|
+
lib/kronk/buffered_io.rb
|
8
9
|
lib/kronk/cmd.rb
|
9
10
|
lib/kronk/constants.rb
|
10
11
|
lib/kronk/data_string.rb
|
@@ -13,9 +14,11 @@ lib/kronk/diff/ascii_format.rb
|
|
13
14
|
lib/kronk/diff/color_format.rb
|
14
15
|
lib/kronk/diff/output.rb
|
15
16
|
lib/kronk/http.rb
|
16
|
-
lib/kronk/
|
17
|
+
lib/kronk/multipart.rb
|
18
|
+
lib/kronk/multipart_io.rb
|
17
19
|
lib/kronk/player.rb
|
18
20
|
lib/kronk/player/benchmark.rb
|
21
|
+
lib/kronk/player/download.rb
|
19
22
|
lib/kronk/player/input_reader.rb
|
20
23
|
lib/kronk/player/request_parser.rb
|
21
24
|
lib/kronk/player/suite.rb
|
@@ -49,6 +52,8 @@ test/test_helper.rb
|
|
49
52
|
test/test_helper_methods.rb
|
50
53
|
test/test_input_reader.rb
|
51
54
|
test/test_kronk.rb
|
55
|
+
test/test_multipart.rb
|
56
|
+
test/test_multipart_io.rb
|
52
57
|
test/test_player.rb
|
53
58
|
test/test_request.rb
|
54
59
|
test/test_request_parser.rb
|
data/README.rdoc
CHANGED
@@ -10,7 +10,7 @@ With Kronk, you easily parse and segregate data, run diffs between
|
|
10
10
|
the parsed data from different queries, and easily replay logs and loadtest
|
11
11
|
your HTTP applications.
|
12
12
|
|
13
|
-
Kronk was made possible by the sponsoring of
|
13
|
+
Kronk was made possible by the sponsoring of YP.com.
|
14
14
|
|
15
15
|
== FEATURES:
|
16
16
|
|
@@ -175,7 +175,10 @@ Bash completion is available by sourcing the file returned by:
|
|
175
175
|
== DATA MANIPULATION:
|
176
176
|
|
177
177
|
One of Kronk's most powerful features is its ability to segregate data.
|
178
|
-
From the command line, this is done by passing data paths after '--'
|
178
|
+
From the command line, this is done by passing data paths after '--'.
|
179
|
+
|
180
|
+
All data manipulation is handled by the
|
181
|
+
{ruby-path gem}[http://github.com/yaksnrainbows/ruby-path].
|
179
182
|
|
180
183
|
=== Selecting and Deleting:
|
181
184
|
|
@@ -335,6 +338,8 @@ single quotes around some of your paths.
|
|
335
338
|
|
336
339
|
* cookiejar gem
|
337
340
|
|
341
|
+
* ruby-path gem
|
342
|
+
|
338
343
|
== INSTALL:
|
339
344
|
|
340
345
|
$ gem install kronk
|
data/Rakefile
CHANGED
@@ -11,8 +11,8 @@ else
|
|
11
11
|
Hoe.plugin :isolate
|
12
12
|
end
|
13
13
|
|
14
|
-
require 'kronk'
|
15
14
|
|
15
|
+
require 'kronk'
|
16
16
|
puts "kronk-#{Kronk::VERSION}"
|
17
17
|
|
18
18
|
|
@@ -22,9 +22,10 @@ Hoe.spec 'kronk' do
|
|
22
22
|
self.history_file = "History.rdoc"
|
23
23
|
self.extra_rdoc_files = FileList['*.rdoc']
|
24
24
|
|
25
|
-
self.extra_deps << ['json',
|
26
|
-
self.extra_deps << ['cookiejar',
|
27
|
-
self.extra_deps << ['ruby-path',
|
25
|
+
self.extra_deps << ['json', '~>1.5']
|
26
|
+
self.extra_deps << ['cookiejar', '~>0.3.0']
|
27
|
+
self.extra_deps << ['ruby-path', '~>1.0.0']
|
28
|
+
self.extra_deps << ['mime-types', '~>1.18.0']
|
28
29
|
|
29
30
|
self.extra_dev_deps << ['plist', '~>3.1.0']
|
30
31
|
self.extra_dev_deps << ['nokogiri', '~>1.4']
|
data/TODO.rdoc
CHANGED
@@ -6,15 +6,13 @@
|
|
6
6
|
|
7
7
|
* Consider getting off of net/http.
|
8
8
|
|
9
|
-
* Consider allowing Output-specific cmd options.
|
10
|
-
|
11
9
|
* Investigate Kronk console.
|
12
10
|
|
13
|
-
==
|
11
|
+
== Done
|
14
12
|
|
15
|
-
*
|
13
|
+
* Consider allowing Output-specific cmd options.
|
16
14
|
|
17
|
-
|
15
|
+
* Use persistent connection pools.
|
18
16
|
|
19
17
|
* Color-coded data output.
|
20
18
|
|
data/lib/kronk.rb
CHANGED
@@ -3,6 +3,7 @@ require 'rubygems' if RUBY_VERSION =~ /1.8/
|
|
3
3
|
require 'json'
|
4
4
|
require 'cookiejar'
|
5
5
|
require 'path'
|
6
|
+
require 'mime/types'
|
6
7
|
|
7
8
|
require 'thread'
|
8
9
|
require 'stringio'
|
@@ -13,7 +14,7 @@ require 'yaml'
|
|
13
14
|
class Kronk
|
14
15
|
|
15
16
|
# This gem's version.
|
16
|
-
VERSION = '1.
|
17
|
+
VERSION = '1.9.0'
|
17
18
|
|
18
19
|
require 'kronk/constants'
|
19
20
|
require 'kronk/queue_runner'
|
@@ -21,6 +22,7 @@ class Kronk
|
|
21
22
|
require 'kronk/player/suite'
|
22
23
|
require 'kronk/player/stream'
|
23
24
|
require 'kronk/player/benchmark'
|
25
|
+
require 'kronk/player/download'
|
24
26
|
require 'kronk/player/tsv'
|
25
27
|
require 'kronk/player/request_parser'
|
26
28
|
require 'kronk/player/input_reader'
|
@@ -31,6 +33,8 @@ class Kronk
|
|
31
33
|
require 'kronk/diff'
|
32
34
|
require 'kronk/http'
|
33
35
|
require 'kronk/buffered_io'
|
36
|
+
require 'kronk/multipart'
|
37
|
+
require 'kronk/multipart_io'
|
34
38
|
require 'kronk/request'
|
35
39
|
require 'kronk/response'
|
36
40
|
require 'kronk/plist_parser'
|
@@ -259,9 +263,6 @@ class Kronk
|
|
259
263
|
new(opts).request uri
|
260
264
|
end
|
261
265
|
|
262
|
-
class << self
|
263
|
-
alias retrieve request
|
264
|
-
end
|
265
266
|
|
266
267
|
attr_accessor :diff, :options, :response, :responses
|
267
268
|
|
@@ -345,7 +346,7 @@ class Kronk
|
|
345
346
|
rdir = options[:follow_redirects]
|
346
347
|
while resp.redirect? && (rdir == true || rdir.to_s.to_i > 0)
|
347
348
|
uri = resp.location
|
348
|
-
Cmd.verbose "Following redirect to #{resp.location}"
|
349
|
+
Cmd.verbose "Following redirect to #{resp.location}" if defined?(Cmd)
|
349
350
|
resp = resp.follow_redirect options_for_uri(resp.location)
|
350
351
|
rdir = rdir - 1 if Fixnum === rdir
|
351
352
|
end
|
@@ -359,15 +360,16 @@ class Kronk
|
|
359
360
|
|
360
361
|
resp
|
361
362
|
|
362
|
-
rescue
|
363
|
+
rescue Errno::ENOENT => e
|
364
|
+
raise NotFoundError, e.message
|
365
|
+
|
366
|
+
rescue SocketError, Errno::ECONNREFUSED => e
|
363
367
|
raise NotFoundError, "#{uri} could not be found (#{e.class})"
|
364
368
|
|
365
369
|
rescue Timeout::Error
|
366
370
|
raise TimeoutError, "#{uri} took too long to respond"
|
367
371
|
end
|
368
372
|
|
369
|
-
alias retrieve request
|
370
|
-
|
371
373
|
|
372
374
|
##
|
373
375
|
# Request without autofilling options.
|
@@ -376,16 +378,16 @@ class Kronk
|
|
376
378
|
uri = opts.delete(:uri)
|
377
379
|
|
378
380
|
if IO === uri || StringIO === uri || BufferedIO === uri
|
379
|
-
Cmd.verbose "Reading IO #{uri}"
|
381
|
+
Cmd.verbose "Reading IO #{uri}" if defined?(Cmd)
|
380
382
|
Response.new uri, options
|
381
383
|
|
382
384
|
elsif File.file? uri.to_s
|
383
|
-
Cmd.verbose "Reading file: #{uri}\n"
|
385
|
+
Cmd.verbose "Reading file: #{uri}\n" if defined?(Cmd)
|
384
386
|
Response.read_file uri, options
|
385
387
|
|
386
388
|
else
|
387
389
|
req = Request.new uri, options
|
388
|
-
Cmd.verbose "Retrieving URL: #{req.uri}\n"
|
390
|
+
Cmd.verbose "Retrieving URL: #{req.uri}\n" if defined?(Cmd)
|
389
391
|
resp = req.retrieve options
|
390
392
|
|
391
393
|
hist_uri = req.uri.to_s[0..-req.uri.request_uri.length]
|
@@ -417,7 +419,7 @@ class Kronk
|
|
417
419
|
case key
|
418
420
|
|
419
421
|
# Hash or uri query String
|
420
|
-
when :data, :query
|
422
|
+
when :data, :query, :form, :form_upload
|
421
423
|
val = Request.parse_nested_query val if String === val
|
422
424
|
|
423
425
|
out_opts[key] = Request.parse_nested_query out_opts[key] if
|
@@ -447,11 +449,15 @@ class Kronk
|
|
447
449
|
next if out_opts.has_key?(key) &&
|
448
450
|
(out_opts[key].class != Array || val == true || val == false)
|
449
451
|
out_opts[key] = (val == true || val == false) ? val :
|
450
|
-
|
452
|
+
Array(out_opts[key]) | Array(val)
|
451
453
|
|
452
454
|
# String or Array
|
453
455
|
when :only_data, :ignore_data
|
454
|
-
out_opts[key] =
|
456
|
+
out_opts[key] = Array(out_opts[key]) | Array(val)
|
457
|
+
|
458
|
+
# Array concatination
|
459
|
+
when :transform
|
460
|
+
out_opts[key] = Array(val).concat Array(out_opts[key])
|
455
461
|
end
|
456
462
|
end
|
457
463
|
end
|
data/lib/kronk/buffered_io.rb
CHANGED
data/lib/kronk/cmd.rb
CHANGED
@@ -350,23 +350,27 @@ Parse and run diffs against data from live and cached http responses.
|
|
350
350
|
end
|
351
351
|
|
352
352
|
|
353
|
-
opt.on('--benchmark
|
354
|
-
'Print benchmark data; same as -p
|
355
|
-
options[:player][:io] = File.open(file, "r") if file
|
353
|
+
opt.on('--benchmark',
|
354
|
+
'Print benchmark data; same as -p -o benchmark') do
|
356
355
|
options[:player][:type] = :benchmark
|
357
356
|
end
|
358
357
|
|
359
358
|
|
360
|
-
opt.on('--
|
361
|
-
'
|
362
|
-
options[:player][:
|
359
|
+
opt.on('--download [DIR]',
|
360
|
+
'Write responses to files; same as -p -o download') do |dir|
|
361
|
+
options[:player][:type] = :download
|
362
|
+
options[:player][:dir] = dir
|
363
|
+
end
|
364
|
+
|
365
|
+
|
366
|
+
opt.on('--stream',
|
367
|
+
'Print response stream; same as -p -o stream') do
|
363
368
|
options[:player][:type] = :stream
|
364
369
|
end
|
365
370
|
|
366
371
|
|
367
|
-
opt.on('--tsv
|
368
|
-
'Print TSV metrics; same as -p
|
369
|
-
options[:player][:io] = File.open(file, "r") if file
|
372
|
+
opt.on('--tsv',
|
373
|
+
'Print TSV metrics; same as -p -o tsv') do
|
370
374
|
options[:player][:type] = :tsv
|
371
375
|
end
|
372
376
|
|
@@ -406,6 +410,12 @@ Parse and run diffs against data from live and cached http responses.
|
|
406
410
|
end
|
407
411
|
|
408
412
|
|
413
|
+
opt.on('-M', '--form-upload STR', String,
|
414
|
+
'Multipart file upload <foo=path.ext&bar=path2.ext>') do |value|
|
415
|
+
options[:form_upload] = value
|
416
|
+
end
|
417
|
+
|
418
|
+
|
409
419
|
opt.on('-H', '--header STR', String,
|
410
420
|
'Header to pass to the server request') do |value|
|
411
421
|
options[:headers] ||= {}
|
@@ -433,6 +443,12 @@ Parse and run diffs against data from live and cached http responses.
|
|
433
443
|
end
|
434
444
|
|
435
445
|
|
446
|
+
opt.on('--no-keepalive', 'Don\'t use persistent connections') do
|
447
|
+
options[:headers] ||= {}
|
448
|
+
options[:headers]['Connection'] = 'close'
|
449
|
+
end
|
450
|
+
|
451
|
+
|
436
452
|
opt.on('-x', '--proxy STR', String,
|
437
453
|
'Use HTTP proxy on given port: host[:port]') do |value|
|
438
454
|
options[:proxy][:host], options[:proxy][:port] = value.split ":", 2
|
data/lib/kronk/constants.rb
CHANGED
@@ -103,4 +103,12 @@ class Kronk
|
|
103
103
|
Kronk::Error, Timeout::Error,
|
104
104
|
SocketError, SystemCallError, URI::InvalidURIError
|
105
105
|
]
|
106
|
+
|
107
|
+
|
108
|
+
# Add Plist to MIME types
|
109
|
+
%w{application/plist application/x-plist text/plist text/x-plist}.
|
110
|
+
each do |mime|
|
111
|
+
MIME::Types.add \
|
112
|
+
MIME::Type.new(mime){|t| t.extensions.concat %w{plist xml}}
|
113
|
+
end
|
106
114
|
end
|
data/lib/kronk/http.rb
CHANGED
@@ -7,6 +7,128 @@ class Kronk
|
|
7
7
|
|
8
8
|
class HTTP < Net::HTTP
|
9
9
|
|
10
|
+
class << self
|
11
|
+
# Total number of http connections ever created.
|
12
|
+
attr_accessor :total_conn
|
13
|
+
end
|
14
|
+
|
15
|
+
self.total_conn = 0
|
16
|
+
|
17
|
+
# Pool of open connections.
|
18
|
+
CONN_POOL = Hash.new{|h,k| h[k] = []}
|
19
|
+
|
20
|
+
# Connections currently in use
|
21
|
+
CONN_USED = {}
|
22
|
+
|
23
|
+
# Max time a pool should hold onto an open connection.
|
24
|
+
MAX_CONN_AGE = 10
|
25
|
+
|
26
|
+
C_MUTEX = Mutex.new
|
27
|
+
M_MUTEX = Mutex.new
|
28
|
+
POOL_MUTEX = Hash.new{|h,k| M_MUTEX.synchronize{ h[k] = Mutex.new} }
|
29
|
+
|
30
|
+
|
31
|
+
# Last time this http connection was used
|
32
|
+
attr_accessor :last_used
|
33
|
+
|
34
|
+
|
35
|
+
##
|
36
|
+
# Create a new http connection or get an existing, unused keep-alive
|
37
|
+
# connection. Supports the following options:
|
38
|
+
# :poxy:: String or Hash with proxy settings (see Kronk::Request)
|
39
|
+
# :ssl:: Boolean specifying whether to use SSL or not
|
40
|
+
|
41
|
+
def self.new(address, port=nil, opts={})
|
42
|
+
port ||= HTTP.default_port
|
43
|
+
proxy = opts[:proxy] || {}
|
44
|
+
|
45
|
+
conn_id = [address, port, !!opts[:ssl],
|
46
|
+
proxy[:host], proxy[:port], proxy[:username]]
|
47
|
+
|
48
|
+
conn = get_conn(conn_id)
|
49
|
+
|
50
|
+
if !conn
|
51
|
+
conn = super(address, port, proxy[:host], proxy[:port],
|
52
|
+
proxy[:username], proxy[:password])
|
53
|
+
|
54
|
+
if opts[:ssl]
|
55
|
+
require 'net/https'
|
56
|
+
conn.use_ssl = true
|
57
|
+
end
|
58
|
+
|
59
|
+
C_MUTEX.synchronize{ @total_conn += 1 }
|
60
|
+
end
|
61
|
+
|
62
|
+
CONN_USED[conn] = true
|
63
|
+
|
64
|
+
conn
|
65
|
+
end
|
66
|
+
|
67
|
+
|
68
|
+
##
|
69
|
+
# Total number of currently active connections.
|
70
|
+
|
71
|
+
def self.conn_count
|
72
|
+
M_MUTEX.synchronize do
|
73
|
+
CONN_USED.length + CONN_POOL.values.flatten.length
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
|
78
|
+
##
|
79
|
+
# Get a connection from the pool based on a connection id.
|
80
|
+
# Connection ids are an Array with the following values:
|
81
|
+
# [addr, port, ssl, proxy_addr, proxy_port, proxy_username]
|
82
|
+
|
83
|
+
def self.get_conn(conn_id)
|
84
|
+
conn = nil
|
85
|
+
pool = CONN_POOL[conn_id]
|
86
|
+
|
87
|
+
POOL_MUTEX[conn_id].synchronize do
|
88
|
+
while !pool.empty? && (!conn || conn.closed? || conn.outdated?)
|
89
|
+
conn = pool.shift
|
90
|
+
end
|
91
|
+
end
|
92
|
+
|
93
|
+
conn
|
94
|
+
end
|
95
|
+
|
96
|
+
|
97
|
+
##
|
98
|
+
# Put this http connection in the pool for use by another request.
|
99
|
+
|
100
|
+
def add_to_pool
|
101
|
+
return if closed? || outdated?
|
102
|
+
conn_id = [@address, @port, @use_ssl,
|
103
|
+
proxy_address, proxy_port, proxy_user]
|
104
|
+
|
105
|
+
POOL_MUTEX[conn_id].synchronize do
|
106
|
+
CONN_POOL[conn_id] << self
|
107
|
+
end
|
108
|
+
end
|
109
|
+
|
110
|
+
|
111
|
+
##
|
112
|
+
# Returns true if this connection was last used more than
|
113
|
+
# MAX_CONN_AGE seconds ago.
|
114
|
+
|
115
|
+
def outdated?
|
116
|
+
Time.now - @last_used > MAX_CONN_AGE
|
117
|
+
end
|
118
|
+
|
119
|
+
|
120
|
+
##
|
121
|
+
# Check if the socket for this http connection can be read and written to.
|
122
|
+
|
123
|
+
def closed?
|
124
|
+
!@socket || @socket.closed?
|
125
|
+
end
|
126
|
+
|
127
|
+
|
128
|
+
##
|
129
|
+
# Make an http request on the connection. Takes a Net::HTTP request instance
|
130
|
+
# for the `req' argument.
|
131
|
+
|
10
132
|
def request(req, body=nil, opts={}, &block) # :yield: +response+
|
11
133
|
unless started?
|
12
134
|
start {
|
@@ -32,8 +154,9 @@ class Kronk
|
|
32
154
|
def transport_request(req, allow_retry=true, opts={})
|
33
155
|
# Check if previous request was made on same socket and needs
|
34
156
|
# to be completed before we can read the new response.
|
35
|
-
if Kronk::BufferedIO === @socket
|
36
|
-
@socket.response.body
|
157
|
+
if Kronk::BufferedIO === @socket
|
158
|
+
@socket.response.send(:read_body, "") if @socket.response.body.nil?
|
159
|
+
@socket.clear
|
37
160
|
end
|
38
161
|
|
39
162
|
begin_transport req
|
@@ -67,11 +190,15 @@ class Kronk
|
|
67
190
|
|
68
191
|
|
69
192
|
def end_transport(req, res)
|
193
|
+
CONN_USED.delete self
|
194
|
+
@last_used = Time.now
|
70
195
|
@curr_http_version = res.http_version
|
196
|
+
|
71
197
|
if @socket.closed?
|
72
198
|
D 'Conn socket closed'
|
73
199
|
elsif keep_alive?(req, res)
|
74
200
|
D 'Conn keep-alive'
|
201
|
+
add_to_pool
|
75
202
|
elsif not res.body and @close_on_empty_response
|
76
203
|
D 'Conn close'
|
77
204
|
@socket.close
|