chuckle 1.0.7 → 1.0.9
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop.yml +60 -0
- data/.travis.yml +0 -2
- data/Gemfile +1 -1
- data/Rakefile +6 -6
- data/bin/chuckle +7 -9
- data/chuckle.gemspec +17 -17
- data/lib/chuckle.rb +10 -10
- data/lib/chuckle/cache.rb +8 -8
- data/lib/chuckle/client.rb +5 -4
- data/lib/chuckle/cookie_jar.rb +1 -1
- data/lib/chuckle/curl.rb +24 -23
- data/lib/chuckle/options.rb +6 -6
- data/lib/chuckle/response.rb +1 -1
- data/lib/chuckle/util.rb +5 -5
- data/lib/chuckle/version.rb +1 -1
- data/test/helper.rb +21 -22
- data/test/test_cache.rb +8 -8
- data/test/test_headers.rb +17 -20
- data/test/test_network.rb +15 -15
- data/test/test_requests.rb +8 -8
- metadata +6 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b98846713307331d88cca61554f826cb93c558f51d479b2265196e737fe17754
|
4
|
+
data.tar.gz: 80536128f6942d4ad5dfff3de94c5077d15772a43cf54a2de5d8d26c75569e82
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 46035fb77f332b0b26ab314e4720097080c4d3bc98c7fe724704c0c288695bec0e5a101b52859d2aa8127bd67055f79aab8bdc26433ef4f7b2cc32b3b6ec5c24
|
7
|
+
data.tar.gz: 0f9fd286ca86253ee4db3e07eac090a8da0eaee2e9202a469aa819be5378019219eda660ec4e5b7a292f08206f43d6f3f83f82d00dcc950497ab01017c487f51
|
data/.rubocop.yml
ADDED
@@ -0,0 +1,60 @@
|
|
1
|
+
AllCops:
|
2
|
+
TargetRubyVersion: 2.1.0
|
3
|
+
|
4
|
+
# amd: customizations
|
5
|
+
Layout/SpaceInsideArrayLiteralBrackets:
|
6
|
+
EnforcedStyle: compact
|
7
|
+
Layout/CaseIndentation:
|
8
|
+
EnforcedStyle: end
|
9
|
+
Layout/EndAlignment:
|
10
|
+
EnforcedStyleAlignWith: variable
|
11
|
+
Layout/IndentHeredoc:
|
12
|
+
Enabled: false
|
13
|
+
Style/CollectionMethods:
|
14
|
+
Enabled: true
|
15
|
+
PreferredMethods:
|
16
|
+
reduce: inject
|
17
|
+
Style/TrailingCommaInArrayLiteral:
|
18
|
+
EnforcedStyleForMultiline: consistent_comma
|
19
|
+
Style/TrailingCommaInHashLiteral:
|
20
|
+
EnforcedStyleForMultiline: consistent_comma
|
21
|
+
|
22
|
+
# amd: these seem extreme
|
23
|
+
Lint/AssignmentInCondition: { Enabled: false } # I do this all the time
|
24
|
+
Lint/HandleExceptions: { Enabled: false } # blank rescues are useful
|
25
|
+
Naming/BinaryOperatorParameterName: { Enabled: false } # silly
|
26
|
+
Naming/HeredocDelimiterNaming: { Enabled: false } # silly
|
27
|
+
Naming/UncommunicativeMethodParamName: { Enabled: false } # silly
|
28
|
+
Performance/FixedSize: { Enabled: false } # silly
|
29
|
+
Performance/RegexpMatch: { Enabled: false } # =~ is fine
|
30
|
+
Performance/TimesMap: { Enabled: false } # silly
|
31
|
+
Rails/FilePath: { Enabled: false }
|
32
|
+
Rails/LexicallyScopedActionFilter: { Enabled: false }
|
33
|
+
Rails/OutputSafety: { Enabled: false }
|
34
|
+
Rails/ReversibleMigration: { Enabled: false } # who cares
|
35
|
+
Rails/SkipsModelValidations: { Enabled: false } # silly
|
36
|
+
Style/ClassAndModuleChildren: { Enabled: false } # silly
|
37
|
+
Style/Documentation: { Enabled: false } # we don't need this
|
38
|
+
Style/FormatStringToken: { Enabled: false } # we like printf here
|
39
|
+
Style/FrozenStringLiteralComment: { Enabled: false } # seems excessive
|
40
|
+
Style/GuardClause: { Enabled: false } # confusing
|
41
|
+
Style/IfUnlessModifier: { Enabled: false } # personally I hate unless
|
42
|
+
Style/NegatedIf: { Enabled: false } # these are fine
|
43
|
+
Style/Next: { Enabled: false } # these are fine
|
44
|
+
Style/NumericPredicate: { Enabled: false } # silly
|
45
|
+
Style/ParallelAssignment: { Enabled: false } # these are fine
|
46
|
+
Style/PerlBackrefs: { Enabled: false } # these are fine
|
47
|
+
Style/RaiseArgs: { Enabled: false } # silly
|
48
|
+
Style/RegexpLiteral: { Enabled: false } # these are fine
|
49
|
+
|
50
|
+
# amd: these Metric rules are annoying, disable
|
51
|
+
Metrics/AbcSize: { Enabled: false }
|
52
|
+
Metrics/BlockLength: { Enabled: false }
|
53
|
+
Metrics/BlockNesting: { Enabled: false }
|
54
|
+
Metrics/ClassLength: { Enabled: false }
|
55
|
+
Metrics/CyclomaticComplexity: { Enabled: false }
|
56
|
+
Metrics/LineLength: { Enabled: false }
|
57
|
+
Metrics/MethodLength: { Enabled: false }
|
58
|
+
Metrics/ModuleLength: { Enabled: false }
|
59
|
+
Metrics/ParameterLists: { Enabled: false }
|
60
|
+
Metrics/PerceivedComplexity: { Enabled: false }
|
data/.travis.yml
CHANGED
data/Gemfile
CHANGED
@@ -1,2 +1,2 @@
|
|
1
|
-
source
|
1
|
+
source 'http://rubygems.org'
|
2
2
|
gemspec
|
data/Rakefile
CHANGED
@@ -1,6 +1,6 @@
|
|
1
|
-
require
|
2
|
-
require
|
3
|
-
require
|
1
|
+
require 'bundler/gem_helper'
|
2
|
+
require 'rake/testtask'
|
3
|
+
require 'rdoc/task'
|
4
4
|
|
5
5
|
Bundler::GemHelper.install_tasks
|
6
6
|
|
@@ -9,7 +9,7 @@ Bundler::GemHelper.install_tasks
|
|
9
9
|
#
|
10
10
|
|
11
11
|
Rake::TestTask.new(:test) do |test|
|
12
|
-
test.libs <<
|
12
|
+
test.libs << 'test'
|
13
13
|
end
|
14
14
|
task default: :test
|
15
15
|
|
@@ -18,7 +18,7 @@ task default: :test
|
|
18
18
|
#
|
19
19
|
|
20
20
|
RDoc::Task.new do |rdoc|
|
21
|
-
rdoc.rdoc_dir =
|
21
|
+
rdoc.rdoc_dir = 'rdoc'
|
22
22
|
rdoc.title = "chuckle #{Chuckle::VERSION}"
|
23
|
-
rdoc.rdoc_files.include(
|
23
|
+
rdoc.rdoc_files.include('lib/**/*.rb')
|
24
24
|
end
|
data/bin/chuckle
CHANGED
@@ -3,8 +3,8 @@
|
|
3
3
|
# in lieu of -w, since we're using env to startup
|
4
4
|
$VERBOSE = true
|
5
5
|
|
6
|
-
require
|
7
|
-
require
|
6
|
+
require 'chuckle'
|
7
|
+
require 'optimist'
|
8
8
|
|
9
9
|
class Main
|
10
10
|
attr_accessor :options
|
@@ -13,7 +13,7 @@ class Main
|
|
13
13
|
self.options = options
|
14
14
|
list = IO.readlines(options[:file]).map(&:chomp)
|
15
15
|
list.each_with_index do |i, index|
|
16
|
-
puts "#{index+1}/#{list.length}. #{i}"
|
16
|
+
puts "#{index + 1}/#{list.length}. #{i}"
|
17
17
|
begin
|
18
18
|
chuckle.get(i)
|
19
19
|
rescue Chuckle::Error => e
|
@@ -29,13 +29,11 @@ class Main
|
|
29
29
|
end
|
30
30
|
end
|
31
31
|
|
32
|
-
|
33
|
-
|
34
32
|
#
|
35
33
|
# parse command line
|
36
34
|
#
|
37
35
|
|
38
|
-
options =
|
36
|
+
options = Optimist::options do
|
39
37
|
banner <<EOF
|
40
38
|
Usage: chuckle <url_file>
|
41
39
|
|
@@ -43,10 +41,10 @@ chuckle fetches urls as listed in url_file. It just populate the disk
|
|
43
41
|
cache.
|
44
42
|
EOF
|
45
43
|
|
46
|
-
opt :nretries,
|
47
|
-
opt :timeout,
|
44
|
+
opt :nretries, 'Number of retries per URL', type: Integer
|
45
|
+
opt :timeout, 'Timeout per retry', type: Integer
|
48
46
|
end
|
49
47
|
|
50
|
-
|
48
|
+
Optimist::die 'need <url_file>' if ARGV.length != 1
|
51
49
|
options[:file] = ARGV.shift
|
52
50
|
Main.new(options)
|
data/chuckle.gemspec
CHANGED
@@ -1,30 +1,30 @@
|
|
1
|
-
$LOAD_PATH << File.expand_path(
|
1
|
+
$LOAD_PATH << File.expand_path('lib', __dir__)
|
2
2
|
|
3
|
-
require
|
3
|
+
require 'chuckle/version'
|
4
4
|
|
5
5
|
Gem::Specification.new do |s|
|
6
|
-
s.name =
|
6
|
+
s.name = 'chuckle'
|
7
7
|
s.version = Chuckle::VERSION
|
8
8
|
s.platform = Gem::Platform::RUBY
|
9
|
-
s.required_ruby_version =
|
10
|
-
s.authors = [
|
11
|
-
s.email = [
|
12
|
-
s.homepage =
|
13
|
-
s.summary =
|
14
|
-
s.description =
|
9
|
+
s.required_ruby_version = '>= 2.1.0'
|
10
|
+
s.authors = [ 'Adam Doppelt' ]
|
11
|
+
s.email = [ 'amd@gurge.com' ]
|
12
|
+
s.homepage = 'http://github.com/gurgeous/chuckle'
|
13
|
+
s.summary = 'Chuckle - an http client that caches on disk.'
|
14
|
+
s.description = 'An http client that caches on disk.'
|
15
15
|
|
16
|
-
s.rubyforge_project =
|
16
|
+
s.rubyforge_project = 'chuckle'
|
17
17
|
|
18
|
-
s.add_runtime_dependency
|
18
|
+
s.add_runtime_dependency 'optimist'
|
19
19
|
|
20
|
-
s.add_development_dependency
|
21
|
-
s.add_development_dependency
|
22
|
-
s.add_development_dependency
|
23
|
-
s.add_development_dependency
|
24
|
-
s.add_development_dependency
|
20
|
+
s.add_development_dependency 'awesome_print'
|
21
|
+
s.add_development_dependency 'json'
|
22
|
+
s.add_development_dependency 'minitest', '~> 5.0'
|
23
|
+
s.add_development_dependency 'rake'
|
24
|
+
s.add_development_dependency 'rdoc'
|
25
25
|
|
26
26
|
s.files = `git ls-files`.split("\n")
|
27
27
|
s.test_files = `git ls-files -- test/*`.split("\n")
|
28
28
|
s.executables = `git ls-files -- bin/*`.split("\n").map { |i| File.basename(i) }
|
29
|
-
s.require_paths = [
|
29
|
+
s.require_paths = [ 'lib' ]
|
30
30
|
end
|
data/lib/chuckle.rb
CHANGED
@@ -1,12 +1,12 @@
|
|
1
|
-
require
|
2
|
-
require
|
3
|
-
require
|
4
|
-
require
|
5
|
-
require
|
6
|
-
require
|
7
|
-
require
|
8
|
-
require
|
1
|
+
require 'chuckle/cache'
|
2
|
+
require 'chuckle/cookie_jar'
|
3
|
+
require 'chuckle/error'
|
4
|
+
require 'chuckle/options'
|
5
|
+
require 'chuckle/request'
|
6
|
+
require 'chuckle/response'
|
7
|
+
require 'chuckle/util'
|
8
|
+
require 'chuckle/version'
|
9
9
|
|
10
10
|
# these last
|
11
|
-
require
|
12
|
-
require
|
11
|
+
require 'chuckle/curl'
|
12
|
+
require 'chuckle/client'
|
data/lib/chuckle/cache.rb
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
require
|
1
|
+
require 'fileutils'
|
2
2
|
|
3
3
|
module Chuckle
|
4
4
|
class Cache
|
@@ -21,7 +21,7 @@ module Chuckle
|
|
21
21
|
|
22
22
|
def set(request, curl)
|
23
23
|
# mkdirs
|
24
|
-
FileUtils.mkdir_p([File.dirname(request.headers_path), File.dirname(request.body_path)])
|
24
|
+
FileUtils.mkdir_p([ File.dirname(request.headers_path), File.dirname(request.body_path) ])
|
25
25
|
|
26
26
|
# now mv
|
27
27
|
FileUtils.mv(curl.headers_path, request.headers_path)
|
@@ -53,7 +53,7 @@ module Chuckle
|
|
53
53
|
|
54
54
|
# calculate body_path
|
55
55
|
s = @client.cache_dir
|
56
|
-
s = "#{s}/#{pathify(uri.host ||
|
56
|
+
s = "#{s}/#{pathify(uri.host || 'file')}"
|
57
57
|
s = "#{s}/#{pathify(uri.path)}"
|
58
58
|
if uri.query
|
59
59
|
q = "?#{uri.query}"
|
@@ -76,15 +76,15 @@ module Chuckle
|
|
76
76
|
|
77
77
|
# turn s into a string that can be a path
|
78
78
|
def pathify(s)
|
79
|
-
s = s.gsub(/^\//,
|
80
|
-
s = s.gsub(
|
81
|
-
s = s.gsub(/[?\/&]/,
|
79
|
+
s = s.gsub(/^\//, '')
|
80
|
+
s = s.gsub('..', ',')
|
81
|
+
s = s.gsub(/[?\/&]/, ',')
|
82
82
|
s = s.gsub(/[^A-Za-z0-9_.,=%-]/) do |i|
|
83
|
-
hex = i.unpack(
|
83
|
+
hex = i.unpack('H2').first
|
84
84
|
"%#{hex}"
|
85
85
|
end
|
86
86
|
s = s.downcase
|
87
|
-
s =
|
87
|
+
s = '_root_' if s.empty?
|
88
88
|
s
|
89
89
|
end
|
90
90
|
end
|
data/lib/chuckle/client.rb
CHANGED
@@ -1,4 +1,5 @@
|
|
1
|
-
require
|
1
|
+
require 'English'
|
2
|
+
require 'uri'
|
2
3
|
|
3
4
|
module Chuckle
|
4
5
|
class Client
|
@@ -8,7 +9,7 @@ module Chuckle
|
|
8
9
|
|
9
10
|
def initialize(options = {})
|
10
11
|
self.options = DEFAULT_OPTIONS.dup
|
11
|
-
options.each { |k, v| self.options[k] = v if v
|
12
|
+
options.each { |k, v| self.options[k] = v if !v.nil? }
|
12
13
|
self.cache = Cache.new(self)
|
13
14
|
sanity!
|
14
15
|
end
|
@@ -50,8 +51,8 @@ module Chuckle
|
|
50
51
|
|
51
52
|
# make sure curl command exists
|
52
53
|
def sanity!
|
53
|
-
system(
|
54
|
-
raise
|
54
|
+
system('which curl > /dev/null')
|
55
|
+
raise 'Chuckle requires curl. Please install it.' if $CHILD_STATUS != 0
|
55
56
|
end
|
56
57
|
|
57
58
|
def curl(request)
|
data/lib/chuckle/cookie_jar.rb
CHANGED
data/lib/chuckle/curl.rb
CHANGED
@@ -1,4 +1,5 @@
|
|
1
|
-
require
|
1
|
+
require 'English'
|
2
|
+
require 'fileutils'
|
2
3
|
|
3
4
|
module Chuckle
|
4
5
|
class Curl
|
@@ -22,9 +23,9 @@ module Chuckle
|
|
22
23
|
Kernel.system(*command)
|
23
24
|
|
24
25
|
# capture exit code, bail on INT
|
25
|
-
exit_code =
|
26
|
-
if
|
27
|
-
Process.kill(:INT,
|
26
|
+
exit_code = $CHILD_STATUS.to_i / 256
|
27
|
+
if $CHILD_STATUS.termsig == Signal.list['INT']
|
28
|
+
Process.kill(:INT, $PROCESS_ID)
|
28
29
|
end
|
29
30
|
|
30
31
|
# create tmp files if there were errors
|
@@ -59,41 +60,41 @@ module Chuckle
|
|
59
60
|
def command(request)
|
60
61
|
client = request.client
|
61
62
|
|
62
|
-
command = [
|
63
|
-
command <<
|
64
|
-
command <<
|
63
|
+
command = [ 'curl' ]
|
64
|
+
command << '--silent'
|
65
|
+
command << '--compressed'
|
65
66
|
|
66
|
-
command += [
|
67
|
-
command += [
|
68
|
-
command += [
|
69
|
-
command += [
|
67
|
+
command += [ '--user-agent', client.user_agent ]
|
68
|
+
command += [ '--max-time', client.timeout ]
|
69
|
+
command += [ '--retry', client.nretries ]
|
70
|
+
command += [ '--location', '--max-redirs', 3 ]
|
70
71
|
|
71
72
|
if request.body
|
72
|
-
command += [
|
73
|
+
command += [ '--data-binary', request.body ]
|
73
74
|
end
|
74
75
|
|
75
76
|
# maintain backwards compatibility for content type
|
76
|
-
client.headers.each do |key,value|
|
77
|
-
if key ==
|
78
|
-
command += [
|
77
|
+
client.headers.each do |key, value|
|
78
|
+
if key == 'Content-Type'
|
79
|
+
command += [ '--header', "#{key}: #{value}" ] if request.body
|
79
80
|
else
|
80
|
-
command += [
|
81
|
+
command += [ '--header', "#{key}: #{value}" ]
|
81
82
|
end
|
82
83
|
end
|
83
84
|
|
84
85
|
if client.cookies?
|
85
86
|
cookie_jar.preflight
|
86
|
-
command += [
|
87
|
-
command += [
|
87
|
+
command += [ '--cookie', cookie_jar.path ]
|
88
|
+
command += [ '--cookie-jar', cookie_jar.path ]
|
88
89
|
end
|
89
90
|
|
90
91
|
# SSL options
|
91
|
-
command += [
|
92
|
-
command += [
|
93
|
-
command += [
|
92
|
+
command += [ '--cacert', client.cacert ] if client.cacert
|
93
|
+
command += [ '--capath', client.capath ] if client.capath
|
94
|
+
command += [ '--insecure' ] if client.insecure?
|
94
95
|
|
95
|
-
command += [
|
96
|
-
command += [
|
96
|
+
command += [ '--dump-header', headers_path ]
|
97
|
+
command += [ '--output', body_path ]
|
97
98
|
|
98
99
|
command << request.uri
|
99
100
|
|
data/lib/chuckle/options.rb
CHANGED
@@ -5,7 +5,7 @@ module Chuckle
|
|
5
5
|
cache_errors: true,
|
6
6
|
cacert: nil,
|
7
7
|
capath: nil,
|
8
|
-
content_type:
|
8
|
+
content_type: 'application/x-www-form-urlencoded',
|
9
9
|
cookies: false,
|
10
10
|
expires_in: :never,
|
11
11
|
headers: nil,
|
@@ -13,22 +13,22 @@ module Chuckle
|
|
13
13
|
nretries: 2,
|
14
14
|
rate_limit: 1,
|
15
15
|
timeout: 30,
|
16
|
-
user_agent:
|
16
|
+
user_agent: 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0',
|
17
17
|
verbose: false,
|
18
|
-
}
|
18
|
+
}.freeze
|
19
19
|
|
20
20
|
# cache root directory
|
21
21
|
def cache_dir
|
22
22
|
@cache_dir ||= begin
|
23
23
|
dir = options[:cache_dir]
|
24
24
|
dir ||= begin
|
25
|
-
if home = ENV[
|
25
|
+
if home = ENV['HOME']
|
26
26
|
if File.exist?(home) && File.stat(home).writable?
|
27
27
|
"#{home}/.chuckle"
|
28
28
|
end
|
29
29
|
end
|
30
30
|
end
|
31
|
-
dir ||=
|
31
|
+
dir ||= '/tmp/chuckle'
|
32
32
|
dir
|
33
33
|
end
|
34
34
|
end
|
@@ -66,7 +66,7 @@ module Chuckle
|
|
66
66
|
def headers
|
67
67
|
@headers ||= begin
|
68
68
|
headers = options[:headers] || {}
|
69
|
-
headers[
|
69
|
+
headers['Content-Type'] = options[:content_type] if options[:content_type]
|
70
70
|
headers
|
71
71
|
end
|
72
72
|
end
|
data/lib/chuckle/response.rb
CHANGED
data/lib/chuckle/util.rb
CHANGED
@@ -1,9 +1,9 @@
|
|
1
|
-
require
|
2
|
-
require
|
1
|
+
require 'digest/md5'
|
2
|
+
require 'tempfile'
|
3
3
|
|
4
4
|
module Chuckle
|
5
5
|
module Util
|
6
|
-
|
6
|
+
module_function
|
7
7
|
|
8
8
|
def hash_to_query(hash)
|
9
9
|
q = hash.map do |key, value|
|
@@ -11,7 +11,7 @@ module Chuckle
|
|
11
11
|
value = CGI.escape(value.to_s)
|
12
12
|
"#{key}=#{value}"
|
13
13
|
end
|
14
|
-
q.sort.join(
|
14
|
+
q.sort.join('&')
|
15
15
|
end
|
16
16
|
|
17
17
|
def md5(s)
|
@@ -23,7 +23,7 @@ module Chuckle
|
|
23
23
|
end
|
24
24
|
|
25
25
|
def tmp_path
|
26
|
-
Tempfile.open(
|
26
|
+
Tempfile.open('chuckle') do |f|
|
27
27
|
path = f.path
|
28
28
|
f.unlink
|
29
29
|
path
|
data/lib/chuckle/version.rb
CHANGED
data/test/helper.rb
CHANGED
@@ -1,39 +1,38 @@
|
|
1
|
-
require
|
2
|
-
require
|
3
|
-
require
|
4
|
-
require
|
5
|
-
require
|
1
|
+
require 'awesome_print'
|
2
|
+
require 'chuckle'
|
3
|
+
require 'json'
|
4
|
+
require 'minitest/autorun'
|
5
|
+
require 'minitest/pride'
|
6
6
|
|
7
7
|
module Helper
|
8
|
-
CACHE_DIR =
|
9
|
-
URL =
|
10
|
-
QUERY = {
|
11
|
-
TMP =
|
8
|
+
CACHE_DIR = '/tmp/_chuckle_tests'.freeze
|
9
|
+
URL = 'http://chuckle'.freeze
|
10
|
+
QUERY = { 'b' => '12', 'a' => '34', 'x y' => '56' }.freeze
|
11
|
+
TMP = '/tmp/_chuckle_tmp.txt'.freeze
|
12
12
|
|
13
13
|
#
|
14
14
|
# fake responses
|
15
15
|
#
|
16
16
|
|
17
|
-
HTTP_200 = <<-EOF.gsub(/(^|\n) +/,
|
17
|
+
HTTP_200 = <<-EOF.gsub(/(^|\n) +/, '\\1')
|
18
18
|
HTTP/1.1 200 OK
|
19
19
|
|
20
20
|
hello
|
21
21
|
EOF
|
22
22
|
|
23
|
-
HTTP_200_ALTERNATE = <<-EOF.gsub(/(^|\n) +/,
|
23
|
+
HTTP_200_ALTERNATE = <<-EOF.gsub(/(^|\n) +/, '\\1')
|
24
24
|
HTTP/1.1 200 OK
|
25
25
|
|
26
26
|
alternate
|
27
27
|
EOF
|
28
28
|
|
29
|
-
|
30
|
-
HTTP2_200 = <<-EOF.gsub(/(^|\n) +/, "\\1")
|
29
|
+
HTTP2_200 = <<-EOF.gsub(/(^|\n) +/, '\\1')
|
31
30
|
HTTP/2 200 OK
|
32
31
|
|
33
32
|
hello
|
34
33
|
EOF
|
35
34
|
|
36
|
-
HTTP_302 = <<-EOF.gsub(/(^|\n) +/,
|
35
|
+
HTTP_302 = <<-EOF.gsub(/(^|\n) +/, '\\1')
|
37
36
|
HTTP/1.1 302 FOUND
|
38
37
|
Location: http://one
|
39
38
|
|
@@ -42,7 +41,7 @@ module Helper
|
|
42
41
|
hello
|
43
42
|
EOF
|
44
43
|
|
45
|
-
HTTP_302_2 = <<-EOF.gsub(/(^|\n) +/,
|
44
|
+
HTTP_302_2 = <<-EOF.gsub(/(^|\n) +/, '\\1')
|
46
45
|
HTTP/1.1 302 FOUND
|
47
46
|
Location: http://one
|
48
47
|
|
@@ -54,7 +53,7 @@ module Helper
|
|
54
53
|
hello
|
55
54
|
EOF
|
56
55
|
|
57
|
-
HTTP_302_RELATIVE = <<-EOF.gsub(/(^|\n) +/,
|
56
|
+
HTTP_302_RELATIVE = <<-EOF.gsub(/(^|\n) +/, '\\1')
|
58
57
|
HTTP/1.1 302 FOUND
|
59
58
|
Location: /two
|
60
59
|
|
@@ -63,7 +62,7 @@ module Helper
|
|
63
62
|
hello
|
64
63
|
EOF
|
65
64
|
|
66
|
-
HTTP_404 = <<-EOF.gsub(/(^|\n) +/,
|
65
|
+
HTTP_404 = <<-EOF.gsub(/(^|\n) +/, '\\1')
|
67
66
|
HTTP/1.1 404 Not Found
|
68
67
|
|
69
68
|
EOF
|
@@ -82,7 +81,7 @@ module Helper
|
|
82
81
|
end
|
83
82
|
|
84
83
|
# pretend to be curl by stubbing Kernel.system
|
85
|
-
def mcurl(response, exit_code = 0
|
84
|
+
def mcurl(response, exit_code = 0)
|
86
85
|
# divide response into headers/body
|
87
86
|
sep = response.rindex("\n\n") + "\n\n".length
|
88
87
|
body = response[sep..-1]
|
@@ -90,8 +89,8 @@ module Helper
|
|
90
89
|
|
91
90
|
# a lambda that pretends to be curl
|
92
91
|
fake_system = lambda do |*command|
|
93
|
-
tmp_headers = command[command.index(
|
94
|
-
tmp_body = command[command.index(
|
92
|
+
tmp_headers = command[command.index('--dump-header') + 1]
|
93
|
+
tmp_body = command[command.index('--output') + 1]
|
95
94
|
IO.write(tmp_headers, headers)
|
96
95
|
IO.write(tmp_body, body)
|
97
96
|
`(exit #{exit_code})`
|
@@ -101,7 +100,7 @@ module Helper
|
|
101
100
|
Kernel.stub(:system, fake_system) { yield }
|
102
101
|
end
|
103
102
|
|
104
|
-
def assert_if_system
|
103
|
+
def assert_if_system
|
105
104
|
fake_system = lambda do |*command|
|
106
105
|
assert false, "system called with #{command.inspect}"
|
107
106
|
end
|
@@ -110,6 +109,6 @@ module Helper
|
|
110
109
|
|
111
110
|
def assert_command(cmd)
|
112
111
|
system("#{cmd} > #{TMP} 2>&1")
|
113
|
-
assert(
|
112
|
+
assert($CHILD_STATUS == 0, File.read(TMP))
|
114
113
|
end
|
115
114
|
end
|
data/test/test_cache.rb
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
require
|
1
|
+
require 'helper'
|
2
2
|
|
3
3
|
class TestCache < Minitest::Test
|
4
4
|
include Helper
|
@@ -10,16 +10,16 @@ class TestCache < Minitest::Test
|
|
10
10
|
# exists? and expired? (and clear)
|
11
11
|
def test_predicates
|
12
12
|
request = client.create_request(URL)
|
13
|
-
assert !client.cache.exists?(request),
|
13
|
+
assert !client.cache.exists?(request), 'uncache! uri said it was cached'
|
14
14
|
|
15
15
|
mcurl(HTTP_200) do
|
16
16
|
client.run(request)
|
17
17
|
end
|
18
18
|
assert client.cache.exists?(request), "cache said it wasn't cached"
|
19
|
-
assert !client.cache.expired?(request),
|
19
|
+
assert !client.cache.expired?(request), 'cache said it was expired'
|
20
20
|
|
21
21
|
client.cache.clear(request)
|
22
|
-
assert !client.cache.exists?(request),
|
22
|
+
assert !client.cache.exists?(request), 'still cached after clear'
|
23
23
|
end
|
24
24
|
|
25
25
|
# cache expiration
|
@@ -71,7 +71,7 @@ class TestCache < Minitest::Test
|
|
71
71
|
mcurl(HTTP_404) do
|
72
72
|
client.get(URL)
|
73
73
|
end
|
74
|
-
rescue Chuckle::Error
|
74
|
+
rescue Chuckle::Error
|
75
75
|
end
|
76
76
|
|
77
77
|
# cache hit
|
@@ -91,7 +91,7 @@ class TestCache < Minitest::Test
|
|
91
91
|
mcurl(HTTP_404, Chuckle::Error::CURL_TIMEOUT) do
|
92
92
|
client.get(URL)
|
93
93
|
end
|
94
|
-
rescue Chuckle::Error
|
94
|
+
rescue Chuckle::Error
|
95
95
|
end
|
96
96
|
|
97
97
|
# cache hit
|
@@ -123,9 +123,9 @@ class TestCache < Minitest::Test
|
|
123
123
|
end
|
124
124
|
|
125
125
|
def test_long_url
|
126
|
-
words = %w
|
126
|
+
words = %w[the quick brown fox jumped over the lazy dog]
|
127
127
|
query = (1..100).map { words[rand(words.length)] }
|
128
|
-
query = query.each_slice(2).map { |i| i.join(
|
128
|
+
query = query.each_slice(2).map { |i| i.join('=') }.join('&')
|
129
129
|
|
130
130
|
# cache miss
|
131
131
|
mcurl(HTTP_200) do
|
data/test/test_headers.rb
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
require
|
1
|
+
require 'helper'
|
2
2
|
|
3
3
|
# these actually use the network, and get skipped unless ENV["NETWORK"].
|
4
4
|
class TestHeaders < Minitest::Test
|
@@ -6,45 +6,43 @@ class TestHeaders < Minitest::Test
|
|
6
6
|
|
7
7
|
def test_default_headers
|
8
8
|
test_client = client(verbose: true)
|
9
|
-
assert_equal({
|
9
|
+
assert_equal({ 'Content-Type' => 'application/x-www-form-urlencoded' }, test_client.headers)
|
10
10
|
end
|
11
11
|
|
12
12
|
def test_content_type
|
13
|
-
test_client = client(content_type:
|
14
|
-
assert_equal({
|
13
|
+
test_client = client(content_type: 'text/json')
|
14
|
+
assert_equal({ 'Content-Type' => 'text/json' }, test_client.headers)
|
15
15
|
end
|
16
16
|
|
17
17
|
def test_arbitrary_headers
|
18
|
-
test_client = client(headers: {
|
19
|
-
assert_equal test_client.headers[
|
20
|
-
assert_equal test_client.headers[
|
21
|
-
|
18
|
+
test_client = client(headers: { 'Referer' => 'http://foo.com' })
|
19
|
+
assert_equal test_client.headers['Referer'], 'http://foo.com'
|
20
|
+
assert_equal test_client.headers['Content-Type'], 'application/x-www-form-urlencoded'
|
22
21
|
end
|
23
22
|
|
24
23
|
def test_basic_get
|
25
24
|
test_client = client(verbose: true)
|
26
|
-
request = test_client.create_request(
|
25
|
+
request = test_client.create_request('http://httpbin.org/get')
|
27
26
|
headers = command_headers(request)
|
28
27
|
assert_equal headers, []
|
29
28
|
end
|
30
29
|
|
31
|
-
|
32
30
|
def test_basic_post
|
33
|
-
request = client(verbose: true).create_request(
|
31
|
+
request = client(verbose: true).create_request('http://httpbin.org/post', 'foo')
|
34
32
|
headers = command_headers(request)
|
35
|
-
assert_equal headers, [
|
33
|
+
assert_equal headers, [ 'Content-Type: application/x-www-form-urlencoded' ]
|
36
34
|
end
|
37
35
|
|
38
36
|
def test_get_headers
|
39
|
-
request = client(headers: {
|
37
|
+
request = client(headers: { 'Referer' => 'http://foo.com' }).create_request('http://httpbin.org/get')
|
40
38
|
headers = command_headers(request)
|
41
|
-
assert_equal headers, [
|
39
|
+
assert_equal headers, [ 'Referer: http://foo.com' ]
|
42
40
|
end
|
43
41
|
|
44
42
|
def test_post_headers
|
45
|
-
request = client(headers: {
|
43
|
+
request = client(headers: { 'Referer' => 'http://foo.com' }).create_request('http://httpbin.org/post', 'foo')
|
46
44
|
headers = command_headers(request)
|
47
|
-
assert_equal headers.sort, [
|
45
|
+
assert_equal headers.sort, [ 'Content-Type: application/x-www-form-urlencoded', 'Referer: http://foo.com' ]
|
48
46
|
end
|
49
47
|
|
50
48
|
private
|
@@ -58,8 +56,8 @@ class TestHeaders < Minitest::Test
|
|
58
56
|
headers = []
|
59
57
|
test_command = command(request)
|
60
58
|
test_command.each_with_index do |item, i|
|
61
|
-
if item ==
|
62
|
-
headers << test_command[i+1]
|
59
|
+
if item == '--header'
|
60
|
+
headers << test_command[i + 1]
|
63
61
|
end
|
64
62
|
end
|
65
63
|
headers
|
@@ -70,5 +68,4 @@ class TestHeaders < Minitest::Test
|
|
70
68
|
command(@request)
|
71
69
|
end
|
72
70
|
end
|
73
|
-
|
74
|
-
end
|
71
|
+
end
|
data/test/test_network.rb
CHANGED
@@ -1,36 +1,36 @@
|
|
1
|
-
require
|
1
|
+
require 'helper'
|
2
2
|
|
3
3
|
# these actually use the network, and get skipped unless ENV["NETWORK"].
|
4
4
|
class TestNetwork < Minitest::Test
|
5
5
|
include Helper
|
6
6
|
|
7
7
|
def after_setup
|
8
|
-
skip if !ENV[
|
8
|
+
skip if !ENV['NETWORK']
|
9
9
|
end
|
10
10
|
|
11
11
|
def test_get
|
12
|
-
response = client.get(
|
12
|
+
response = client.get('http://httpbin.org/get')
|
13
13
|
assert_equal 200, response.code
|
14
14
|
end
|
15
15
|
|
16
16
|
def test_timeout
|
17
17
|
e = assert_raises Chuckle::Error do
|
18
|
-
client(nretries: 0, timeout: 2).get(
|
18
|
+
client(nretries: 0, timeout: 2).get('http://httpbin.org/delay/3')
|
19
19
|
end
|
20
20
|
assert e.timeout?, "exception didn't indicate timeout"
|
21
21
|
end
|
22
22
|
|
23
23
|
def test_post
|
24
|
-
response = client.post(
|
25
|
-
assert_equal JSON.parse(response.body)[
|
24
|
+
response = client.post('http://httpbin.org/post', QUERY)
|
25
|
+
assert_equal JSON.parse(response.body)['form'], QUERY
|
26
26
|
end
|
27
27
|
|
28
28
|
def test_cookies
|
29
|
-
cookies = {
|
29
|
+
cookies = { 'a' => 'b' }
|
30
30
|
|
31
31
|
client(cookies: true, expires_in: 60) # set options
|
32
32
|
|
33
|
-
request = client.create_request(
|
33
|
+
request = client.create_request('http://httpbin.org/get')
|
34
34
|
cookie_jar = Chuckle::CookieJar.new(request).path
|
35
35
|
|
36
36
|
# make sure there are no cookies after the GET
|
@@ -39,23 +39,23 @@ class TestNetwork < Minitest::Test
|
|
39
39
|
|
40
40
|
# make sure there ARE cookies after a Set-Cookie
|
41
41
|
client.get("http://httpbin.org/cookies/set?#{Chuckle::Util.hash_to_query(cookies)}")
|
42
|
-
assert File.exist?(cookie_jar),
|
42
|
+
assert File.exist?(cookie_jar), 'cookie jar SHOULD exist now'
|
43
43
|
|
44
44
|
# make sure cookies come back from the server
|
45
|
-
response = client.get(
|
46
|
-
assert_equal JSON.parse(response.body)[
|
45
|
+
response = client.get('http://httpbin.org/cookies')
|
46
|
+
assert_equal JSON.parse(response.body)['cookies'], cookies
|
47
47
|
|
48
48
|
# Finally, test cache expiry on cookie_jar. Note that this has to
|
49
49
|
# be an un-cached URL, otherwise the cookie_jar never gets
|
50
50
|
# checked!
|
51
51
|
tm = Time.now - (client.expires_in + 9999)
|
52
52
|
File.utime(tm, tm, cookie_jar)
|
53
|
-
client.get(
|
53
|
+
client.get('http://httpbin.org/robots.txt')
|
54
54
|
assert !File.exist?(cookie_jar), "cookie jar should've expired"
|
55
55
|
end
|
56
56
|
|
57
57
|
def test_https
|
58
|
-
response = client.get(
|
58
|
+
response = client.get('https://httpbin.org/get')
|
59
59
|
assert_equal 200, response.code
|
60
60
|
end
|
61
61
|
|
@@ -66,7 +66,7 @@ class TestNetwork < Minitest::Test
|
|
66
66
|
# end
|
67
67
|
|
68
68
|
def test_bin
|
69
|
-
Dir.chdir(File.expand_path(
|
70
|
-
assert_command(
|
69
|
+
Dir.chdir(File.expand_path(__dir__))
|
70
|
+
assert_command('../bin/chuckle url_file.txt')
|
71
71
|
end
|
72
72
|
end
|
data/test/test_requests.rb
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
require
|
1
|
+
require 'helper'
|
2
2
|
|
3
3
|
class TestRequests < Minitest::Test
|
4
4
|
include Helper
|
@@ -20,14 +20,14 @@ class TestRequests < Minitest::Test
|
|
20
20
|
def test_302
|
21
21
|
response = mcurl(HTTP_302) { client.get(URL) }
|
22
22
|
assert_equal 200, response.code
|
23
|
-
assert_equal URI.parse(
|
23
|
+
assert_equal URI.parse('http://one'), response.uri
|
24
24
|
assert_equal "hello\n", response.body
|
25
25
|
end
|
26
26
|
|
27
27
|
def test_302_2
|
28
28
|
response = mcurl(HTTP_302_2) { client.get(URL) }
|
29
29
|
assert_equal 200, response.code
|
30
|
-
assert_equal URI.parse(
|
30
|
+
assert_equal URI.parse('http://two'), response.uri
|
31
31
|
assert_equal "hello\n", response.body
|
32
32
|
end
|
33
33
|
|
@@ -58,7 +58,7 @@ class TestRequests < Minitest::Test
|
|
58
58
|
|
59
59
|
def test_post
|
60
60
|
# just test hash_to_query first
|
61
|
-
assert_equal
|
61
|
+
assert_equal 'a=34&b=12&x+y=56', Chuckle::Util.hash_to_query(QUERY)
|
62
62
|
|
63
63
|
response = mcurl(HTTP_200) { client.post(URL, QUERY) }
|
64
64
|
assert_equal response.request.body, Chuckle::Util.hash_to_query(QUERY)
|
@@ -69,10 +69,10 @@ class TestRequests < Minitest::Test
|
|
69
69
|
|
70
70
|
def test_strange_urls
|
71
71
|
[
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
72
|
+
'gub',
|
73
|
+
'file://gub',
|
74
|
+
'file://gub/gub',
|
75
|
+
'file:///gub',
|
76
76
|
].each do |url|
|
77
77
|
response = mcurl(HTTP_200) { client.get(url) }
|
78
78
|
assert_equal 200, response.code
|
metadata
CHANGED
@@ -1,17 +1,17 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: chuckle
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.9
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Adam Doppelt
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2019-02-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
-
name:
|
14
|
+
name: optimist
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
17
|
- - ">="
|
@@ -103,6 +103,7 @@ extensions: []
|
|
103
103
|
extra_rdoc_files: []
|
104
104
|
files:
|
105
105
|
- ".gitignore"
|
106
|
+
- ".rubocop.yml"
|
106
107
|
- ".travis.yml"
|
107
108
|
- Gemfile
|
108
109
|
- LICENSE
|
@@ -138,7 +139,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
138
139
|
requirements:
|
139
140
|
- - ">="
|
140
141
|
- !ruby/object:Gem::Version
|
141
|
-
version: 1.
|
142
|
+
version: 2.1.0
|
142
143
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
143
144
|
requirements:
|
144
145
|
- - ">="
|
@@ -146,7 +147,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
146
147
|
version: '0'
|
147
148
|
requirements: []
|
148
149
|
rubyforge_project: chuckle
|
149
|
-
rubygems_version: 2.7.
|
150
|
+
rubygems_version: 2.7.6
|
150
151
|
signing_key:
|
151
152
|
specification_version: 4
|
152
153
|
summary: Chuckle - an http client that caches on disk.
|