chuckle 1.0.7 → 1.0.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +60 -0
- data/.travis.yml +0 -2
- data/Gemfile +1 -1
- data/Rakefile +6 -6
- data/bin/chuckle +7 -9
- data/chuckle.gemspec +17 -17
- data/lib/chuckle.rb +10 -10
- data/lib/chuckle/cache.rb +8 -8
- data/lib/chuckle/client.rb +5 -4
- data/lib/chuckle/cookie_jar.rb +1 -1
- data/lib/chuckle/curl.rb +24 -23
- data/lib/chuckle/options.rb +6 -6
- data/lib/chuckle/response.rb +1 -1
- data/lib/chuckle/util.rb +5 -5
- data/lib/chuckle/version.rb +1 -1
- data/test/helper.rb +21 -22
- data/test/test_cache.rb +8 -8
- data/test/test_headers.rb +17 -20
- data/test/test_network.rb +15 -15
- data/test/test_requests.rb +8 -8
- metadata +6 -5
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: b98846713307331d88cca61554f826cb93c558f51d479b2265196e737fe17754
|
4
|
+
data.tar.gz: 80536128f6942d4ad5dfff3de94c5077d15772a43cf54a2de5d8d26c75569e82
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 46035fb77f332b0b26ab314e4720097080c4d3bc98c7fe724704c0c288695bec0e5a101b52859d2aa8127bd67055f79aab8bdc26433ef4f7b2cc32b3b6ec5c24
|
7
|
+
data.tar.gz: 0f9fd286ca86253ee4db3e07eac090a8da0eaee2e9202a469aa819be5378019219eda660ec4e5b7a292f08206f43d6f3f83f82d00dcc950497ab01017c487f51
|
data/.rubocop.yml
ADDED
@@ -0,0 +1,60 @@
|
|
1
|
+
AllCops:
|
2
|
+
TargetRubyVersion: 2.1.0
|
3
|
+
|
4
|
+
# amd: customizations
|
5
|
+
Layout/SpaceInsideArrayLiteralBrackets:
|
6
|
+
EnforcedStyle: compact
|
7
|
+
Layout/CaseIndentation:
|
8
|
+
EnforcedStyle: end
|
9
|
+
Layout/EndAlignment:
|
10
|
+
EnforcedStyleAlignWith: variable
|
11
|
+
Layout/IndentHeredoc:
|
12
|
+
Enabled: false
|
13
|
+
Style/CollectionMethods:
|
14
|
+
Enabled: true
|
15
|
+
PreferredMethods:
|
16
|
+
reduce: inject
|
17
|
+
Style/TrailingCommaInArrayLiteral:
|
18
|
+
EnforcedStyleForMultiline: consistent_comma
|
19
|
+
Style/TrailingCommaInHashLiteral:
|
20
|
+
EnforcedStyleForMultiline: consistent_comma
|
21
|
+
|
22
|
+
# amd: these seem extreme
|
23
|
+
Lint/AssignmentInCondition: { Enabled: false } # I do this all the time
|
24
|
+
Lint/HandleExceptions: { Enabled: false } # blank rescues are useful
|
25
|
+
Naming/BinaryOperatorParameterName: { Enabled: false } # silly
|
26
|
+
Naming/HeredocDelimiterNaming: { Enabled: false } # silly
|
27
|
+
Naming/UncommunicativeMethodParamName: { Enabled: false } # silly
|
28
|
+
Performance/FixedSize: { Enabled: false } # silly
|
29
|
+
Performance/RegexpMatch: { Enabled: false } # =~ is fine
|
30
|
+
Performance/TimesMap: { Enabled: false } # silly
|
31
|
+
Rails/FilePath: { Enabled: false }
|
32
|
+
Rails/LexicallyScopedActionFilter: { Enabled: false }
|
33
|
+
Rails/OutputSafety: { Enabled: false }
|
34
|
+
Rails/ReversibleMigration: { Enabled: false } # who cares
|
35
|
+
Rails/SkipsModelValidations: { Enabled: false } # silly
|
36
|
+
Style/ClassAndModuleChildren: { Enabled: false } # silly
|
37
|
+
Style/Documentation: { Enabled: false } # we don't need this
|
38
|
+
Style/FormatStringToken: { Enabled: false } # we like printf here
|
39
|
+
Style/FrozenStringLiteralComment: { Enabled: false } # seems excessive
|
40
|
+
Style/GuardClause: { Enabled: false } # confusing
|
41
|
+
Style/IfUnlessModifier: { Enabled: false } # personally I hate unless
|
42
|
+
Style/NegatedIf: { Enabled: false } # these are fine
|
43
|
+
Style/Next: { Enabled: false } # these are fine
|
44
|
+
Style/NumericPredicate: { Enabled: false } # silly
|
45
|
+
Style/ParallelAssignment: { Enabled: false } # these are fine
|
46
|
+
Style/PerlBackrefs: { Enabled: false } # these are fine
|
47
|
+
Style/RaiseArgs: { Enabled: false } # silly
|
48
|
+
Style/RegexpLiteral: { Enabled: false } # these are fine
|
49
|
+
|
50
|
+
# amd: these Metric rules are annoying, disable
|
51
|
+
Metrics/AbcSize: { Enabled: false }
|
52
|
+
Metrics/BlockLength: { Enabled: false }
|
53
|
+
Metrics/BlockNesting: { Enabled: false }
|
54
|
+
Metrics/ClassLength: { Enabled: false }
|
55
|
+
Metrics/CyclomaticComplexity: { Enabled: false }
|
56
|
+
Metrics/LineLength: { Enabled: false }
|
57
|
+
Metrics/MethodLength: { Enabled: false }
|
58
|
+
Metrics/ModuleLength: { Enabled: false }
|
59
|
+
Metrics/ParameterLists: { Enabled: false }
|
60
|
+
Metrics/PerceivedComplexity: { Enabled: false }
|
data/.travis.yml
CHANGED
data/Gemfile
CHANGED
@@ -1,2 +1,2 @@
|
|
1
|
-
source
|
1
|
+
source 'http://rubygems.org'
|
2
2
|
gemspec
|
data/Rakefile
CHANGED
@@ -1,6 +1,6 @@
|
|
1
|
-
require
|
2
|
-
require
|
3
|
-
require
|
1
|
+
require 'bundler/gem_helper'
|
2
|
+
require 'rake/testtask'
|
3
|
+
require 'rdoc/task'
|
4
4
|
|
5
5
|
Bundler::GemHelper.install_tasks
|
6
6
|
|
@@ -9,7 +9,7 @@ Bundler::GemHelper.install_tasks
|
|
9
9
|
#
|
10
10
|
|
11
11
|
Rake::TestTask.new(:test) do |test|
|
12
|
-
test.libs <<
|
12
|
+
test.libs << 'test'
|
13
13
|
end
|
14
14
|
task default: :test
|
15
15
|
|
@@ -18,7 +18,7 @@ task default: :test
|
|
18
18
|
#
|
19
19
|
|
20
20
|
RDoc::Task.new do |rdoc|
|
21
|
-
rdoc.rdoc_dir =
|
21
|
+
rdoc.rdoc_dir = 'rdoc'
|
22
22
|
rdoc.title = "chuckle #{Chuckle::VERSION}"
|
23
|
-
rdoc.rdoc_files.include(
|
23
|
+
rdoc.rdoc_files.include('lib/**/*.rb')
|
24
24
|
end
|
data/bin/chuckle
CHANGED
@@ -3,8 +3,8 @@
|
|
3
3
|
# in lieu of -w, since we're using env to startup
|
4
4
|
$VERBOSE = true
|
5
5
|
|
6
|
-
require
|
7
|
-
require
|
6
|
+
require 'chuckle'
|
7
|
+
require 'optimist'
|
8
8
|
|
9
9
|
class Main
|
10
10
|
attr_accessor :options
|
@@ -13,7 +13,7 @@ class Main
|
|
13
13
|
self.options = options
|
14
14
|
list = IO.readlines(options[:file]).map(&:chomp)
|
15
15
|
list.each_with_index do |i, index|
|
16
|
-
puts "#{index+1}/#{list.length}. #{i}"
|
16
|
+
puts "#{index + 1}/#{list.length}. #{i}"
|
17
17
|
begin
|
18
18
|
chuckle.get(i)
|
19
19
|
rescue Chuckle::Error => e
|
@@ -29,13 +29,11 @@ class Main
|
|
29
29
|
end
|
30
30
|
end
|
31
31
|
|
32
|
-
|
33
|
-
|
34
32
|
#
|
35
33
|
# parse command line
|
36
34
|
#
|
37
35
|
|
38
|
-
options =
|
36
|
+
options = Optimist::options do
|
39
37
|
banner <<EOF
|
40
38
|
Usage: chuckle <url_file>
|
41
39
|
|
@@ -43,10 +41,10 @@ chuckle fetches urls as listed in url_file. It just populate the disk
|
|
43
41
|
cache.
|
44
42
|
EOF
|
45
43
|
|
46
|
-
opt :nretries,
|
47
|
-
opt :timeout,
|
44
|
+
opt :nretries, 'Number of retries per URL', type: Integer
|
45
|
+
opt :timeout, 'Timeout per retry', type: Integer
|
48
46
|
end
|
49
47
|
|
50
|
-
|
48
|
+
Optimist::die 'need <url_file>' if ARGV.length != 1
|
51
49
|
options[:file] = ARGV.shift
|
52
50
|
Main.new(options)
|
data/chuckle.gemspec
CHANGED
@@ -1,30 +1,30 @@
|
|
1
|
-
$LOAD_PATH << File.expand_path(
|
1
|
+
$LOAD_PATH << File.expand_path('lib', __dir__)
|
2
2
|
|
3
|
-
require
|
3
|
+
require 'chuckle/version'
|
4
4
|
|
5
5
|
Gem::Specification.new do |s|
|
6
|
-
s.name =
|
6
|
+
s.name = 'chuckle'
|
7
7
|
s.version = Chuckle::VERSION
|
8
8
|
s.platform = Gem::Platform::RUBY
|
9
|
-
s.required_ruby_version =
|
10
|
-
s.authors = [
|
11
|
-
s.email = [
|
12
|
-
s.homepage =
|
13
|
-
s.summary =
|
14
|
-
s.description =
|
9
|
+
s.required_ruby_version = '>= 2.1.0'
|
10
|
+
s.authors = [ 'Adam Doppelt' ]
|
11
|
+
s.email = [ 'amd@gurge.com' ]
|
12
|
+
s.homepage = 'http://github.com/gurgeous/chuckle'
|
13
|
+
s.summary = 'Chuckle - an http client that caches on disk.'
|
14
|
+
s.description = 'An http client that caches on disk.'
|
15
15
|
|
16
|
-
s.rubyforge_project =
|
16
|
+
s.rubyforge_project = 'chuckle'
|
17
17
|
|
18
|
-
s.add_runtime_dependency
|
18
|
+
s.add_runtime_dependency 'optimist'
|
19
19
|
|
20
|
-
s.add_development_dependency
|
21
|
-
s.add_development_dependency
|
22
|
-
s.add_development_dependency
|
23
|
-
s.add_development_dependency
|
24
|
-
s.add_development_dependency
|
20
|
+
s.add_development_dependency 'awesome_print'
|
21
|
+
s.add_development_dependency 'json'
|
22
|
+
s.add_development_dependency 'minitest', '~> 5.0'
|
23
|
+
s.add_development_dependency 'rake'
|
24
|
+
s.add_development_dependency 'rdoc'
|
25
25
|
|
26
26
|
s.files = `git ls-files`.split("\n")
|
27
27
|
s.test_files = `git ls-files -- test/*`.split("\n")
|
28
28
|
s.executables = `git ls-files -- bin/*`.split("\n").map { |i| File.basename(i) }
|
29
|
-
s.require_paths = [
|
29
|
+
s.require_paths = [ 'lib' ]
|
30
30
|
end
|
data/lib/chuckle.rb
CHANGED
@@ -1,12 +1,12 @@
|
|
1
|
-
require
|
2
|
-
require
|
3
|
-
require
|
4
|
-
require
|
5
|
-
require
|
6
|
-
require
|
7
|
-
require
|
8
|
-
require
|
1
|
+
require 'chuckle/cache'
|
2
|
+
require 'chuckle/cookie_jar'
|
3
|
+
require 'chuckle/error'
|
4
|
+
require 'chuckle/options'
|
5
|
+
require 'chuckle/request'
|
6
|
+
require 'chuckle/response'
|
7
|
+
require 'chuckle/util'
|
8
|
+
require 'chuckle/version'
|
9
9
|
|
10
10
|
# these last
|
11
|
-
require
|
12
|
-
require
|
11
|
+
require 'chuckle/curl'
|
12
|
+
require 'chuckle/client'
|
data/lib/chuckle/cache.rb
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
require
|
1
|
+
require 'fileutils'
|
2
2
|
|
3
3
|
module Chuckle
|
4
4
|
class Cache
|
@@ -21,7 +21,7 @@ module Chuckle
|
|
21
21
|
|
22
22
|
def set(request, curl)
|
23
23
|
# mkdirs
|
24
|
-
FileUtils.mkdir_p([File.dirname(request.headers_path), File.dirname(request.body_path)])
|
24
|
+
FileUtils.mkdir_p([ File.dirname(request.headers_path), File.dirname(request.body_path) ])
|
25
25
|
|
26
26
|
# now mv
|
27
27
|
FileUtils.mv(curl.headers_path, request.headers_path)
|
@@ -53,7 +53,7 @@ module Chuckle
|
|
53
53
|
|
54
54
|
# calculate body_path
|
55
55
|
s = @client.cache_dir
|
56
|
-
s = "#{s}/#{pathify(uri.host ||
|
56
|
+
s = "#{s}/#{pathify(uri.host || 'file')}"
|
57
57
|
s = "#{s}/#{pathify(uri.path)}"
|
58
58
|
if uri.query
|
59
59
|
q = "?#{uri.query}"
|
@@ -76,15 +76,15 @@ module Chuckle
|
|
76
76
|
|
77
77
|
# turn s into a string that can be a path
|
78
78
|
def pathify(s)
|
79
|
-
s = s.gsub(/^\//,
|
80
|
-
s = s.gsub(
|
81
|
-
s = s.gsub(/[?\/&]/,
|
79
|
+
s = s.gsub(/^\//, '')
|
80
|
+
s = s.gsub('..', ',')
|
81
|
+
s = s.gsub(/[?\/&]/, ',')
|
82
82
|
s = s.gsub(/[^A-Za-z0-9_.,=%-]/) do |i|
|
83
|
-
hex = i.unpack(
|
83
|
+
hex = i.unpack('H2').first
|
84
84
|
"%#{hex}"
|
85
85
|
end
|
86
86
|
s = s.downcase
|
87
|
-
s =
|
87
|
+
s = '_root_' if s.empty?
|
88
88
|
s
|
89
89
|
end
|
90
90
|
end
|
data/lib/chuckle/client.rb
CHANGED
@@ -1,4 +1,5 @@
|
|
1
|
-
require
|
1
|
+
require 'English'
|
2
|
+
require 'uri'
|
2
3
|
|
3
4
|
module Chuckle
|
4
5
|
class Client
|
@@ -8,7 +9,7 @@ module Chuckle
|
|
8
9
|
|
9
10
|
def initialize(options = {})
|
10
11
|
self.options = DEFAULT_OPTIONS.dup
|
11
|
-
options.each { |k, v| self.options[k] = v if v
|
12
|
+
options.each { |k, v| self.options[k] = v if !v.nil? }
|
12
13
|
self.cache = Cache.new(self)
|
13
14
|
sanity!
|
14
15
|
end
|
@@ -50,8 +51,8 @@ module Chuckle
|
|
50
51
|
|
51
52
|
# make sure curl command exists
|
52
53
|
def sanity!
|
53
|
-
system(
|
54
|
-
raise
|
54
|
+
system('which curl > /dev/null')
|
55
|
+
raise 'Chuckle requires curl. Please install it.' if $CHILD_STATUS != 0
|
55
56
|
end
|
56
57
|
|
57
58
|
def curl(request)
|
data/lib/chuckle/cookie_jar.rb
CHANGED
data/lib/chuckle/curl.rb
CHANGED
@@ -1,4 +1,5 @@
|
|
1
|
-
require
|
1
|
+
require 'English'
|
2
|
+
require 'fileutils'
|
2
3
|
|
3
4
|
module Chuckle
|
4
5
|
class Curl
|
@@ -22,9 +23,9 @@ module Chuckle
|
|
22
23
|
Kernel.system(*command)
|
23
24
|
|
24
25
|
# capture exit code, bail on INT
|
25
|
-
exit_code =
|
26
|
-
if
|
27
|
-
Process.kill(:INT,
|
26
|
+
exit_code = $CHILD_STATUS.to_i / 256
|
27
|
+
if $CHILD_STATUS.termsig == Signal.list['INT']
|
28
|
+
Process.kill(:INT, $PROCESS_ID)
|
28
29
|
end
|
29
30
|
|
30
31
|
# create tmp files if there were errors
|
@@ -59,41 +60,41 @@ module Chuckle
|
|
59
60
|
def command(request)
|
60
61
|
client = request.client
|
61
62
|
|
62
|
-
command = [
|
63
|
-
command <<
|
64
|
-
command <<
|
63
|
+
command = [ 'curl' ]
|
64
|
+
command << '--silent'
|
65
|
+
command << '--compressed'
|
65
66
|
|
66
|
-
command += [
|
67
|
-
command += [
|
68
|
-
command += [
|
69
|
-
command += [
|
67
|
+
command += [ '--user-agent', client.user_agent ]
|
68
|
+
command += [ '--max-time', client.timeout ]
|
69
|
+
command += [ '--retry', client.nretries ]
|
70
|
+
command += [ '--location', '--max-redirs', 3 ]
|
70
71
|
|
71
72
|
if request.body
|
72
|
-
command += [
|
73
|
+
command += [ '--data-binary', request.body ]
|
73
74
|
end
|
74
75
|
|
75
76
|
# maintain backwards compatibility for content type
|
76
|
-
client.headers.each do |key,value|
|
77
|
-
if key ==
|
78
|
-
command += [
|
77
|
+
client.headers.each do |key, value|
|
78
|
+
if key == 'Content-Type'
|
79
|
+
command += [ '--header', "#{key}: #{value}" ] if request.body
|
79
80
|
else
|
80
|
-
command += [
|
81
|
+
command += [ '--header', "#{key}: #{value}" ]
|
81
82
|
end
|
82
83
|
end
|
83
84
|
|
84
85
|
if client.cookies?
|
85
86
|
cookie_jar.preflight
|
86
|
-
command += [
|
87
|
-
command += [
|
87
|
+
command += [ '--cookie', cookie_jar.path ]
|
88
|
+
command += [ '--cookie-jar', cookie_jar.path ]
|
88
89
|
end
|
89
90
|
|
90
91
|
# SSL options
|
91
|
-
command += [
|
92
|
-
command += [
|
93
|
-
command += [
|
92
|
+
command += [ '--cacert', client.cacert ] if client.cacert
|
93
|
+
command += [ '--capath', client.capath ] if client.capath
|
94
|
+
command += [ '--insecure' ] if client.insecure?
|
94
95
|
|
95
|
-
command += [
|
96
|
-
command += [
|
96
|
+
command += [ '--dump-header', headers_path ]
|
97
|
+
command += [ '--output', body_path ]
|
97
98
|
|
98
99
|
command << request.uri
|
99
100
|
|
data/lib/chuckle/options.rb
CHANGED
@@ -5,7 +5,7 @@ module Chuckle
|
|
5
5
|
cache_errors: true,
|
6
6
|
cacert: nil,
|
7
7
|
capath: nil,
|
8
|
-
content_type:
|
8
|
+
content_type: 'application/x-www-form-urlencoded',
|
9
9
|
cookies: false,
|
10
10
|
expires_in: :never,
|
11
11
|
headers: nil,
|
@@ -13,22 +13,22 @@ module Chuckle
|
|
13
13
|
nretries: 2,
|
14
14
|
rate_limit: 1,
|
15
15
|
timeout: 30,
|
16
|
-
user_agent:
|
16
|
+
user_agent: 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0',
|
17
17
|
verbose: false,
|
18
|
-
}
|
18
|
+
}.freeze
|
19
19
|
|
20
20
|
# cache root directory
|
21
21
|
def cache_dir
|
22
22
|
@cache_dir ||= begin
|
23
23
|
dir = options[:cache_dir]
|
24
24
|
dir ||= begin
|
25
|
-
if home = ENV[
|
25
|
+
if home = ENV['HOME']
|
26
26
|
if File.exist?(home) && File.stat(home).writable?
|
27
27
|
"#{home}/.chuckle"
|
28
28
|
end
|
29
29
|
end
|
30
30
|
end
|
31
|
-
dir ||=
|
31
|
+
dir ||= '/tmp/chuckle'
|
32
32
|
dir
|
33
33
|
end
|
34
34
|
end
|
@@ -66,7 +66,7 @@ module Chuckle
|
|
66
66
|
def headers
|
67
67
|
@headers ||= begin
|
68
68
|
headers = options[:headers] || {}
|
69
|
-
headers[
|
69
|
+
headers['Content-Type'] = options[:content_type] if options[:content_type]
|
70
70
|
headers
|
71
71
|
end
|
72
72
|
end
|
data/lib/chuckle/response.rb
CHANGED
data/lib/chuckle/util.rb
CHANGED
@@ -1,9 +1,9 @@
|
|
1
|
-
require
|
2
|
-
require
|
1
|
+
require 'digest/md5'
|
2
|
+
require 'tempfile'
|
3
3
|
|
4
4
|
module Chuckle
|
5
5
|
module Util
|
6
|
-
|
6
|
+
module_function
|
7
7
|
|
8
8
|
def hash_to_query(hash)
|
9
9
|
q = hash.map do |key, value|
|
@@ -11,7 +11,7 @@ module Chuckle
|
|
11
11
|
value = CGI.escape(value.to_s)
|
12
12
|
"#{key}=#{value}"
|
13
13
|
end
|
14
|
-
q.sort.join(
|
14
|
+
q.sort.join('&')
|
15
15
|
end
|
16
16
|
|
17
17
|
def md5(s)
|
@@ -23,7 +23,7 @@ module Chuckle
|
|
23
23
|
end
|
24
24
|
|
25
25
|
def tmp_path
|
26
|
-
Tempfile.open(
|
26
|
+
Tempfile.open('chuckle') do |f|
|
27
27
|
path = f.path
|
28
28
|
f.unlink
|
29
29
|
path
|
data/lib/chuckle/version.rb
CHANGED
data/test/helper.rb
CHANGED
@@ -1,39 +1,38 @@
|
|
1
|
-
require
|
2
|
-
require
|
3
|
-
require
|
4
|
-
require
|
5
|
-
require
|
1
|
+
require 'awesome_print'
|
2
|
+
require 'chuckle'
|
3
|
+
require 'json'
|
4
|
+
require 'minitest/autorun'
|
5
|
+
require 'minitest/pride'
|
6
6
|
|
7
7
|
module Helper
|
8
|
-
CACHE_DIR =
|
9
|
-
URL =
|
10
|
-
QUERY = {
|
11
|
-
TMP =
|
8
|
+
CACHE_DIR = '/tmp/_chuckle_tests'.freeze
|
9
|
+
URL = 'http://chuckle'.freeze
|
10
|
+
QUERY = { 'b' => '12', 'a' => '34', 'x y' => '56' }.freeze
|
11
|
+
TMP = '/tmp/_chuckle_tmp.txt'.freeze
|
12
12
|
|
13
13
|
#
|
14
14
|
# fake responses
|
15
15
|
#
|
16
16
|
|
17
|
-
HTTP_200 = <<-EOF.gsub(/(^|\n) +/,
|
17
|
+
HTTP_200 = <<-EOF.gsub(/(^|\n) +/, '\\1')
|
18
18
|
HTTP/1.1 200 OK
|
19
19
|
|
20
20
|
hello
|
21
21
|
EOF
|
22
22
|
|
23
|
-
HTTP_200_ALTERNATE = <<-EOF.gsub(/(^|\n) +/,
|
23
|
+
HTTP_200_ALTERNATE = <<-EOF.gsub(/(^|\n) +/, '\\1')
|
24
24
|
HTTP/1.1 200 OK
|
25
25
|
|
26
26
|
alternate
|
27
27
|
EOF
|
28
28
|
|
29
|
-
|
30
|
-
HTTP2_200 = <<-EOF.gsub(/(^|\n) +/, "\\1")
|
29
|
+
HTTP2_200 = <<-EOF.gsub(/(^|\n) +/, '\\1')
|
31
30
|
HTTP/2 200 OK
|
32
31
|
|
33
32
|
hello
|
34
33
|
EOF
|
35
34
|
|
36
|
-
HTTP_302 = <<-EOF.gsub(/(^|\n) +/,
|
35
|
+
HTTP_302 = <<-EOF.gsub(/(^|\n) +/, '\\1')
|
37
36
|
HTTP/1.1 302 FOUND
|
38
37
|
Location: http://one
|
39
38
|
|
@@ -42,7 +41,7 @@ module Helper
|
|
42
41
|
hello
|
43
42
|
EOF
|
44
43
|
|
45
|
-
HTTP_302_2 = <<-EOF.gsub(/(^|\n) +/,
|
44
|
+
HTTP_302_2 = <<-EOF.gsub(/(^|\n) +/, '\\1')
|
46
45
|
HTTP/1.1 302 FOUND
|
47
46
|
Location: http://one
|
48
47
|
|
@@ -54,7 +53,7 @@ module Helper
|
|
54
53
|
hello
|
55
54
|
EOF
|
56
55
|
|
57
|
-
HTTP_302_RELATIVE = <<-EOF.gsub(/(^|\n) +/,
|
56
|
+
HTTP_302_RELATIVE = <<-EOF.gsub(/(^|\n) +/, '\\1')
|
58
57
|
HTTP/1.1 302 FOUND
|
59
58
|
Location: /two
|
60
59
|
|
@@ -63,7 +62,7 @@ module Helper
|
|
63
62
|
hello
|
64
63
|
EOF
|
65
64
|
|
66
|
-
HTTP_404 = <<-EOF.gsub(/(^|\n) +/,
|
65
|
+
HTTP_404 = <<-EOF.gsub(/(^|\n) +/, '\\1')
|
67
66
|
HTTP/1.1 404 Not Found
|
68
67
|
|
69
68
|
EOF
|
@@ -82,7 +81,7 @@ module Helper
|
|
82
81
|
end
|
83
82
|
|
84
83
|
# pretend to be curl by stubbing Kernel.system
|
85
|
-
def mcurl(response, exit_code = 0
|
84
|
+
def mcurl(response, exit_code = 0)
|
86
85
|
# divide response into headers/body
|
87
86
|
sep = response.rindex("\n\n") + "\n\n".length
|
88
87
|
body = response[sep..-1]
|
@@ -90,8 +89,8 @@ module Helper
|
|
90
89
|
|
91
90
|
# a lambda that pretends to be curl
|
92
91
|
fake_system = lambda do |*command|
|
93
|
-
tmp_headers = command[command.index(
|
94
|
-
tmp_body = command[command.index(
|
92
|
+
tmp_headers = command[command.index('--dump-header') + 1]
|
93
|
+
tmp_body = command[command.index('--output') + 1]
|
95
94
|
IO.write(tmp_headers, headers)
|
96
95
|
IO.write(tmp_body, body)
|
97
96
|
`(exit #{exit_code})`
|
@@ -101,7 +100,7 @@ module Helper
|
|
101
100
|
Kernel.stub(:system, fake_system) { yield }
|
102
101
|
end
|
103
102
|
|
104
|
-
def assert_if_system
|
103
|
+
def assert_if_system
|
105
104
|
fake_system = lambda do |*command|
|
106
105
|
assert false, "system called with #{command.inspect}"
|
107
106
|
end
|
@@ -110,6 +109,6 @@ module Helper
|
|
110
109
|
|
111
110
|
def assert_command(cmd)
|
112
111
|
system("#{cmd} > #{TMP} 2>&1")
|
113
|
-
assert(
|
112
|
+
assert($CHILD_STATUS == 0, File.read(TMP))
|
114
113
|
end
|
115
114
|
end
|
data/test/test_cache.rb
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
require
|
1
|
+
require 'helper'
|
2
2
|
|
3
3
|
class TestCache < Minitest::Test
|
4
4
|
include Helper
|
@@ -10,16 +10,16 @@ class TestCache < Minitest::Test
|
|
10
10
|
# exists? and expired? (and clear)
|
11
11
|
def test_predicates
|
12
12
|
request = client.create_request(URL)
|
13
|
-
assert !client.cache.exists?(request),
|
13
|
+
assert !client.cache.exists?(request), 'uncache! uri said it was cached'
|
14
14
|
|
15
15
|
mcurl(HTTP_200) do
|
16
16
|
client.run(request)
|
17
17
|
end
|
18
18
|
assert client.cache.exists?(request), "cache said it wasn't cached"
|
19
|
-
assert !client.cache.expired?(request),
|
19
|
+
assert !client.cache.expired?(request), 'cache said it was expired'
|
20
20
|
|
21
21
|
client.cache.clear(request)
|
22
|
-
assert !client.cache.exists?(request),
|
22
|
+
assert !client.cache.exists?(request), 'still cached after clear'
|
23
23
|
end
|
24
24
|
|
25
25
|
# cache expiration
|
@@ -71,7 +71,7 @@ class TestCache < Minitest::Test
|
|
71
71
|
mcurl(HTTP_404) do
|
72
72
|
client.get(URL)
|
73
73
|
end
|
74
|
-
rescue Chuckle::Error
|
74
|
+
rescue Chuckle::Error
|
75
75
|
end
|
76
76
|
|
77
77
|
# cache hit
|
@@ -91,7 +91,7 @@ class TestCache < Minitest::Test
|
|
91
91
|
mcurl(HTTP_404, Chuckle::Error::CURL_TIMEOUT) do
|
92
92
|
client.get(URL)
|
93
93
|
end
|
94
|
-
rescue Chuckle::Error
|
94
|
+
rescue Chuckle::Error
|
95
95
|
end
|
96
96
|
|
97
97
|
# cache hit
|
@@ -123,9 +123,9 @@ class TestCache < Minitest::Test
|
|
123
123
|
end
|
124
124
|
|
125
125
|
def test_long_url
|
126
|
-
words = %w
|
126
|
+
words = %w[the quick brown fox jumped over the lazy dog]
|
127
127
|
query = (1..100).map { words[rand(words.length)] }
|
128
|
-
query = query.each_slice(2).map { |i| i.join(
|
128
|
+
query = query.each_slice(2).map { |i| i.join('=') }.join('&')
|
129
129
|
|
130
130
|
# cache miss
|
131
131
|
mcurl(HTTP_200) do
|
data/test/test_headers.rb
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
require
|
1
|
+
require 'helper'
|
2
2
|
|
3
3
|
# these actually use the network, and get skipped unless ENV["NETWORK"].
|
4
4
|
class TestHeaders < Minitest::Test
|
@@ -6,45 +6,43 @@ class TestHeaders < Minitest::Test
|
|
6
6
|
|
7
7
|
def test_default_headers
|
8
8
|
test_client = client(verbose: true)
|
9
|
-
assert_equal({
|
9
|
+
assert_equal({ 'Content-Type' => 'application/x-www-form-urlencoded' }, test_client.headers)
|
10
10
|
end
|
11
11
|
|
12
12
|
def test_content_type
|
13
|
-
test_client = client(content_type:
|
14
|
-
assert_equal({
|
13
|
+
test_client = client(content_type: 'text/json')
|
14
|
+
assert_equal({ 'Content-Type' => 'text/json' }, test_client.headers)
|
15
15
|
end
|
16
16
|
|
17
17
|
def test_arbitrary_headers
|
18
|
-
test_client = client(headers: {
|
19
|
-
assert_equal test_client.headers[
|
20
|
-
assert_equal test_client.headers[
|
21
|
-
|
18
|
+
test_client = client(headers: { 'Referer' => 'http://foo.com' })
|
19
|
+
assert_equal test_client.headers['Referer'], 'http://foo.com'
|
20
|
+
assert_equal test_client.headers['Content-Type'], 'application/x-www-form-urlencoded'
|
22
21
|
end
|
23
22
|
|
24
23
|
def test_basic_get
|
25
24
|
test_client = client(verbose: true)
|
26
|
-
request = test_client.create_request(
|
25
|
+
request = test_client.create_request('http://httpbin.org/get')
|
27
26
|
headers = command_headers(request)
|
28
27
|
assert_equal headers, []
|
29
28
|
end
|
30
29
|
|
31
|
-
|
32
30
|
def test_basic_post
|
33
|
-
request = client(verbose: true).create_request(
|
31
|
+
request = client(verbose: true).create_request('http://httpbin.org/post', 'foo')
|
34
32
|
headers = command_headers(request)
|
35
|
-
assert_equal headers, [
|
33
|
+
assert_equal headers, [ 'Content-Type: application/x-www-form-urlencoded' ]
|
36
34
|
end
|
37
35
|
|
38
36
|
def test_get_headers
|
39
|
-
request = client(headers: {
|
37
|
+
request = client(headers: { 'Referer' => 'http://foo.com' }).create_request('http://httpbin.org/get')
|
40
38
|
headers = command_headers(request)
|
41
|
-
assert_equal headers, [
|
39
|
+
assert_equal headers, [ 'Referer: http://foo.com' ]
|
42
40
|
end
|
43
41
|
|
44
42
|
def test_post_headers
|
45
|
-
request = client(headers: {
|
43
|
+
request = client(headers: { 'Referer' => 'http://foo.com' }).create_request('http://httpbin.org/post', 'foo')
|
46
44
|
headers = command_headers(request)
|
47
|
-
assert_equal headers.sort, [
|
45
|
+
assert_equal headers.sort, [ 'Content-Type: application/x-www-form-urlencoded', 'Referer: http://foo.com' ]
|
48
46
|
end
|
49
47
|
|
50
48
|
private
|
@@ -58,8 +56,8 @@ class TestHeaders < Minitest::Test
|
|
58
56
|
headers = []
|
59
57
|
test_command = command(request)
|
60
58
|
test_command.each_with_index do |item, i|
|
61
|
-
if item ==
|
62
|
-
headers << test_command[i+1]
|
59
|
+
if item == '--header'
|
60
|
+
headers << test_command[i + 1]
|
63
61
|
end
|
64
62
|
end
|
65
63
|
headers
|
@@ -70,5 +68,4 @@ class TestHeaders < Minitest::Test
|
|
70
68
|
command(@request)
|
71
69
|
end
|
72
70
|
end
|
73
|
-
|
74
|
-
end
|
71
|
+
end
|
data/test/test_network.rb
CHANGED
@@ -1,36 +1,36 @@
|
|
1
|
-
require
|
1
|
+
require 'helper'
|
2
2
|
|
3
3
|
# these actually use the network, and get skipped unless ENV["NETWORK"].
|
4
4
|
class TestNetwork < Minitest::Test
|
5
5
|
include Helper
|
6
6
|
|
7
7
|
def after_setup
|
8
|
-
skip if !ENV[
|
8
|
+
skip if !ENV['NETWORK']
|
9
9
|
end
|
10
10
|
|
11
11
|
def test_get
|
12
|
-
response = client.get(
|
12
|
+
response = client.get('http://httpbin.org/get')
|
13
13
|
assert_equal 200, response.code
|
14
14
|
end
|
15
15
|
|
16
16
|
def test_timeout
|
17
17
|
e = assert_raises Chuckle::Error do
|
18
|
-
client(nretries: 0, timeout: 2).get(
|
18
|
+
client(nretries: 0, timeout: 2).get('http://httpbin.org/delay/3')
|
19
19
|
end
|
20
20
|
assert e.timeout?, "exception didn't indicate timeout"
|
21
21
|
end
|
22
22
|
|
23
23
|
def test_post
|
24
|
-
response = client.post(
|
25
|
-
assert_equal JSON.parse(response.body)[
|
24
|
+
response = client.post('http://httpbin.org/post', QUERY)
|
25
|
+
assert_equal JSON.parse(response.body)['form'], QUERY
|
26
26
|
end
|
27
27
|
|
28
28
|
def test_cookies
|
29
|
-
cookies = {
|
29
|
+
cookies = { 'a' => 'b' }
|
30
30
|
|
31
31
|
client(cookies: true, expires_in: 60) # set options
|
32
32
|
|
33
|
-
request = client.create_request(
|
33
|
+
request = client.create_request('http://httpbin.org/get')
|
34
34
|
cookie_jar = Chuckle::CookieJar.new(request).path
|
35
35
|
|
36
36
|
# make sure there are no cookies after the GET
|
@@ -39,23 +39,23 @@ class TestNetwork < Minitest::Test
|
|
39
39
|
|
40
40
|
# make sure there ARE cookies after a Set-Cookie
|
41
41
|
client.get("http://httpbin.org/cookies/set?#{Chuckle::Util.hash_to_query(cookies)}")
|
42
|
-
assert File.exist?(cookie_jar),
|
42
|
+
assert File.exist?(cookie_jar), 'cookie jar SHOULD exist now'
|
43
43
|
|
44
44
|
# make sure cookies come back from the server
|
45
|
-
response = client.get(
|
46
|
-
assert_equal JSON.parse(response.body)[
|
45
|
+
response = client.get('http://httpbin.org/cookies')
|
46
|
+
assert_equal JSON.parse(response.body)['cookies'], cookies
|
47
47
|
|
48
48
|
# Finally, test cache expiry on cookie_jar. Note that this has to
|
49
49
|
# be an un-cached URL, otherwise the cookie_jar never gets
|
50
50
|
# checked!
|
51
51
|
tm = Time.now - (client.expires_in + 9999)
|
52
52
|
File.utime(tm, tm, cookie_jar)
|
53
|
-
client.get(
|
53
|
+
client.get('http://httpbin.org/robots.txt')
|
54
54
|
assert !File.exist?(cookie_jar), "cookie jar should've expired"
|
55
55
|
end
|
56
56
|
|
57
57
|
def test_https
|
58
|
-
response = client.get(
|
58
|
+
response = client.get('https://httpbin.org/get')
|
59
59
|
assert_equal 200, response.code
|
60
60
|
end
|
61
61
|
|
@@ -66,7 +66,7 @@ class TestNetwork < Minitest::Test
|
|
66
66
|
# end
|
67
67
|
|
68
68
|
def test_bin
|
69
|
-
Dir.chdir(File.expand_path(
|
70
|
-
assert_command(
|
69
|
+
Dir.chdir(File.expand_path(__dir__))
|
70
|
+
assert_command('../bin/chuckle url_file.txt')
|
71
71
|
end
|
72
72
|
end
|
data/test/test_requests.rb
CHANGED
@@ -1,4 +1,4 @@
|
|
1
|
-
require
|
1
|
+
require 'helper'
|
2
2
|
|
3
3
|
class TestRequests < Minitest::Test
|
4
4
|
include Helper
|
@@ -20,14 +20,14 @@ class TestRequests < Minitest::Test
|
|
20
20
|
def test_302
|
21
21
|
response = mcurl(HTTP_302) { client.get(URL) }
|
22
22
|
assert_equal 200, response.code
|
23
|
-
assert_equal URI.parse(
|
23
|
+
assert_equal URI.parse('http://one'), response.uri
|
24
24
|
assert_equal "hello\n", response.body
|
25
25
|
end
|
26
26
|
|
27
27
|
def test_302_2
|
28
28
|
response = mcurl(HTTP_302_2) { client.get(URL) }
|
29
29
|
assert_equal 200, response.code
|
30
|
-
assert_equal URI.parse(
|
30
|
+
assert_equal URI.parse('http://two'), response.uri
|
31
31
|
assert_equal "hello\n", response.body
|
32
32
|
end
|
33
33
|
|
@@ -58,7 +58,7 @@ class TestRequests < Minitest::Test
|
|
58
58
|
|
59
59
|
def test_post
|
60
60
|
# just test hash_to_query first
|
61
|
-
assert_equal
|
61
|
+
assert_equal 'a=34&b=12&x+y=56', Chuckle::Util.hash_to_query(QUERY)
|
62
62
|
|
63
63
|
response = mcurl(HTTP_200) { client.post(URL, QUERY) }
|
64
64
|
assert_equal response.request.body, Chuckle::Util.hash_to_query(QUERY)
|
@@ -69,10 +69,10 @@ class TestRequests < Minitest::Test
|
|
69
69
|
|
70
70
|
def test_strange_urls
|
71
71
|
[
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
72
|
+
'gub',
|
73
|
+
'file://gub',
|
74
|
+
'file://gub/gub',
|
75
|
+
'file:///gub',
|
76
76
|
].each do |url|
|
77
77
|
response = mcurl(HTTP_200) { client.get(url) }
|
78
78
|
assert_equal 200, response.code
|
metadata
CHANGED
@@ -1,17 +1,17 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: chuckle
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.9
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Adam Doppelt
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2019-02-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
-
name:
|
14
|
+
name: optimist
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
17
|
- - ">="
|
@@ -103,6 +103,7 @@ extensions: []
|
|
103
103
|
extra_rdoc_files: []
|
104
104
|
files:
|
105
105
|
- ".gitignore"
|
106
|
+
- ".rubocop.yml"
|
106
107
|
- ".travis.yml"
|
107
108
|
- Gemfile
|
108
109
|
- LICENSE
|
@@ -138,7 +139,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
138
139
|
requirements:
|
139
140
|
- - ">="
|
140
141
|
- !ruby/object:Gem::Version
|
141
|
-
version: 1.
|
142
|
+
version: 2.1.0
|
142
143
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
143
144
|
requirements:
|
144
145
|
- - ">="
|
@@ -146,7 +147,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
146
147
|
version: '0'
|
147
148
|
requirements: []
|
148
149
|
rubyforge_project: chuckle
|
149
|
-
rubygems_version: 2.7.
|
150
|
+
rubygems_version: 2.7.6
|
150
151
|
signing_key:
|
151
152
|
specification_version: 4
|
152
153
|
summary: Chuckle - an http client that caches on disk.
|