httpdisk 0.2.0 → 0.5.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +9 -0
- data/Gemfile.lock +16 -4
- data/README.md +32 -6
- data/Rakefile +14 -11
- data/bin/httpdisk +9 -7
- data/bin/httpdisk-grep +46 -0
- data/httpdisk.gemspec +1 -0
- data/lib/httpdisk.rb +10 -5
- data/lib/httpdisk/cache.rb +31 -21
- data/lib/httpdisk/cache_key.rb +15 -6
- data/lib/httpdisk/cli/args.rb +57 -0
- data/lib/httpdisk/cli/main.rb +169 -0
- data/lib/httpdisk/client.rb +82 -19
- data/lib/httpdisk/error.rb +4 -0
- data/lib/httpdisk/grep/args.rb +35 -0
- data/lib/httpdisk/grep/main.rb +112 -0
- data/lib/httpdisk/grep/printer.rb +99 -0
- data/lib/httpdisk/payload.rb +7 -5
- data/lib/httpdisk/slop_duration.rb +24 -0
- data/lib/httpdisk/sloptions.rb +105 -0
- data/lib/httpdisk/version.rb +1 -1
- metadata +25 -4
- data/lib/httpdisk/cli.rb +0 -223
- data/lib/httpdisk/cli_slop.rb +0 -54
@@ -0,0 +1,99 @@
|
|
1
|
+
module HTTPDisk
|
2
|
+
module Grep
|
3
|
+
class Printer
|
4
|
+
GREP_COLOR = '37;45'.freeze
|
5
|
+
|
6
|
+
attr_reader :output
|
7
|
+
|
8
|
+
def initialize(output)
|
9
|
+
@output = output
|
10
|
+
end
|
11
|
+
|
12
|
+
def print(path, payload, all_matches); end
|
13
|
+
|
14
|
+
protected
|
15
|
+
|
16
|
+
#
|
17
|
+
# helpers for subclasses
|
18
|
+
#
|
19
|
+
|
20
|
+
def grep_color
|
21
|
+
@grep_color ||= (ENV['GREP_COLOR'] || GREP_COLOR)
|
22
|
+
end
|
23
|
+
|
24
|
+
def print_matches(matches)
|
25
|
+
s = matches.first.string
|
26
|
+
if output.tty?
|
27
|
+
s = [].tap do |result|
|
28
|
+
ii = 0
|
29
|
+
matches.each do
|
30
|
+
result << s[ii..._1.begin(0)]
|
31
|
+
result << "\e["
|
32
|
+
result << grep_color
|
33
|
+
result << 'm'
|
34
|
+
result << _1[0]
|
35
|
+
result << "\e[0m"
|
36
|
+
ii = _1.end(0)
|
37
|
+
end
|
38
|
+
result << s[ii..]
|
39
|
+
end.join
|
40
|
+
end
|
41
|
+
output.puts s
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
#
|
46
|
+
# subclasses
|
47
|
+
#
|
48
|
+
|
49
|
+
# path:count
|
50
|
+
class CountPrinter < Printer
|
51
|
+
def print(path, _payload, all_matches)
|
52
|
+
output.puts "#{path}:#{all_matches.length}"
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
# header, then each match
|
57
|
+
class HeaderPrinter < Printer
|
58
|
+
attr_reader :head, :printed
|
59
|
+
|
60
|
+
def initialize(output, head)
|
61
|
+
super(output)
|
62
|
+
@head = head
|
63
|
+
@printed = 0
|
64
|
+
end
|
65
|
+
|
66
|
+
def print(path, payload, all_matches)
|
67
|
+
# separator & filename
|
68
|
+
output.puts if (@printed += 1) > 1
|
69
|
+
output.puts path
|
70
|
+
|
71
|
+
# --head
|
72
|
+
if head
|
73
|
+
io = StringIO.new
|
74
|
+
payload.write_header(io)
|
75
|
+
io.string.lines.each { output.puts "< #{_1}" }
|
76
|
+
end
|
77
|
+
|
78
|
+
# matches
|
79
|
+
all_matches.each { print_matches(_1) }
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
class SilentPrinter < Printer
|
84
|
+
def initialize
|
85
|
+
super(nil)
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
# each match as path:match
|
90
|
+
class TersePrinter < Printer
|
91
|
+
def print(path, _payload, all_matches)
|
92
|
+
all_matches.each do
|
93
|
+
output.write("#{path}:")
|
94
|
+
print_matches(_1)
|
95
|
+
end
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
99
|
+
end
|
data/lib/httpdisk/payload.rb
CHANGED
@@ -39,11 +39,11 @@ module HTTPDisk
|
|
39
39
|
@headers = Faraday::Utils::Headers.new
|
40
40
|
end
|
41
41
|
|
42
|
-
def
|
43
|
-
status
|
42
|
+
def error?
|
43
|
+
status >= 400
|
44
44
|
end
|
45
45
|
|
46
|
-
def
|
46
|
+
def write_header(f)
|
47
47
|
# comment
|
48
48
|
f.puts "# #{comment}"
|
49
49
|
|
@@ -52,9 +52,11 @@ module HTTPDisk
|
|
52
52
|
|
53
53
|
# headers
|
54
54
|
headers.each { f.puts("#{_1}: #{_2}") }
|
55
|
-
|
55
|
+
end
|
56
56
|
|
57
|
-
|
57
|
+
def write(f)
|
58
|
+
write_header(f)
|
59
|
+
f.puts
|
58
60
|
f.write(body)
|
59
61
|
end
|
60
62
|
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
require 'slop'
|
2
|
+
|
3
|
+
module Slop
|
4
|
+
# Custom duration type for Slop, used for --expires. Raises aggressively
|
5
|
+
# because this is a tricky and lightly documented option.
|
6
|
+
class DurationOption < Option
|
7
|
+
UNITS = {
|
8
|
+
s: 1,
|
9
|
+
m: 60,
|
10
|
+
h: 60 * 60,
|
11
|
+
d: 24 * 60 * 60,
|
12
|
+
w: 7 * 24 * 60 * 60,
|
13
|
+
y: 365 * 7 * 24 * 60 * 60,
|
14
|
+
}.freeze
|
15
|
+
|
16
|
+
def call(value)
|
17
|
+
m = value.match(/^(\d+)([smhdwy])?$/)
|
18
|
+
raise Slop::Error, "invalid --expires #{value.inspect}" if !m
|
19
|
+
|
20
|
+
num, unit = m[1].to_i, (m[2] || 's').to_sym
|
21
|
+
num * UNITS[unit]
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,105 @@
|
|
1
|
+
module HTTPDisk
|
2
|
+
# Like Slop, but for sanity checking method options. Useful for library entry
|
3
|
+
# points that want to be strict. Example usage:
|
4
|
+
#
|
5
|
+
# options = Sloptions.new(options) do
|
6
|
+
# _1.boolean :force
|
7
|
+
# _1.integer :retries, required: true
|
8
|
+
# _1.string :hello, default: 'world'
|
9
|
+
# ...
|
10
|
+
# end
|
11
|
+
class Sloptions
|
12
|
+
attr_reader :flags
|
13
|
+
|
14
|
+
def self.parse(options, &block)
|
15
|
+
Sloptions.new(&block).parse(options)
|
16
|
+
end
|
17
|
+
|
18
|
+
def initialize
|
19
|
+
@flags = {}
|
20
|
+
yield(self)
|
21
|
+
end
|
22
|
+
|
23
|
+
#
|
24
|
+
# _1.on and friends
|
25
|
+
#
|
26
|
+
|
27
|
+
def on(flag, foptions = {})
|
28
|
+
raise ":#{flag} already defined" if flags[flag]
|
29
|
+
|
30
|
+
flags[flag] = foptions
|
31
|
+
end
|
32
|
+
|
33
|
+
%i[array boolean float hash integer string symbol].each do |method|
|
34
|
+
define_method(method) do |flag, foptions = {}|
|
35
|
+
on(flag, { type: method }.merge(foptions))
|
36
|
+
end
|
37
|
+
end
|
38
|
+
alias bool boolean
|
39
|
+
|
40
|
+
#
|
41
|
+
# return parsed options
|
42
|
+
#
|
43
|
+
|
44
|
+
def parse(options)
|
45
|
+
# defaults
|
46
|
+
options = defaults.merge(options.compact)
|
47
|
+
|
48
|
+
flags.each do |flag, foptions|
|
49
|
+
# nil check
|
50
|
+
value = options[flag]
|
51
|
+
if value.nil?
|
52
|
+
raise ArgumentError, ":#{flag} is required" if foptions[:required]
|
53
|
+
|
54
|
+
next
|
55
|
+
end
|
56
|
+
|
57
|
+
# type cast (for boolean)
|
58
|
+
if foptions[:type] == :boolean
|
59
|
+
value = options[flag] = !!options[flag]
|
60
|
+
end
|
61
|
+
|
62
|
+
# type check
|
63
|
+
types = Array(foptions[:type])
|
64
|
+
raise ArgumentError, error_message(flag, value, types) if !valid?(value, types)
|
65
|
+
end
|
66
|
+
|
67
|
+
# return
|
68
|
+
options
|
69
|
+
end
|
70
|
+
|
71
|
+
protected
|
72
|
+
|
73
|
+
def defaults
|
74
|
+
flags.map { |flag, foptions| [flag, foptions[:default]] }.to_h.compact
|
75
|
+
end
|
76
|
+
|
77
|
+
# does value match valid?
|
78
|
+
def valid?(value, types)
|
79
|
+
types.any? do
|
80
|
+
case _1
|
81
|
+
when :array then true if value.is_a?(Array)
|
82
|
+
when :boolean then true # in Ruby everything is a boolean
|
83
|
+
when :float then true if value.is_a?(Float) || value.is_a?(Integer)
|
84
|
+
when :hash then true if value.is_a?(Hash)
|
85
|
+
when :integer then true if value.is_a?(Integer)
|
86
|
+
when :string then true if value.is_a?(String)
|
87
|
+
when :symbol then true if value.is_a?(Symbol)
|
88
|
+
when Class then true if value.is_a?(_1) # for custom checks
|
89
|
+
else
|
90
|
+
raise "unknown flag type #{_1.inspect}"
|
91
|
+
end
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
# nice error message for when value is invalid
|
96
|
+
def error_message(flag, value, valid)
|
97
|
+
classes = valid.compact.map do
|
98
|
+
s = _1.to_s
|
99
|
+
s = s.downcase if s =~ /\b(Array|Float|Hash|Integer|String|Symbol)\b/
|
100
|
+
s
|
101
|
+
end.join('/')
|
102
|
+
"expected :#{flag} to be #{classes}, not #{value.inspect}"
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
data/lib/httpdisk/version.rb
CHANGED
metadata
CHANGED
@@ -1,15 +1,29 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: httpdisk
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2
|
4
|
+
version: 0.5.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Adam Doppelt
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-05
|
11
|
+
date: 2021-07-05 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: content-type
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0.0'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0.0'
|
13
27
|
- !ruby/object:Gem::Dependency
|
14
28
|
name: faraday
|
15
29
|
requirement: !ruby/object:Gem::Requirement
|
@@ -70,6 +84,7 @@ description: httpdisk works with faraday to aggressively cache responses on disk
|
|
70
84
|
email: amd@gurge.com
|
71
85
|
executables:
|
72
86
|
- httpdisk
|
87
|
+
- httpdisk-grep
|
73
88
|
extensions: []
|
74
89
|
extra_rdoc_files: []
|
75
90
|
files:
|
@@ -82,16 +97,22 @@ files:
|
|
82
97
|
- README.md
|
83
98
|
- Rakefile
|
84
99
|
- bin/httpdisk
|
100
|
+
- bin/httpdisk-grep
|
85
101
|
- examples.rb
|
86
102
|
- httpdisk.gemspec
|
87
103
|
- lib/httpdisk.rb
|
88
104
|
- lib/httpdisk/cache.rb
|
89
105
|
- lib/httpdisk/cache_key.rb
|
90
|
-
- lib/httpdisk/cli.rb
|
91
|
-
- lib/httpdisk/
|
106
|
+
- lib/httpdisk/cli/args.rb
|
107
|
+
- lib/httpdisk/cli/main.rb
|
92
108
|
- lib/httpdisk/client.rb
|
93
109
|
- lib/httpdisk/error.rb
|
110
|
+
- lib/httpdisk/grep/args.rb
|
111
|
+
- lib/httpdisk/grep/main.rb
|
112
|
+
- lib/httpdisk/grep/printer.rb
|
94
113
|
- lib/httpdisk/payload.rb
|
114
|
+
- lib/httpdisk/slop_duration.rb
|
115
|
+
- lib/httpdisk/sloptions.rb
|
95
116
|
- lib/httpdisk/version.rb
|
96
117
|
- logo.svg
|
97
118
|
homepage: http://github.com/gurgeous/httpdisk
|
data/lib/httpdisk/cli.rb
DELETED
@@ -1,223 +0,0 @@
|
|
1
|
-
require 'faraday-cookie_jar'
|
2
|
-
require 'faraday_middleware'
|
3
|
-
require 'ostruct'
|
4
|
-
|
5
|
-
module HTTPDisk
|
6
|
-
# Command line httpdisk command.
|
7
|
-
class Cli
|
8
|
-
attr_reader :options
|
9
|
-
|
10
|
-
# for --expires
|
11
|
-
UNITS = {
|
12
|
-
s: 1,
|
13
|
-
m: 60,
|
14
|
-
h: 60 * 60,
|
15
|
-
d: 24 * 60 * 60,
|
16
|
-
w: 7 * 24 * 60 * 60,
|
17
|
-
y: 365 * 7 * 24 * 60 * 60,
|
18
|
-
}.freeze
|
19
|
-
|
20
|
-
def initialize(options)
|
21
|
-
@options = options
|
22
|
-
end
|
23
|
-
|
24
|
-
# we have a very liberal retry policy
|
25
|
-
RETRY_OPTIONS = {
|
26
|
-
methods: %w[delete get head options patch post put trace],
|
27
|
-
retry_statuses: (400..600).to_a,
|
28
|
-
retry_if: ->(_env, _err) { true },
|
29
|
-
}.freeze
|
30
|
-
|
31
|
-
# Make the request (or print status)
|
32
|
-
def run
|
33
|
-
# short circuit --status
|
34
|
-
if options[:status]
|
35
|
-
status
|
36
|
-
return
|
37
|
-
end
|
38
|
-
|
39
|
-
# create Faraday client
|
40
|
-
faraday = create_faraday
|
41
|
-
|
42
|
-
# run request
|
43
|
-
response = faraday.run_request(request_method, request_url, request_body, request_headers)
|
44
|
-
if response.status >= 400
|
45
|
-
raise CliError, "the requested URL returned error: #{response.status} #{response.reason_phrase}"
|
46
|
-
end
|
47
|
-
|
48
|
-
# output
|
49
|
-
if options[:output]
|
50
|
-
File.open(options[:output], 'w') { output(response, _1) }
|
51
|
-
else
|
52
|
-
output(response, $stdout)
|
53
|
-
end
|
54
|
-
end
|
55
|
-
|
56
|
-
def create_faraday
|
57
|
-
Faraday.new do
|
58
|
-
# connection settings
|
59
|
-
_1.proxy = proxy if options[:proxy]
|
60
|
-
_1.options.timeout = options[:max_time] if options[:max_time]
|
61
|
-
|
62
|
-
# cookie middleware
|
63
|
-
_1.use :cookie_jar
|
64
|
-
|
65
|
-
# BEFORE httpdisk so each redirect segment is cached
|
66
|
-
_1.response :follow_redirects
|
67
|
-
|
68
|
-
# httpdisk
|
69
|
-
_1.use :httpdisk, client_options
|
70
|
-
|
71
|
-
# AFTER httpdisk so transient failures are not cached
|
72
|
-
if options[:retry]
|
73
|
-
_1.request :retry, RETRY_OPTIONS.merge(max: options[:retry])
|
74
|
-
end
|
75
|
-
end
|
76
|
-
end
|
77
|
-
|
78
|
-
# Support for --status
|
79
|
-
def status
|
80
|
-
# build env
|
81
|
-
env = Faraday::Env.new.tap do
|
82
|
-
_1.method = request_method
|
83
|
-
_1.request_body = request_body
|
84
|
-
_1.request_headers = request_headers
|
85
|
-
_1.url = request_url
|
86
|
-
end
|
87
|
-
|
88
|
-
# now print status
|
89
|
-
client = HTTPDisk::Client.new(nil, client_options)
|
90
|
-
client.status(env).each do
|
91
|
-
puts "#{_1}: #{_2.inspect}"
|
92
|
-
end
|
93
|
-
end
|
94
|
-
|
95
|
-
# Output response to f
|
96
|
-
def output(response, f)
|
97
|
-
if options[:include]
|
98
|
-
f.puts "HTTPDISK #{response.status} #{response.reason_phrase}"
|
99
|
-
response.headers.each { f.puts("#{_1}: #{_2}") }
|
100
|
-
f.puts
|
101
|
-
end
|
102
|
-
f.write(response.body)
|
103
|
-
end
|
104
|
-
|
105
|
-
#
|
106
|
-
# request_XXX
|
107
|
-
#
|
108
|
-
|
109
|
-
# HTTP method (get, post, etc.)
|
110
|
-
def request_method
|
111
|
-
method = if options[:request]
|
112
|
-
options[:request]
|
113
|
-
elsif options[:data]
|
114
|
-
'post'
|
115
|
-
end
|
116
|
-
method ||= 'get'
|
117
|
-
method = method.downcase.to_sym
|
118
|
-
|
119
|
-
if !Faraday::Connection::METHODS.include?(method)
|
120
|
-
raise CliError, "invalid --request #{method.inspect}"
|
121
|
-
end
|
122
|
-
|
123
|
-
method
|
124
|
-
end
|
125
|
-
|
126
|
-
# Request url
|
127
|
-
def request_url
|
128
|
-
url = options[:url]
|
129
|
-
# recover from missing http:
|
130
|
-
if url !~ %r{^https?://}i
|
131
|
-
if url =~ %r{^\w+://}
|
132
|
-
raise CliError, 'only http/https supported'
|
133
|
-
end
|
134
|
-
|
135
|
-
url = "http://#{url}"
|
136
|
-
end
|
137
|
-
URI.parse(url)
|
138
|
-
rescue URI::InvalidURIError
|
139
|
-
raise CliError, "invalid url #{url.inspect}"
|
140
|
-
end
|
141
|
-
|
142
|
-
# Request body
|
143
|
-
def request_body
|
144
|
-
options[:data]
|
145
|
-
end
|
146
|
-
|
147
|
-
# Request headers
|
148
|
-
def request_headers
|
149
|
-
{}.tap do |headers|
|
150
|
-
if options[:user_agent]
|
151
|
-
headers['User-Agent'] = options[:user_agent]
|
152
|
-
end
|
153
|
-
|
154
|
-
options[:header].each do |header|
|
155
|
-
key, value = header.split(': ', 2)
|
156
|
-
if !key || !value || key.empty? || value.empty?
|
157
|
-
raise CliError, "invalid --header #{header.inspect}"
|
158
|
-
end
|
159
|
-
|
160
|
-
headers[key] = value
|
161
|
-
end
|
162
|
-
end
|
163
|
-
end
|
164
|
-
|
165
|
-
#
|
166
|
-
# helpers
|
167
|
-
#
|
168
|
-
|
169
|
-
# Options to HTTPDisk::Client
|
170
|
-
def client_options
|
171
|
-
{}.tap do |client_options|
|
172
|
-
client_options[:dir] = options[:dir]
|
173
|
-
if options[:expires]
|
174
|
-
seconds = parse_expires(options[:expires])
|
175
|
-
if !seconds
|
176
|
-
raise CliError, "invalid --expires #{options[:expires].inspect}"
|
177
|
-
end
|
178
|
-
|
179
|
-
client_options[:expires_in] = seconds
|
180
|
-
end
|
181
|
-
client_options[:force] = options[:force]
|
182
|
-
client_options[:force_errors] = options[:force_errors]
|
183
|
-
end
|
184
|
-
end
|
185
|
-
|
186
|
-
# Return validated --proxy flag if present
|
187
|
-
def proxy
|
188
|
-
return if !options[:proxy]
|
189
|
-
|
190
|
-
proxy = parse_proxy(options[:proxy])
|
191
|
-
raise CliError, "--proxy should be host[:port], not #{options[:proxy].inspect}" if !proxy
|
192
|
-
|
193
|
-
proxy
|
194
|
-
end
|
195
|
-
|
196
|
-
# Parse --expires flag
|
197
|
-
def parse_expires(s)
|
198
|
-
m = s.match(/^(\d+)([smhdwy])?$/)
|
199
|
-
return if !m
|
200
|
-
|
201
|
-
num, unit = m[1].to_i, (m[2] || 's').to_sym
|
202
|
-
return if !UNITS.key?(unit)
|
203
|
-
|
204
|
-
num * UNITS[unit]
|
205
|
-
end
|
206
|
-
|
207
|
-
# Parse --proxy flag
|
208
|
-
def parse_proxy(proxy_flag)
|
209
|
-
host, port = proxy_flag.split(':', 2)
|
210
|
-
return if !host || host.empty?
|
211
|
-
return if port&.empty?
|
212
|
-
|
213
|
-
URI.parse('http://placeholder').tap do
|
214
|
-
begin
|
215
|
-
_1.host = host
|
216
|
-
_1.port = port if port
|
217
|
-
rescue URI::InvalidComponentError
|
218
|
-
return
|
219
|
-
end
|
220
|
-
end.to_s
|
221
|
-
end
|
222
|
-
end
|
223
|
-
end
|