httpdisk 0.1.0 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +28 -0
- data/Gemfile +1 -0
- data/Gemfile.lock +36 -4
- data/README.md +36 -4
- data/Rakefile +21 -10
- data/bin/httpdisk +10 -8
- data/bin/httpdisk-grep +46 -0
- data/examples.rb +1 -2
- data/httpdisk.gemspec +3 -2
- data/lib/httpdisk.rb +10 -5
- data/lib/httpdisk/cache.rb +33 -22
- data/lib/httpdisk/cache_key.rb +17 -9
- data/lib/httpdisk/cli/args.rb +57 -0
- data/lib/httpdisk/cli/main.rb +169 -0
- data/lib/httpdisk/client.rb +94 -18
- data/lib/httpdisk/error.rb +4 -0
- data/lib/httpdisk/grep/args.rb +35 -0
- data/lib/httpdisk/grep/main.rb +112 -0
- data/lib/httpdisk/grep/printer.rb +99 -0
- data/lib/httpdisk/payload.rb +11 -9
- data/lib/httpdisk/slop_duration.rb +24 -0
- data/lib/httpdisk/sloptions.rb +105 -0
- data/lib/httpdisk/version.rb +1 -1
- metadata +26 -4
- data/lib/httpdisk/cli.rb +0 -218
- data/lib/httpdisk/cli_slop.rb +0 -54
data/lib/httpdisk/cli.rb
DELETED
@@ -1,218 +0,0 @@
|
|
1
|
-
require 'faraday-cookie_jar'
|
2
|
-
require 'faraday_middleware'
|
3
|
-
require 'ostruct'
|
4
|
-
|
5
|
-
module HTTPDisk
|
6
|
-
# Command line httpdisk command.
|
7
|
-
class Cli
|
8
|
-
attr_reader :options
|
9
|
-
|
10
|
-
# for --expires
|
11
|
-
UNITS = {
|
12
|
-
s: 1,
|
13
|
-
m: 60,
|
14
|
-
h: 60 * 60,
|
15
|
-
d: 24 * 60 * 60,
|
16
|
-
w: 7 * 24 * 60 * 60,
|
17
|
-
y: 365 * 7 * 24 * 60 * 60,
|
18
|
-
}.freeze
|
19
|
-
|
20
|
-
def initialize(options)
|
21
|
-
@options = options
|
22
|
-
end
|
23
|
-
|
24
|
-
# we have a very liberal retry policy
|
25
|
-
RETRY_OPTIONS = {
|
26
|
-
methods: %w[delete get head options patch post put trace],
|
27
|
-
retry_statuses: (400..600).to_a,
|
28
|
-
retry_if: ->(_env, _err) { true },
|
29
|
-
}.freeze
|
30
|
-
|
31
|
-
# Make the request (or print status)
|
32
|
-
def run
|
33
|
-
# short circuit --status
|
34
|
-
if options[:status]
|
35
|
-
status
|
36
|
-
return
|
37
|
-
end
|
38
|
-
|
39
|
-
# create Faraday client
|
40
|
-
faraday = create_faraday
|
41
|
-
|
42
|
-
# run request
|
43
|
-
response = faraday.run_request(request_method, request_url, request_body, request_headers)
|
44
|
-
if response.status >= 400
|
45
|
-
raise CliError, "the requested URL returned error: #{response.status} #{response.reason_phrase}"
|
46
|
-
end
|
47
|
-
|
48
|
-
# output
|
49
|
-
if options[:output]
|
50
|
-
File.open(options[:output], 'w') { output(response, _1) }
|
51
|
-
else
|
52
|
-
output(response, $stdout)
|
53
|
-
end
|
54
|
-
end
|
55
|
-
|
56
|
-
def create_faraday
|
57
|
-
Faraday.new do
|
58
|
-
# connection settings
|
59
|
-
_1.proxy = proxy if options[:proxy]
|
60
|
-
_1.options.timeout = options[:max_time] if options[:max_time]
|
61
|
-
|
62
|
-
# cookie middleware
|
63
|
-
_1.use :cookie_jar
|
64
|
-
|
65
|
-
# BEFORE httpdisk so each redirect segment is cached
|
66
|
-
_1.response :follow_redirects
|
67
|
-
|
68
|
-
# httpdisk
|
69
|
-
_1.use :httpdisk, client_options
|
70
|
-
|
71
|
-
# AFTER httpdisk so transient failures are not cached
|
72
|
-
if options[:retry]
|
73
|
-
_1.request :retry, RETRY_OPTIONS.merge(max: options[:retry])
|
74
|
-
end
|
75
|
-
end
|
76
|
-
end
|
77
|
-
|
78
|
-
# Support for --status
|
79
|
-
def status
|
80
|
-
# build env
|
81
|
-
env = Faraday::Env.new.tap do
|
82
|
-
_1.method = request_method
|
83
|
-
_1.request_body = request_body
|
84
|
-
_1.request_headers = request_headers
|
85
|
-
_1.url = request_url
|
86
|
-
end
|
87
|
-
|
88
|
-
# now print status
|
89
|
-
client = HTTPDisk::Client.new(nil, client_options)
|
90
|
-
client.status(env).each do
|
91
|
-
puts "#{_1}: #{_2.inspect}"
|
92
|
-
end
|
93
|
-
end
|
94
|
-
|
95
|
-
# Output response to f
|
96
|
-
def output(response, f)
|
97
|
-
if options[:include]
|
98
|
-
f.puts "HTTPDISK #{response.status} #{response.reason_phrase}"
|
99
|
-
response.headers.each { f.puts("#{_1}: #{_2}") }
|
100
|
-
f.puts
|
101
|
-
end
|
102
|
-
f.write(response.body)
|
103
|
-
end
|
104
|
-
|
105
|
-
#
|
106
|
-
# request_XXX
|
107
|
-
#
|
108
|
-
|
109
|
-
# HTTP method (get, post, etc.)
|
110
|
-
def request_method
|
111
|
-
method = if options[:request]
|
112
|
-
options[:request]
|
113
|
-
elsif options[:data]
|
114
|
-
'post'
|
115
|
-
end
|
116
|
-
method ||= 'get'
|
117
|
-
method = method.downcase.to_sym
|
118
|
-
|
119
|
-
if !Faraday::Connection::METHODS.include?(method)
|
120
|
-
raise CliError, "invalid --request #{method.inspect}"
|
121
|
-
end
|
122
|
-
method
|
123
|
-
end
|
124
|
-
|
125
|
-
# Request url
|
126
|
-
def request_url
|
127
|
-
url = options[:url]
|
128
|
-
# recover from missing http:
|
129
|
-
if url !~ %r{^https?://}i
|
130
|
-
if url =~ %r{^\w+://}
|
131
|
-
raise CliError, 'only http/https supported'
|
132
|
-
end
|
133
|
-
url = "http://#{url}"
|
134
|
-
end
|
135
|
-
URI.parse(url)
|
136
|
-
rescue URI::InvalidURIError
|
137
|
-
raise CliError, "invalid url #{url.inspect}"
|
138
|
-
end
|
139
|
-
|
140
|
-
# Request body
|
141
|
-
def request_body
|
142
|
-
options[:data]
|
143
|
-
end
|
144
|
-
|
145
|
-
# Request headers
|
146
|
-
def request_headers
|
147
|
-
{}.tap do |headers|
|
148
|
-
if options[:user_agent]
|
149
|
-
headers['User-Agent'] = options[:user_agent]
|
150
|
-
end
|
151
|
-
|
152
|
-
options[:header].each do |header|
|
153
|
-
key, value = header.split(': ', 2)
|
154
|
-
if !key || !value || key.empty? || value.empty?
|
155
|
-
raise CliError, "invalid --header #{header.inspect}"
|
156
|
-
end
|
157
|
-
headers[key] = value
|
158
|
-
end
|
159
|
-
end
|
160
|
-
end
|
161
|
-
|
162
|
-
#
|
163
|
-
# helpers
|
164
|
-
#
|
165
|
-
|
166
|
-
# Options to HTTPDisk::Client
|
167
|
-
def client_options
|
168
|
-
{}.tap do |client_options|
|
169
|
-
client_options[:dir] = options[:dir]
|
170
|
-
if options[:expires]
|
171
|
-
seconds = parse_expires(options[:expires])
|
172
|
-
if !seconds
|
173
|
-
raise CliError, "invalid --expires #{options[:expires].inspect}"
|
174
|
-
end
|
175
|
-
client_options[:expires_in] = seconds
|
176
|
-
end
|
177
|
-
client_options[:force] = options[:force]
|
178
|
-
client_options[:force_errors] = options[:force_errors]
|
179
|
-
end
|
180
|
-
end
|
181
|
-
|
182
|
-
# Return validated --proxy flag if present
|
183
|
-
def proxy
|
184
|
-
return if !options[:proxy]
|
185
|
-
|
186
|
-
proxy = parse_proxy(options[:proxy])
|
187
|
-
raise CliError, "--proxy should be host[:port], not #{options[:proxy].inspect}" if !proxy
|
188
|
-
proxy
|
189
|
-
end
|
190
|
-
|
191
|
-
# Parse --expires flag
|
192
|
-
def parse_expires(s)
|
193
|
-
m = s.match(/^(\d+)([smhdwy])?$/)
|
194
|
-
return if !m
|
195
|
-
|
196
|
-
num, unit = m[1].to_i, (m[2] || 's').to_sym
|
197
|
-
return if !UNITS.key?(unit)
|
198
|
-
|
199
|
-
num * UNITS[unit]
|
200
|
-
end
|
201
|
-
|
202
|
-
# Parse --proxy flag
|
203
|
-
def parse_proxy(proxy_flag)
|
204
|
-
host, port = proxy_flag.split(':', 2)
|
205
|
-
return if !host || host.empty?
|
206
|
-
return if port&.empty?
|
207
|
-
|
208
|
-
URI.parse('http://placeholder').tap do
|
209
|
-
begin
|
210
|
-
_1.host = host
|
211
|
-
_1.port = port if port
|
212
|
-
rescue URI::InvalidComponentError
|
213
|
-
return
|
214
|
-
end
|
215
|
-
end.to_s
|
216
|
-
end
|
217
|
-
end
|
218
|
-
end
|
data/lib/httpdisk/cli_slop.rb
DELETED
@@ -1,54 +0,0 @@
|
|
1
|
-
# manually load dependencies here since this is loaded standalone by bin
|
2
|
-
require 'httpdisk/error'
|
3
|
-
require 'httpdisk/version'
|
4
|
-
require 'slop'
|
5
|
-
|
6
|
-
module HTTPDisk
|
7
|
-
# Slop parsing. This is broken out so we can run without require 'httpdisk'.
|
8
|
-
module CliSlop
|
9
|
-
def self.slop(args)
|
10
|
-
slop = Slop.parse(args) do |o|
|
11
|
-
o.banner = 'httpdisk [options] [url]'
|
12
|
-
|
13
|
-
# similar to curl
|
14
|
-
o.separator 'Similar to curl:'
|
15
|
-
o.string '-d', '--data', 'HTTP POST data'
|
16
|
-
o.array '-H', '--header', 'pass custom header(s) to server', delimiter: nil
|
17
|
-
o.boolean '-i', '--include', 'include response headers in the output'
|
18
|
-
o.integer '-m', '--max-time', 'maximum time allowed for the transfer'
|
19
|
-
o.string '-o', '--output', 'write to file instead of stdout'
|
20
|
-
o.string '-x', '--proxy', 'use host[:port] as proxy'
|
21
|
-
o.string '-X', '--request', 'HTTP method to use'
|
22
|
-
o.integer '--retry', 'retry request if problems occur'
|
23
|
-
o.boolean '-s', '--silent', "silent mode (don't print errors)"
|
24
|
-
o.string '-A', '--user-agent', 'send User-Agent to server'
|
25
|
-
|
26
|
-
# from httpdisk
|
27
|
-
o.separator 'Specific to httpdisk:'
|
28
|
-
o.string '--dir', 'httpdisk cache directory (defaults to ~/httpdisk)'
|
29
|
-
o.string '--expires', 'when to expire cached requests (ex: 1h, 2d, 3w)'
|
30
|
-
o.boolean '--force', "don't read anything from cache (but still write)"
|
31
|
-
o.boolean '--force-errors', "don't read errors from cache (but still write)"
|
32
|
-
o.boolean '--status', 'show status for a url in the cache'
|
33
|
-
|
34
|
-
# generic
|
35
|
-
o.boolean '--version', 'show version' do
|
36
|
-
puts "httpdisk #{HTTPDisk::VERSION}"
|
37
|
-
exit
|
38
|
-
end
|
39
|
-
o.on '--help', 'show this help' do
|
40
|
-
puts o
|
41
|
-
exit
|
42
|
-
end
|
43
|
-
end
|
44
|
-
|
45
|
-
raise Slop::Error, '' if args.empty?
|
46
|
-
raise Slop::Error, 'no URL specified' if slop.args.empty?
|
47
|
-
raise Slop::Error, 'more than one URL specified' if slop.args.length > 1
|
48
|
-
|
49
|
-
slop.to_h.tap do
|
50
|
-
_1[:url] = slop.args.first
|
51
|
-
end
|
52
|
-
end
|
53
|
-
end
|
54
|
-
end
|