httpdisk 0.2.0 → 0.5.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,99 @@
1
+ module HTTPDisk
2
+ module Grep
3
+ class Printer
4
+ GREP_COLOR = '37;45'.freeze
5
+
6
+ attr_reader :output
7
+
8
+ def initialize(output)
9
+ @output = output
10
+ end
11
+
12
+ def print(path, payload, all_matches); end
13
+
14
+ protected
15
+
16
+ #
17
+ # helpers for subclasses
18
+ #
19
+
20
+ def grep_color
21
+ @grep_color ||= (ENV['GREP_COLOR'] || GREP_COLOR)
22
+ end
23
+
24
+ def print_matches(matches)
25
+ s = matches.first.string
26
+ if output.tty?
27
+ s = [].tap do |result|
28
+ ii = 0
29
+ matches.each do
30
+ result << s[ii..._1.begin(0)]
31
+ result << "\e["
32
+ result << grep_color
33
+ result << 'm'
34
+ result << _1[0]
35
+ result << "\e[0m"
36
+ ii = _1.end(0)
37
+ end
38
+ result << s[ii..]
39
+ end.join
40
+ end
41
+ output.puts s
42
+ end
43
+ end
44
+
45
+ #
46
+ # subclasses
47
+ #
48
+
49
+ # path:count
50
+ class CountPrinter < Printer
51
+ def print(path, _payload, all_matches)
52
+ output.puts "#{path}:#{all_matches.length}"
53
+ end
54
+ end
55
+
56
+ # header, then each match
57
+ class HeaderPrinter < Printer
58
+ attr_reader :head, :printed
59
+
60
+ def initialize(output, head)
61
+ super(output)
62
+ @head = head
63
+ @printed = 0
64
+ end
65
+
66
+ def print(path, payload, all_matches)
67
+ # separator & filename
68
+ output.puts if (@printed += 1) > 1
69
+ output.puts path
70
+
71
+ # --head
72
+ if head
73
+ io = StringIO.new
74
+ payload.write_header(io)
75
+ io.string.lines.each { output.puts "< #{_1}" }
76
+ end
77
+
78
+ # matches
79
+ all_matches.each { print_matches(_1) }
80
+ end
81
+ end
82
+
83
+ class SilentPrinter < Printer
84
+ def initialize
85
+ super(nil)
86
+ end
87
+ end
88
+
89
+ # each match as path:match
90
+ class TersePrinter < Printer
91
+ def print(path, _payload, all_matches)
92
+ all_matches.each do
93
+ output.write("#{path}:")
94
+ print_matches(_1)
95
+ end
96
+ end
97
+ end
98
+ end
99
+ end
@@ -39,11 +39,11 @@ module HTTPDisk
39
39
  @headers = Faraday::Utils::Headers.new
40
40
  end
41
41
 
42
- def error_999?
43
- status == HTTPDisk::ERROR_STATUS
42
+ def error?
43
+ status >= 400
44
44
  end
45
45
 
46
- def write(f)
46
+ def write_header(f)
47
47
  # comment
48
48
  f.puts "# #{comment}"
49
49
 
@@ -52,9 +52,11 @@ module HTTPDisk
52
52
 
53
53
  # headers
54
54
  headers.each { f.puts("#{_1}: #{_2}") }
55
- f.puts
55
+ end
56
56
 
57
- # body
57
+ def write(f)
58
+ write_header(f)
59
+ f.puts
58
60
  f.write(body)
59
61
  end
60
62
  end
@@ -0,0 +1,24 @@
1
+ require 'slop'
2
+
3
+ module Slop
4
+ # Custom duration type for Slop, used for --expires. Raises aggressively
5
+ # because this is a tricky and lightly documented option.
6
+ class DurationOption < Option
7
+ UNITS = {
8
+ s: 1,
9
+ m: 60,
10
+ h: 60 * 60,
11
+ d: 24 * 60 * 60,
12
+ w: 7 * 24 * 60 * 60,
13
+ y: 365 * 7 * 24 * 60 * 60,
14
+ }.freeze
15
+
16
+ def call(value)
17
+ m = value.match(/^(\d+)([smhdwy])?$/)
18
+ raise Slop::Error, "invalid --expires #{value.inspect}" if !m
19
+
20
+ num, unit = m[1].to_i, (m[2] || 's').to_sym
21
+ num * UNITS[unit]
22
+ end
23
+ end
24
+ end
@@ -0,0 +1,105 @@
1
+ module HTTPDisk
2
+ # Like Slop, but for sanity checking method options. Useful for library entry
3
+ # points that want to be strict. Example usage:
4
+ #
5
+ # options = Sloptions.new(options) do
6
+ # _1.boolean :force
7
+ # _1.integer :retries, required: true
8
+ # _1.string :hello, default: 'world'
9
+ # ...
10
+ # end
11
+ class Sloptions
12
+ attr_reader :flags
13
+
14
+ def self.parse(options, &block)
15
+ Sloptions.new(&block).parse(options)
16
+ end
17
+
18
+ def initialize
19
+ @flags = {}
20
+ yield(self)
21
+ end
22
+
23
+ #
24
+ # _1.on and friends
25
+ #
26
+
27
+ def on(flag, foptions = {})
28
+ raise ":#{flag} already defined" if flags[flag]
29
+
30
+ flags[flag] = foptions
31
+ end
32
+
33
+ %i[array boolean float hash integer string symbol].each do |method|
34
+ define_method(method) do |flag, foptions = {}|
35
+ on(flag, { type: method }.merge(foptions))
36
+ end
37
+ end
38
+ alias bool boolean
39
+
40
+ #
41
+ # return parsed options
42
+ #
43
+
44
+ def parse(options)
45
+ # defaults
46
+ options = defaults.merge(options.compact)
47
+
48
+ flags.each do |flag, foptions|
49
+ # nil check
50
+ value = options[flag]
51
+ if value.nil?
52
+ raise ArgumentError, ":#{flag} is required" if foptions[:required]
53
+
54
+ next
55
+ end
56
+
57
+ # type cast (for boolean)
58
+ if foptions[:type] == :boolean
59
+ value = options[flag] = !!options[flag]
60
+ end
61
+
62
+ # type check
63
+ types = Array(foptions[:type])
64
+ raise ArgumentError, error_message(flag, value, types) if !valid?(value, types)
65
+ end
66
+
67
+ # return
68
+ options
69
+ end
70
+
71
+ protected
72
+
73
+ def defaults
74
+ flags.map { |flag, foptions| [flag, foptions[:default]] }.to_h.compact
75
+ end
76
+
77
+ # does value match valid?
78
+ def valid?(value, types)
79
+ types.any? do
80
+ case _1
81
+ when :array then true if value.is_a?(Array)
82
+ when :boolean then true # in Ruby everything is a boolean
83
+ when :float then true if value.is_a?(Float) || value.is_a?(Integer)
84
+ when :hash then true if value.is_a?(Hash)
85
+ when :integer then true if value.is_a?(Integer)
86
+ when :string then true if value.is_a?(String)
87
+ when :symbol then true if value.is_a?(Symbol)
88
+ when Class then true if value.is_a?(_1) # for custom checks
89
+ else
90
+ raise "unknown flag type #{_1.inspect}"
91
+ end
92
+ end
93
+ end
94
+
95
+ # nice error message for when value is invalid
96
+ def error_message(flag, value, valid)
97
+ classes = valid.compact.map do
98
+ s = _1.to_s
99
+ s = s.downcase if s =~ /\b(Array|Float|Hash|Integer|String|Symbol)\b/
100
+ s
101
+ end.join('/')
102
+ "expected :#{flag} to be #{classes}, not #{value.inspect}"
103
+ end
104
+ end
105
+ end
@@ -1,3 +1,3 @@
1
1
  module HTTPDisk
2
- VERSION = '0.2.0'.freeze
2
+ VERSION = '0.5.2'.freeze
3
3
  end
metadata CHANGED
@@ -1,15 +1,29 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: httpdisk
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.0
4
+ version: 0.5.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Adam Doppelt
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2021-05-03 00:00:00.000000000 Z
11
+ date: 2021-07-05 00:00:00.000000000 Z
12
12
  dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: content-type
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '0.0'
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '0.0'
13
27
  - !ruby/object:Gem::Dependency
14
28
  name: faraday
15
29
  requirement: !ruby/object:Gem::Requirement
@@ -70,6 +84,7 @@ description: httpdisk works with faraday to aggressively cache responses on disk
70
84
  email: amd@gurge.com
71
85
  executables:
72
86
  - httpdisk
87
+ - httpdisk-grep
73
88
  extensions: []
74
89
  extra_rdoc_files: []
75
90
  files:
@@ -82,16 +97,22 @@ files:
82
97
  - README.md
83
98
  - Rakefile
84
99
  - bin/httpdisk
100
+ - bin/httpdisk-grep
85
101
  - examples.rb
86
102
  - httpdisk.gemspec
87
103
  - lib/httpdisk.rb
88
104
  - lib/httpdisk/cache.rb
89
105
  - lib/httpdisk/cache_key.rb
90
- - lib/httpdisk/cli.rb
91
- - lib/httpdisk/cli_slop.rb
106
+ - lib/httpdisk/cli/args.rb
107
+ - lib/httpdisk/cli/main.rb
92
108
  - lib/httpdisk/client.rb
93
109
  - lib/httpdisk/error.rb
110
+ - lib/httpdisk/grep/args.rb
111
+ - lib/httpdisk/grep/main.rb
112
+ - lib/httpdisk/grep/printer.rb
94
113
  - lib/httpdisk/payload.rb
114
+ - lib/httpdisk/slop_duration.rb
115
+ - lib/httpdisk/sloptions.rb
95
116
  - lib/httpdisk/version.rb
96
117
  - logo.svg
97
118
  homepage: http://github.com/gurgeous/httpdisk
data/lib/httpdisk/cli.rb DELETED
@@ -1,223 +0,0 @@
1
- require 'faraday-cookie_jar'
2
- require 'faraday_middleware'
3
- require 'ostruct'
4
-
5
- module HTTPDisk
6
- # Command line httpdisk command.
7
- class Cli
8
- attr_reader :options
9
-
10
- # for --expires
11
- UNITS = {
12
- s: 1,
13
- m: 60,
14
- h: 60 * 60,
15
- d: 24 * 60 * 60,
16
- w: 7 * 24 * 60 * 60,
17
- y: 365 * 7 * 24 * 60 * 60,
18
- }.freeze
19
-
20
- def initialize(options)
21
- @options = options
22
- end
23
-
24
- # we have a very liberal retry policy
25
- RETRY_OPTIONS = {
26
- methods: %w[delete get head options patch post put trace],
27
- retry_statuses: (400..600).to_a,
28
- retry_if: ->(_env, _err) { true },
29
- }.freeze
30
-
31
- # Make the request (or print status)
32
- def run
33
- # short circuit --status
34
- if options[:status]
35
- status
36
- return
37
- end
38
-
39
- # create Faraday client
40
- faraday = create_faraday
41
-
42
- # run request
43
- response = faraday.run_request(request_method, request_url, request_body, request_headers)
44
- if response.status >= 400
45
- raise CliError, "the requested URL returned error: #{response.status} #{response.reason_phrase}"
46
- end
47
-
48
- # output
49
- if options[:output]
50
- File.open(options[:output], 'w') { output(response, _1) }
51
- else
52
- output(response, $stdout)
53
- end
54
- end
55
-
56
- def create_faraday
57
- Faraday.new do
58
- # connection settings
59
- _1.proxy = proxy if options[:proxy]
60
- _1.options.timeout = options[:max_time] if options[:max_time]
61
-
62
- # cookie middleware
63
- _1.use :cookie_jar
64
-
65
- # BEFORE httpdisk so each redirect segment is cached
66
- _1.response :follow_redirects
67
-
68
- # httpdisk
69
- _1.use :httpdisk, client_options
70
-
71
- # AFTER httpdisk so transient failures are not cached
72
- if options[:retry]
73
- _1.request :retry, RETRY_OPTIONS.merge(max: options[:retry])
74
- end
75
- end
76
- end
77
-
78
- # Support for --status
79
- def status
80
- # build env
81
- env = Faraday::Env.new.tap do
82
- _1.method = request_method
83
- _1.request_body = request_body
84
- _1.request_headers = request_headers
85
- _1.url = request_url
86
- end
87
-
88
- # now print status
89
- client = HTTPDisk::Client.new(nil, client_options)
90
- client.status(env).each do
91
- puts "#{_1}: #{_2.inspect}"
92
- end
93
- end
94
-
95
- # Output response to f
96
- def output(response, f)
97
- if options[:include]
98
- f.puts "HTTPDISK #{response.status} #{response.reason_phrase}"
99
- response.headers.each { f.puts("#{_1}: #{_2}") }
100
- f.puts
101
- end
102
- f.write(response.body)
103
- end
104
-
105
- #
106
- # request_XXX
107
- #
108
-
109
- # HTTP method (get, post, etc.)
110
- def request_method
111
- method = if options[:request]
112
- options[:request]
113
- elsif options[:data]
114
- 'post'
115
- end
116
- method ||= 'get'
117
- method = method.downcase.to_sym
118
-
119
- if !Faraday::Connection::METHODS.include?(method)
120
- raise CliError, "invalid --request #{method.inspect}"
121
- end
122
-
123
- method
124
- end
125
-
126
- # Request url
127
- def request_url
128
- url = options[:url]
129
- # recover from missing http:
130
- if url !~ %r{^https?://}i
131
- if url =~ %r{^\w+://}
132
- raise CliError, 'only http/https supported'
133
- end
134
-
135
- url = "http://#{url}"
136
- end
137
- URI.parse(url)
138
- rescue URI::InvalidURIError
139
- raise CliError, "invalid url #{url.inspect}"
140
- end
141
-
142
- # Request body
143
- def request_body
144
- options[:data]
145
- end
146
-
147
- # Request headers
148
- def request_headers
149
- {}.tap do |headers|
150
- if options[:user_agent]
151
- headers['User-Agent'] = options[:user_agent]
152
- end
153
-
154
- options[:header].each do |header|
155
- key, value = header.split(': ', 2)
156
- if !key || !value || key.empty? || value.empty?
157
- raise CliError, "invalid --header #{header.inspect}"
158
- end
159
-
160
- headers[key] = value
161
- end
162
- end
163
- end
164
-
165
- #
166
- # helpers
167
- #
168
-
169
- # Options to HTTPDisk::Client
170
- def client_options
171
- {}.tap do |client_options|
172
- client_options[:dir] = options[:dir]
173
- if options[:expires]
174
- seconds = parse_expires(options[:expires])
175
- if !seconds
176
- raise CliError, "invalid --expires #{options[:expires].inspect}"
177
- end
178
-
179
- client_options[:expires_in] = seconds
180
- end
181
- client_options[:force] = options[:force]
182
- client_options[:force_errors] = options[:force_errors]
183
- end
184
- end
185
-
186
- # Return validated --proxy flag if present
187
- def proxy
188
- return if !options[:proxy]
189
-
190
- proxy = parse_proxy(options[:proxy])
191
- raise CliError, "--proxy should be host[:port], not #{options[:proxy].inspect}" if !proxy
192
-
193
- proxy
194
- end
195
-
196
- # Parse --expires flag
197
- def parse_expires(s)
198
- m = s.match(/^(\d+)([smhdwy])?$/)
199
- return if !m
200
-
201
- num, unit = m[1].to_i, (m[2] || 's').to_sym
202
- return if !UNITS.key?(unit)
203
-
204
- num * UNITS[unit]
205
- end
206
-
207
- # Parse --proxy flag
208
- def parse_proxy(proxy_flag)
209
- host, port = proxy_flag.split(':', 2)
210
- return if !host || host.empty?
211
- return if port&.empty?
212
-
213
- URI.parse('http://placeholder').tap do
214
- begin
215
- _1.host = host
216
- _1.port = port if port
217
- rescue URI::InvalidComponentError
218
- return
219
- end
220
- end.to_s
221
- end
222
- end
223
- end