httpdisk 0.1.0 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +28 -0
- data/Gemfile +1 -0
- data/Gemfile.lock +36 -4
- data/README.md +36 -4
- data/Rakefile +21 -10
- data/bin/httpdisk +10 -8
- data/bin/httpdisk-grep +46 -0
- data/examples.rb +1 -2
- data/httpdisk.gemspec +3 -2
- data/lib/httpdisk.rb +10 -5
- data/lib/httpdisk/cache.rb +33 -22
- data/lib/httpdisk/cache_key.rb +17 -9
- data/lib/httpdisk/cli/args.rb +57 -0
- data/lib/httpdisk/cli/main.rb +169 -0
- data/lib/httpdisk/client.rb +94 -18
- data/lib/httpdisk/error.rb +4 -0
- data/lib/httpdisk/grep/args.rb +35 -0
- data/lib/httpdisk/grep/main.rb +112 -0
- data/lib/httpdisk/grep/printer.rb +99 -0
- data/lib/httpdisk/payload.rb +11 -9
- data/lib/httpdisk/slop_duration.rb +24 -0
- data/lib/httpdisk/sloptions.rb +105 -0
- data/lib/httpdisk/version.rb +1 -1
- metadata +26 -4
- data/lib/httpdisk/cli.rb +0 -218
- data/lib/httpdisk/cli_slop.rb +0 -54
@@ -0,0 +1,112 @@
|
|
1
|
+
require 'find'
|
2
|
+
require 'json'
|
3
|
+
|
4
|
+
module HTTPDisk
|
5
|
+
module Grep
|
6
|
+
class Main
|
7
|
+
attr_reader :options, :success
|
8
|
+
|
9
|
+
def initialize(options)
|
10
|
+
@options = options
|
11
|
+
end
|
12
|
+
|
13
|
+
# Enumerate file paths one at a time. Returns true if matches were found.
|
14
|
+
def run
|
15
|
+
paths.each do
|
16
|
+
begin
|
17
|
+
run_one(_1)
|
18
|
+
rescue StandardError => e
|
19
|
+
if ENV['HTTPDISK_DEBUG']
|
20
|
+
$stderr.puts
|
21
|
+
$stderr.puts e.class
|
22
|
+
$stderr.puts e.backtrace.join("\n")
|
23
|
+
end
|
24
|
+
raise CliError, "#{e.message[0, 70]} (#{_1})"
|
25
|
+
end
|
26
|
+
end
|
27
|
+
success
|
28
|
+
end
|
29
|
+
|
30
|
+
def run_one(path)
|
31
|
+
# read payload & body
|
32
|
+
payload = Zlib::GzipReader.open(path, encoding: 'ASCII-8BIT') do
|
33
|
+
Payload.read(_1)
|
34
|
+
end
|
35
|
+
body = prepare_body(payload)
|
36
|
+
|
37
|
+
# collect all_matches
|
38
|
+
all_matches = body.each_line.map do |line|
|
39
|
+
[].tap do |matches|
|
40
|
+
line.scan(pattern) { matches << Regexp.last_match }
|
41
|
+
end
|
42
|
+
end.reject(&:empty?)
|
43
|
+
return if all_matches.empty?
|
44
|
+
|
45
|
+
# print
|
46
|
+
@success = true
|
47
|
+
printer.print(path, payload, all_matches)
|
48
|
+
end
|
49
|
+
|
50
|
+
# file paths to be searched
|
51
|
+
def paths
|
52
|
+
# roots
|
53
|
+
roots = options[:roots]
|
54
|
+
roots = ['.'] if roots.empty?
|
55
|
+
|
56
|
+
# find files in roots
|
57
|
+
paths = roots.flat_map { Find.find(_1).to_a }.sort
|
58
|
+
paths = paths.select { File.file?(_1) }
|
59
|
+
|
60
|
+
# strip default './'
|
61
|
+
paths = paths.map { _1.gsub(%r{^\./}, '') } if options[:roots].empty?
|
62
|
+
paths
|
63
|
+
end
|
64
|
+
|
65
|
+
# convert raw body into something palatable for pattern matching
|
66
|
+
def prepare_body(payload)
|
67
|
+
body = payload.body
|
68
|
+
|
69
|
+
if content_type = payload.headers['Content-Type']
|
70
|
+
# Mismatches between Content-Type and body.encoding are fatal, so make
|
71
|
+
# an effort to align them.
|
72
|
+
if charset = content_type[/charset=([^;]+)/, 1]
|
73
|
+
encoding = begin
|
74
|
+
Encoding.find(charset)
|
75
|
+
rescue StandardError
|
76
|
+
nil
|
77
|
+
end
|
78
|
+
if encoding && body.encoding != encoding
|
79
|
+
body.force_encoding(encoding)
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
# pretty print json for easier searching
|
84
|
+
if content_type =~ /\bjson\b/
|
85
|
+
body = JSON.pretty_generate(JSON.parse(body))
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
body
|
90
|
+
end
|
91
|
+
|
92
|
+
# regex pattern from options
|
93
|
+
def pattern
|
94
|
+
@pattern ||= Regexp.new(options[:pattern], Regexp::IGNORECASE)
|
95
|
+
end
|
96
|
+
|
97
|
+
# printer for output
|
98
|
+
def printer
|
99
|
+
@printer ||= case
|
100
|
+
when options[:silent]
|
101
|
+
Grep::SilentPrinter.new
|
102
|
+
when options[:count]
|
103
|
+
Grep::CountPrinter.new($stdout)
|
104
|
+
when options[:head] || $stdout.tty?
|
105
|
+
Grep::HeaderPrinter.new($stdout, options[:head])
|
106
|
+
else
|
107
|
+
Grep::TersePrinter.new($stdout)
|
108
|
+
end
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|
112
|
+
end
|
@@ -0,0 +1,99 @@
|
|
1
|
+
module HTTPDisk
|
2
|
+
module Grep
|
3
|
+
class Printer
|
4
|
+
GREP_COLOR = '37;45'.freeze
|
5
|
+
|
6
|
+
attr_reader :output
|
7
|
+
|
8
|
+
def initialize(output)
|
9
|
+
@output = output
|
10
|
+
end
|
11
|
+
|
12
|
+
def print(path, payload, all_matches); end
|
13
|
+
|
14
|
+
protected
|
15
|
+
|
16
|
+
#
|
17
|
+
# helpers for subclasses
|
18
|
+
#
|
19
|
+
|
20
|
+
def grep_color
|
21
|
+
@grep_color ||= (ENV['GREP_COLOR'] || GREP_COLOR)
|
22
|
+
end
|
23
|
+
|
24
|
+
def print_matches(matches)
|
25
|
+
s = matches.first.string
|
26
|
+
if output.tty?
|
27
|
+
s = [].tap do |result|
|
28
|
+
ii = 0
|
29
|
+
matches.each do
|
30
|
+
result << s[ii..._1.begin(0)]
|
31
|
+
result << "\e["
|
32
|
+
result << grep_color
|
33
|
+
result << 'm'
|
34
|
+
result << _1[0]
|
35
|
+
result << "\e[0m"
|
36
|
+
ii = _1.end(0)
|
37
|
+
end
|
38
|
+
result << s[ii..]
|
39
|
+
end.join
|
40
|
+
end
|
41
|
+
output.puts s
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
#
|
46
|
+
# subclasses
|
47
|
+
#
|
48
|
+
|
49
|
+
# path:count
|
50
|
+
class CountPrinter < Printer
|
51
|
+
def print(path, _payload, all_matches)
|
52
|
+
output.puts "#{path}:#{all_matches.length}"
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
# header, then each match
|
57
|
+
class HeaderPrinter < Printer
|
58
|
+
attr_reader :head, :printed
|
59
|
+
|
60
|
+
def initialize(output, head)
|
61
|
+
super(output)
|
62
|
+
@head = head
|
63
|
+
@printed = 0
|
64
|
+
end
|
65
|
+
|
66
|
+
def print(path, payload, all_matches)
|
67
|
+
# separator & filename
|
68
|
+
output.puts if (@printed += 1) > 1
|
69
|
+
output.puts path
|
70
|
+
|
71
|
+
# --head
|
72
|
+
if head
|
73
|
+
io = StringIO.new
|
74
|
+
payload.write_header(io)
|
75
|
+
io.string.lines.each { output.puts "< #{_1}" }
|
76
|
+
end
|
77
|
+
|
78
|
+
# matches
|
79
|
+
all_matches.each { print_matches(_1) }
|
80
|
+
end
|
81
|
+
end
|
82
|
+
|
83
|
+
class SilentPrinter < Printer
|
84
|
+
def initialize
|
85
|
+
super(nil)
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
# each match as path:match
|
90
|
+
class TersePrinter < Printer
|
91
|
+
def print(path, _payload, all_matches)
|
92
|
+
all_matches.each do
|
93
|
+
output.write("#{path}:")
|
94
|
+
print_matches(_1)
|
95
|
+
end
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
99
|
+
end
|
data/lib/httpdisk/payload.rb
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
module HTTPDisk
|
2
2
|
class Payload
|
3
3
|
class << self
|
4
|
-
def read(f)
|
4
|
+
def read(f, peek: false)
|
5
5
|
Payload.new.tap do |p|
|
6
6
|
# comment
|
7
7
|
p.comment = f.gets[/^# (.*)/, 1]
|
8
8
|
|
9
9
|
# status line
|
10
|
-
m = f.gets.match(
|
10
|
+
m = f.gets.match(/^HTTPDISK (\d+) (.*)$/)
|
11
11
|
p.status, p.reason_phrase = m[1].to_i, m[2]
|
12
12
|
|
13
13
|
# headers
|
@@ -16,8 +16,8 @@ module HTTPDisk
|
|
16
16
|
p.headers[key] = value
|
17
17
|
end
|
18
18
|
|
19
|
-
# body
|
20
|
-
p.body = f.read
|
19
|
+
# body (if not peeking)
|
20
|
+
p.body = f.read if !peek
|
21
21
|
end
|
22
22
|
end
|
23
23
|
|
@@ -39,11 +39,11 @@ module HTTPDisk
|
|
39
39
|
@headers = Faraday::Utils::Headers.new
|
40
40
|
end
|
41
41
|
|
42
|
-
def
|
43
|
-
status
|
42
|
+
def error?
|
43
|
+
status >= 400
|
44
44
|
end
|
45
45
|
|
46
|
-
def
|
46
|
+
def write_header(f)
|
47
47
|
# comment
|
48
48
|
f.puts "# #{comment}"
|
49
49
|
|
@@ -52,9 +52,11 @@ module HTTPDisk
|
|
52
52
|
|
53
53
|
# headers
|
54
54
|
headers.each { f.puts("#{_1}: #{_2}") }
|
55
|
-
|
55
|
+
end
|
56
56
|
|
57
|
-
|
57
|
+
def write(f)
|
58
|
+
write_header(f)
|
59
|
+
f.puts
|
58
60
|
f.write(body)
|
59
61
|
end
|
60
62
|
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
require 'slop'
|
2
|
+
|
3
|
+
module Slop
|
4
|
+
# Custom duration type for Slop, used for --expires. Raises aggressively
|
5
|
+
# because this is a tricky and lightly documented option.
|
6
|
+
class DurationOption < Option
|
7
|
+
UNITS = {
|
8
|
+
s: 1,
|
9
|
+
m: 60,
|
10
|
+
h: 60 * 60,
|
11
|
+
d: 24 * 60 * 60,
|
12
|
+
w: 7 * 24 * 60 * 60,
|
13
|
+
y: 365 * 7 * 24 * 60 * 60,
|
14
|
+
}.freeze
|
15
|
+
|
16
|
+
def call(value)
|
17
|
+
m = value.match(/^(\d+)([smhdwy])?$/)
|
18
|
+
raise Slop::Error, "invalid --expires #{value.inspect}" if !m
|
19
|
+
|
20
|
+
num, unit = m[1].to_i, (m[2] || 's').to_sym
|
21
|
+
num * UNITS[unit]
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,105 @@
|
|
1
|
+
module HTTPDisk
|
2
|
+
# Like Slop, but for sanity checking method options. Useful for library entry
|
3
|
+
# points that want to be strict. Example usage:
|
4
|
+
#
|
5
|
+
# options = Sloptions.new(options) do
|
6
|
+
# _1.boolean :force
|
7
|
+
# _1.integer :retries, required: true
|
8
|
+
# _1.string :hello, default: 'world'
|
9
|
+
# ...
|
10
|
+
# end
|
11
|
+
class Sloptions
|
12
|
+
attr_reader :flags
|
13
|
+
|
14
|
+
def self.parse(options, &block)
|
15
|
+
Sloptions.new(&block).parse(options)
|
16
|
+
end
|
17
|
+
|
18
|
+
def initialize
|
19
|
+
@flags = {}
|
20
|
+
yield(self)
|
21
|
+
end
|
22
|
+
|
23
|
+
#
|
24
|
+
# _1.on and friends
|
25
|
+
#
|
26
|
+
|
27
|
+
def on(flag, foptions = {})
|
28
|
+
raise ":#{flag} already defined" if flags[flag]
|
29
|
+
|
30
|
+
flags[flag] = foptions
|
31
|
+
end
|
32
|
+
|
33
|
+
%i[array boolean float hash integer string symbol].each do |method|
|
34
|
+
define_method(method) do |flag, foptions = {}|
|
35
|
+
on(flag, { type: method }.merge(foptions))
|
36
|
+
end
|
37
|
+
end
|
38
|
+
alias bool boolean
|
39
|
+
|
40
|
+
#
|
41
|
+
# return parsed options
|
42
|
+
#
|
43
|
+
|
44
|
+
def parse(options)
|
45
|
+
# defaults
|
46
|
+
options = defaults.merge(options.compact)
|
47
|
+
|
48
|
+
flags.each do |flag, foptions|
|
49
|
+
# nil check
|
50
|
+
value = options[flag]
|
51
|
+
if value.nil?
|
52
|
+
raise ArgumentError, ":#{flag} is required" if foptions[:required]
|
53
|
+
|
54
|
+
next
|
55
|
+
end
|
56
|
+
|
57
|
+
# type cast (for boolean)
|
58
|
+
if foptions[:type] == :boolean
|
59
|
+
value = options[flag] = !!options[flag]
|
60
|
+
end
|
61
|
+
|
62
|
+
# type check
|
63
|
+
types = Array(foptions[:type])
|
64
|
+
raise ArgumentError, error_message(flag, value, types) if !valid?(value, types)
|
65
|
+
end
|
66
|
+
|
67
|
+
# return
|
68
|
+
options
|
69
|
+
end
|
70
|
+
|
71
|
+
protected
|
72
|
+
|
73
|
+
def defaults
|
74
|
+
flags.map { |flag, foptions| [flag, foptions[:default]] }.to_h.compact
|
75
|
+
end
|
76
|
+
|
77
|
+
# does value match valid?
|
78
|
+
def valid?(value, types)
|
79
|
+
types.any? do
|
80
|
+
case _1
|
81
|
+
when :array then true if value.is_a?(Array)
|
82
|
+
when :boolean then true # in Ruby everything is a boolean
|
83
|
+
when :float then true if value.is_a?(Float) || value.is_a?(Integer)
|
84
|
+
when :hash then true if value.is_a?(Hash)
|
85
|
+
when :integer then true if value.is_a?(Integer)
|
86
|
+
when :string then true if value.is_a?(String)
|
87
|
+
when :symbol then true if value.is_a?(Symbol)
|
88
|
+
when Class then true if value.is_a?(_1) # for custom checks
|
89
|
+
else
|
90
|
+
raise "unknown flag type #{_1.inspect}"
|
91
|
+
end
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
# nice error message for when value is invalid
|
96
|
+
def error_message(flag, value, valid)
|
97
|
+
classes = valid.compact.map do
|
98
|
+
s = _1.to_s
|
99
|
+
s = s.downcase if s =~ /\b(Array|Float|Hash|Integer|String|Symbol)\b/
|
100
|
+
s
|
101
|
+
end.join('/')
|
102
|
+
"expected :#{flag} to be #{classes}, not #{value.inspect}"
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
data/lib/httpdisk/version.rb
CHANGED
metadata
CHANGED
@@ -1,15 +1,29 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: httpdisk
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1
|
4
|
+
version: 0.5.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Adam Doppelt
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2021-
|
11
|
+
date: 2021-07-05 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: content-type
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '0.0'
|
20
|
+
type: :runtime
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '0.0'
|
13
27
|
- !ruby/object:Gem::Dependency
|
14
28
|
name: faraday
|
15
29
|
requirement: !ruby/object:Gem::Requirement
|
@@ -70,27 +84,35 @@ description: httpdisk works with faraday to aggressively cache responses on disk
|
|
70
84
|
email: amd@gurge.com
|
71
85
|
executables:
|
72
86
|
- httpdisk
|
87
|
+
- httpdisk-grep
|
73
88
|
extensions: []
|
74
89
|
extra_rdoc_files: []
|
75
90
|
files:
|
76
91
|
- ".github/workflows/test.yml"
|
77
92
|
- ".gitignore"
|
93
|
+
- ".rubocop.yml"
|
78
94
|
- Gemfile
|
79
95
|
- Gemfile.lock
|
80
96
|
- LICENSE
|
81
97
|
- README.md
|
82
98
|
- Rakefile
|
83
99
|
- bin/httpdisk
|
100
|
+
- bin/httpdisk-grep
|
84
101
|
- examples.rb
|
85
102
|
- httpdisk.gemspec
|
86
103
|
- lib/httpdisk.rb
|
87
104
|
- lib/httpdisk/cache.rb
|
88
105
|
- lib/httpdisk/cache_key.rb
|
89
|
-
- lib/httpdisk/cli.rb
|
90
|
-
- lib/httpdisk/
|
106
|
+
- lib/httpdisk/cli/args.rb
|
107
|
+
- lib/httpdisk/cli/main.rb
|
91
108
|
- lib/httpdisk/client.rb
|
92
109
|
- lib/httpdisk/error.rb
|
110
|
+
- lib/httpdisk/grep/args.rb
|
111
|
+
- lib/httpdisk/grep/main.rb
|
112
|
+
- lib/httpdisk/grep/printer.rb
|
93
113
|
- lib/httpdisk/payload.rb
|
114
|
+
- lib/httpdisk/slop_duration.rb
|
115
|
+
- lib/httpdisk/sloptions.rb
|
94
116
|
- lib/httpdisk/version.rb
|
95
117
|
- logo.svg
|
96
118
|
homepage: http://github.com/gurgeous/httpdisk
|