httpdisk 0.5.1 → 1.0.0

data/lib/httpdisk/cache_key.rb

@@ -1,6 +1,6 @@
-require 'cgi'
-require 'digest/md5'
-require 'uri'
+require "cgi"
+require "digest/md5"
+require "uri"
 
 module HTTPDisk
   class CacheKey
@@ -10,7 +10,7 @@ module HTTPDisk
       @env, @ignore_params = env, ignore_params
 
       # sanity checks
-      raise InvalidUrl, "http/https required #{env.url.inspect}" if env.url.scheme !~ /^https?$/
+      raise InvalidUrl, "http/https required #{env.url.inspect}" if !/^https?$/.match?(env.url.scheme)
       raise InvalidUrl, "hostname required #{env.url.inspect}" if !env.url.host
     end
 
@@ -43,23 +43,23 @@ module HTTPDisk
     def calculate_key
       key = []
       key << env.method.upcase
-      key << ' '
+      key << " "
       key << url.scheme
-      key << '://'
+      key << "://"
       key << url.host.downcase
       if !default_port?
-        key << ':'
+        key << ":"
         key << url.port
       end
-      if url.path != '/'
+      if url.path != "/"
         key << url.path
       end
-      if (q = url.query) && q != ''
-        key << '?'
+      if (q = url.query) && q != ""
+        key << "?"
         key << querykey(q)
       end
       if env.request_body
-        key << ' '
+        key << " "
         key << bodykey
       end
       key.join
@@ -68,7 +68,7 @@ module HTTPDisk
     # Calculate cache key segment for body
     def bodykey
       body = env.request_body.to_s
-      if env.request_headers['Content-Type'] == 'application/x-www-form-urlencoded'
+      if env.request_headers["Content-Type"] == "application/x-www-form-urlencoded"
         querykey(body)
       elsif body.length < 50
         body
@@ -79,16 +79,16 @@ module HTTPDisk
 
     # Calculate canonical key for a query
     def querykey(q)
-      parts = q.split('&').sort
+      parts = q.split("&").sort
       if !ignore_params.empty?
         parts = parts.map do |part|
-          key, value = part.split('=', 2)
+          key, value = part.split("=", 2)
           next if ignore_params.include?(key)
 
           "#{key}=#{value}"
         end.compact
       end
-      parts.join('&')
+      parts.join("&")
     end
 
     def default_port?
@@ -98,10 +98,10 @@ module HTTPDisk
     # Calculate nice directory name from url.host
     def hostdir
       hostdir = url.host.downcase
-      hostdir = hostdir.gsub(/^www\./, '')
-      hostdir = hostdir.gsub(/[^a-z0-9._-]/, '')
-      hostdir = hostdir.squeeze('.')
-      hostdir = 'any' if hostdir.empty?
+      hostdir = hostdir.gsub(/^www\./, "")
+      hostdir = hostdir.gsub(/[^a-z0-9._-]/, "")
+      hostdir = hostdir.squeeze(".")
+      hostdir = "any" if hostdir.empty?
       hostdir
     end
   end

data/lib/httpdisk/cli/args.rb

@@ -1,8 +1,8 @@
 # manually load dependencies here since this is loaded standalone by bin
-require 'httpdisk/error'
-require 'httpdisk/slop_duration'
-require 'httpdisk/version'
-require 'slop'
+require "httpdisk/error"
+require "httpdisk/slop_duration"
+require "httpdisk/version"
+require "slop"
 
 module HTTPDisk
   module Cli
@@ -10,43 +10,43 @@ module HTTPDisk
     module Args
       def self.slop(args)
         slop = Slop.parse(args) do |o|
-          o.banner = 'httpdisk [options] [url]'
+          o.banner = "httpdisk [options] [url]"
 
           # similar to curl
-          o.separator 'Similar to curl:'
-          o.string '-d', '--data', 'HTTP POST data'
-          o.array '-H', '--header', 'pass custom header(s) to server', delimiter: nil
-          o.boolean '-i', '--include', 'include response headers in the output'
-          o.integer '-m', '--max-time', 'maximum time allowed for the transfer'
-          o.string '-o', '--output', 'write to file instead of stdout'
-          o.string '-x', '--proxy', 'use host[:port] as proxy'
-          o.string '-X', '--request', 'HTTP method to use'
-          o.integer '--retry', 'retry request if problems occur'
-          o.boolean '-s', '--silent', "silent mode (don't print errors)"
-          o.string '-A', '--user-agent', 'send User-Agent to server'
+          o.separator "Similar to curl:"
+          o.string "-d", "--data", "HTTP POST data"
+          o.array "-H", "--header", "pass custom header(s) to server", delimiter: nil
+          o.boolean "-i", "--include", "include response headers in the output"
+          o.integer "-m", "--max-time", "maximum time allowed for the transfer"
+          o.string "-o", "--output", "write to file instead of stdout"
+          o.string "-x", "--proxy", "use host[:port] as proxy"
+          o.string "-X", "--request", "HTTP method to use"
+          o.integer "--retry", "retry request if problems occur"
+          o.boolean "-s", "--silent", "silent mode (don't print errors)"
+          o.string "-A", "--user-agent", "send User-Agent to server"
 
           # from httpdisk
-          o.separator 'Specific to httpdisk:'
-          o.string '--dir', 'httpdisk cache directory (defaults to ~/httpdisk)'
-          o.duration '--expires', 'when to expire cached requests (ex: 1h, 2d, 3w)'
-          o.boolean '--force', "don't read anything from cache (but still write)"
-          o.boolean '--force-errors', "don't read errors from cache (but still write)"
-          o.boolean '--status', 'show status for a url in the cache'
+          o.separator "Specific to httpdisk:"
+          o.string "--dir", "httpdisk cache directory (defaults to ~/httpdisk)"
+          o.duration "--expires", "when to expire cached requests (ex: 1h, 2d, 3w)"
+          o.boolean "--force", "don't read anything from cache (but still write)"
+          o.boolean "--force-errors", "don't read errors from cache (but still write)"
+          o.boolean "--status", "show status for a url in the cache"
 
           # generic
-          o.boolean '--version', 'show version' do
+          o.boolean "--version", "show version" do
             puts "httpdisk #{HTTPDisk::VERSION}"
             exit
           end
-          o.on '--help', 'show this help' do
+          o.on "--help", "show this help" do
             puts o
             exit
           end
         end
 
-        raise Slop::Error, '' if args.empty?
-        raise Slop::Error, 'no URL specified' if slop.args.empty?
-        raise Slop::Error, 'more than one URL specified' if slop.args.length > 1
+        raise Slop::Error, "" if args.empty?
+        raise Slop::Error, "no URL specified" if slop.args.empty?
+        raise Slop::Error, "more than one URL specified" if slop.args.length > 1
 
         slop.to_h.tap do
           _1[:url] = slop.args.first

data/lib/httpdisk/cli/main.rb

@@ -1,6 +1,6 @@
-require 'faraday-cookie_jar'
-require 'faraday_middleware'
-require 'ostruct'
+require "faraday-cookie_jar"
+require "faraday/follow_redirects"
+require "ostruct"
 
 module HTTPDisk
   module Cli
@@ -31,7 +31,7 @@ module HTTPDisk
 
         # output
         if options[:output]
-          File.open(options[:output], 'w') { output(response, _1) }
+          File.open(options[:output], "w") { output(response, _1) }
         else
           output(response, $stdout)
         end
@@ -59,7 +59,7 @@ module HTTPDisk
               max: options[:retry],
               methods: %w[delete get head options patch post put trace],
               retry_statuses: (500..600).to_a,
-              retry_if: ->(_env, _err) { true },
+              retry_if: ->(_env, _err) { true }
             }
             _1.request :retry, retry_options
           end
@@ -103,9 +103,9 @@ module HTTPDisk
         method = if options[:request]
           options[:request]
         elsif options[:data]
-          'post'
+          "post"
         end
-        method ||= 'get'
+        method ||= "get"
         method = method.downcase.to_sym
 
         if !Faraday::Connection::METHODS.include?(method)
@@ -119,9 +119,9 @@ module HTTPDisk
       def request_url
         url = options[:url]
         # recover from missing http:
-        if url !~ %r{^https?://}i
-          if url =~ %r{^\w+://}
-            raise CliError, 'only http/https supported'
+        if !%r{^https?://}i.match?(url)
+          if %r{^\w+://}.match?(url)
+            raise CliError, "only http/https supported"
           end
 
           url = "http://#{url}"
@@ -140,11 +140,11 @@ module HTTPDisk
       def request_headers
         {}.tap do |headers|
           if options[:user_agent]
-            headers['User-Agent'] = options[:user_agent]
+            headers["User-Agent"] = options[:user_agent]
           end
 
           options[:header].each do |header|
-            key, value = header.split(': ', 2)
+            key, value = header.split(": ", 2)
             if !key || !value || key.empty? || value.empty?
               raise CliError, "invalid --header #{header.inspect}"
             end

data/lib/httpdisk/client.rb

@@ -1,6 +1,6 @@
-require 'content-type'
-require 'faraday'
-require 'logger'
+require "content-type"
+require "faraday"
+require "logger"
 
 module HTTPDisk
   # Middleware and main entry point.
@@ -9,7 +9,7 @@ module HTTPDisk
 
     def initialize(app, options = {})
       options = Sloptions.parse(options) do
-        _1.string :dir, default: File.join(ENV['HOME'], 'httpdisk')
+        _1.string :dir, default: File.join(ENV["HOME"], "httpdisk")
         _1.integer :expires
         _1.boolean :force
         _1.boolean :force_errors
@@ -23,12 +23,12 @@ module HTTPDisk
     end
 
     def call(env)
-      cache_key = CacheKey.new(env, ignore_params: ignore_params)
+      cache_key = CacheKey.new(env, ignore_params:)
       logger&.info("#{env.method.upcase} #{env.url} (#{cache.status(cache_key)})")
       env[:httpdisk_diskpath] = cache.diskpath(cache_key)
 
       # check cache, fallback to network
-      if response = read(cache_key, env)
+      if (response = read(cache_key, env))
         response.env[:httpdisk] = true
       else
         response = perform(env)
@@ -48,7 +48,7 @@ module HTTPDisk
         status: cache.status(cache_key).to_s,
         key: cache_key.key,
         digest: cache_key.digest,
-        path: cache.diskpath(cache_key),
+        path: cache.diskpath(cache_key)
       }
     end
 
@@ -90,7 +90,7 @@ module HTTPDisk
     def stuff_999_response(env, err)
       env.tap do
         _1.reason_phrase = "#{err.class} #{err.message}"
-        _1.response_body = ''
+        _1.response_body = ""
         _1.response_headers = Faraday::Utils::Headers.new
         _1.status = HTTPDisk::ERROR_STATUS
       end
@@ -110,11 +110,11 @@ module HTTPDisk
     # network. Not all adapters honor Content-Type (including the default
     # adapter).
     def encode_body(response)
-      body = response.body || ''
+      body = response.body || ""
 
       # parse Content-Type
       begin
-        content_type = response['Content-Type'] && ContentType.parse(response['Content-Type'])
+        content_type = response["Content-Type"] && ContentType.parse(response["Content-Type"])
       rescue Parslet::ParseFailed
         # unparsable
       end
@@ -130,7 +130,7 @@ module HTTPDisk
       if options[:utf8] && content_type && response_text?(content_type)
         body = body.dup if body.frozen?
         begin
-          body.encode!('UTF-8', invalid: :replace, undef: :replace, replace: '?')
+          body.encode!("UTF-8", invalid: :replace, undef: :replace, replace: "?")
         rescue Encoding::ConverterNotFoundError
           # rare, can't do anything here
           body = "httpdisk could not convert from #{body.encoding.name} to UTF-8"
@@ -152,7 +152,7 @@ module HTTPDisk
     end
 
     def response_text?(content_type)
-      content_type.type == 'text' || content_type.mime_type == 'application/json'
+      content_type.type == "text" || content_type.mime_type == "application/json"
     end
 
     #

data/lib/httpdisk/grep/args.rb

@@ -1,6 +1,6 @@
 # manually load dependencies here since this is loaded standalone by bin
-require 'httpdisk/version'
-require 'slop'
+require "httpdisk/version"
+require "slop"
 
 module HTTPDisk
   module Grep
@@ -8,22 +8,22 @@ module HTTPDisk
       # Slop parsing. This is broken out so we can run without require 'httpdisk'.
       def self.slop(args)
         slop = Slop.parse(args) do |o|
-          o.banner = 'httpdisk-grep [options] pattern [path ...]'
-          o.boolean '-c', '--count', 'suppress normal output and show count'
-          o.boolean '-h', '--head', 'show req headers before each match'
-          o.boolean '-s', '--silent', 'do not print anything to stdout'
-          o.boolean '--version', 'show version' do
+          o.banner = "httpdisk-grep [options] pattern [path ...]"
+          o.boolean "-c", "--count", "suppress normal output and show count"
+          o.boolean "-h", "--head", "show req headers before each match"
+          o.boolean "-s", "--silent", "do not print anything to stdout"
+          o.boolean "--version", "show version" do
             puts "httpdisk-grep #{HTTPDisk::VERSION}"
             exit
           end
-          o.on '--help', 'show this help' do
+          o.on "--help", "show this help" do
             puts o
             exit
           end
         end
 
-        raise Slop::Error, '' if args.empty?
-        raise Slop::Error, 'no PATTERN specified' if slop.args.empty?
+        raise Slop::Error, "" if args.empty?
+        raise Slop::Error, "no PATTERN specified" if slop.args.empty?
 
         slop.to_h.tap do
           _1[:pattern] = slop.args.shift

data/lib/httpdisk/grep/main.rb

@@ -1,5 +1,5 @@
-require 'find'
-require 'json'
+require "find"
+require "json"
 
 module HTTPDisk
   module Grep
@@ -13,25 +13,29 @@ module HTTPDisk
       # Enumerate file paths one at a time. Returns true if matches were found.
       def run
         paths.each do
-          begin
-            run_one(_1)
-          rescue StandardError => e
-            if ENV['HTTPDISK_DEBUG']
-              $stderr.puts
-              $stderr.puts e.class
-              $stderr.puts e.backtrace.join("\n")
-            end
-            raise CliError, "#{e.message[0, 70]} (#{_1})"
-          end
+          run_one(_1)
+      rescue => e
+        if ENV["HTTPDISK_DEBUG"]
+          $stderr.puts
+          warn e.class
+          warn e.backtrace.join("\n")
+        end
+        raise CliError, "#{e.message[0, 70]} (#{_1})"
         end
         success
       end
 
       def run_one(path)
         # read payload & body
-        payload = Zlib::GzipReader.open(path, encoding: 'ASCII-8BIT') do
-          Payload.read(_1)
+        begin
+          payload = Zlib::GzipReader.open(path, encoding: "ASCII-8BIT") do
+            Payload.read(_1)
+          end
+        rescue Zlib::GzipFile::Error
+          puts "httpdisk: #{path} not in gzip format, skipping" if !options[:silent]
+          return
         end
+
         body = prepare_body(payload)
 
         # collect all_matches
@@ -51,14 +55,14 @@ module HTTPDisk
       def paths
         # roots
         roots = options[:roots]
-        roots = ['.'] if roots.empty?
+        roots = ["."] if roots.empty?
 
         # find files in roots
         paths = roots.flat_map { Find.find(_1).to_a }.sort
         paths = paths.select { File.file?(_1) }
 
         # strip default './'
-        paths = paths.map { _1.gsub(%r{^\./}, '') } if options[:roots].empty?
+        paths = paths.map { _1.gsub(%r{^\./}, "") } if options[:roots].empty?
         paths
       end
 
@@ -66,13 +70,13 @@ module HTTPDisk
       def prepare_body(payload)
         body = payload.body
 
-        if content_type = payload.headers['Content-Type']
+        if (content_type = payload.headers["Content-Type"])
           # Mismatches between Content-Type and body.encoding are fatal, so make
           # an effort to align them.
-          if charset = content_type[/charset=([^;]+)/, 1]
+          if (charset = content_type[/charset=([^;]+)/, 1])
             encoding = begin
               Encoding.find(charset)
-            rescue StandardError
+            rescue
               nil
             end
             if encoding && body.encoding != encoding
@@ -81,7 +85,7 @@ module HTTPDisk
           end
 
           # pretty print json for easier searching
-          if content_type =~ /\bjson\b/
+          if /\bjson\b/.match?(content_type)
             body = JSON.pretty_generate(JSON.parse(body))
           end
         end
@@ -96,12 +100,11 @@ module HTTPDisk
 
       # printer for output
       def printer
-        @printer ||= case
-        when options[:silent]
+        @printer ||= if options[:silent]
           Grep::SilentPrinter.new
-        when options[:count]
+        elsif options[:count]
           Grep::CountPrinter.new($stdout)
-        when options[:head] || $stdout.tty?
+        elsif options[:head] || $stdout.tty?
           Grep::HeaderPrinter.new($stdout, options[:head])
         else
           Grep::TersePrinter.new($stdout)

data/lib/httpdisk/grep/printer.rb

@@ -1,7 +1,7 @@
 module HTTPDisk
   module Grep
     class Printer
-      GREP_COLOR = '37;45'.freeze
+      GREP_COLOR = "37;45".freeze
 
       attr_reader :output
 
@@ -9,7 +9,8 @@ module HTTPDisk
         @output = output
       end
 
-      def print(path, payload, all_matches); end
+      def print(path, payload, all_matches)
+      end
 
       protected
 
@@ -18,7 +19,7 @@ module HTTPDisk
       #
 
       def grep_color
-        @grep_color ||= (ENV['GREP_COLOR'] || GREP_COLOR)
+        @grep_color ||= (ENV["GREP_COLOR"] || GREP_COLOR)
       end
 
       def print_matches(matches)
@@ -30,7 +31,7 @@ module HTTPDisk
               result << s[ii..._1.begin(0)]
               result << "\e["
               result << grep_color
-              result << 'm'
+              result << "m"
               result << _1[0]
               result << "\e[0m"
               ii = _1.end(0)

data/lib/httpdisk/payload.rb

@@ -12,7 +12,7 @@ module HTTPDisk
 
           # headers
           while (line = f.gets.chomp) && !line.empty?
-            key, value = line.split(': ', 2)
+            key, value = line.split(": ", 2)
             p.headers[key] = value
           end
 
@@ -34,8 +34,8 @@ module HTTPDisk
     attr_accessor :body, :comment, :headers, :reason_phrase, :status
 
     def initialize
-      @body = ''
-      @comment = ''
+      @body = ""
+      @comment = ""
       @headers = Faraday::Utils::Headers.new
     end
 

data/lib/httpdisk/slop_duration.rb

@@ -1,4 +1,4 @@
-require 'slop'
+require "slop"
 
 module Slop
   # Custom duration type for Slop, used for --expires. Raises aggressively
@@ -10,14 +10,14 @@ module Slop
       h: 60 * 60,
       d: 24 * 60 * 60,
       w: 7 * 24 * 60 * 60,
-      y: 365 * 7 * 24 * 60 * 60,
+      y: 365 * 7 * 24 * 60 * 60
     }.freeze
 
     def call(value)
       m = value.match(/^(\d+)([smhdwy])?$/)
       raise Slop::Error, "invalid --expires #{value.inspect}" if !m
 
-      num, unit = m[1].to_i, (m[2] || 's').to_sym
+      num, unit = m[1].to_i, (m[2] || "s").to_sym
       num * UNITS[unit]
     end
   end

data/lib/httpdisk/sloptions.rb

@@ -32,10 +32,10 @@ module HTTPDisk
 
     %i[array boolean float hash integer string symbol].each do |method|
       define_method(method) do |flag, foptions = {}|
-        on(flag, { type: method }.merge(foptions))
+        on(flag, {type: method}.merge(foptions))
       end
     end
-    alias bool boolean
+    alias_method :bool, :boolean
 
     #
     # return parsed options
@@ -96,9 +96,9 @@ module HTTPDisk
     def error_message(flag, value, valid)
       classes = valid.compact.map do
         s = _1.to_s
-        s = s.downcase if s =~ /\b(Array|Float|Hash|Integer|String|Symbol)\b/
+        s = s.downcase if /\b(Array|Float|Hash|Integer|String|Symbol)\b/.match?(s)
         s
-      end.join('/')
+      end.join("/")
       "expected :#{flag} to be #{classes}, not #{value.inspect}"
     end
   end

data/lib/httpdisk/version.rb

@@ -1,3 +1,3 @@
 module HTTPDisk
-  VERSION = '0.5.1'.freeze
+  VERSION = "1.0.0".freeze
 end

data/lib/httpdisk.rb

@@ -1,17 +1,17 @@
-require 'httpdisk/cache_key'
-require 'httpdisk/cache'
-require 'httpdisk/client'
-require 'httpdisk/error'
-require 'httpdisk/payload'
-require 'httpdisk/slop_duration'
-require 'httpdisk/sloptions'
-require 'httpdisk/version'
+require "httpdisk/cache_key"
+require "httpdisk/cache"
+require "httpdisk/client"
+require "httpdisk/error"
+require "httpdisk/payload"
+require "httpdisk/slop_duration"
+require "httpdisk/sloptions"
+require "httpdisk/version"
 
 # cli
-require 'httpdisk/cli/args'
-require 'httpdisk/cli/main'
+require "httpdisk/cli/args"
+require "httpdisk/cli/main"
 
 # grep
-require 'httpdisk/grep/args'
-require 'httpdisk/grep/main'
-require 'httpdisk/grep/printer'
+require "httpdisk/grep/args"
+require "httpdisk/grep/main"
+require "httpdisk/grep/printer"