httpdisk 0.5.2 → 1.0.1

data/lib/httpdisk/cache.rb

@@ -1,8 +1,9 @@
-require 'fileutils'
-require 'tempfile'
+require "fileutils"
+require "tempfile"
+require "zlib"
 
 module HTTPDisk
-  # Disk cache for cache_keys => response. Files are compressed.
+  # Disk cache for cache_keys => response. Files may be compressed or plain.
   class Cache
     attr_reader :options
 
@@ -10,13 +11,14 @@ module HTTPDisk
       @options = options
     end
 
-    %i[dir expires force force_errors].each do |method|
+    %i[compress dir expires force force_errors].each do |method|
       define_method(method) do
         options[method]
       end
     end
-    alias force? force
-    alias force_errors? force_errors
+    alias_method :compress?, :compress
+    alias_method :force?, :force
+    alias_method :force_errors?, :force_errors
 
     # Get cached response. If there is a cached error it will be raised.
     def read(cache_key)
@@ -37,15 +39,11 @@ module HTTPDisk
       path = diskpath(cache_key)
       FileUtils.mkdir_p(File.dirname(path))
 
-      # Atomically write gzipped payload. Put our underlying Tempfile into
-      # binmode to avoid accidental newline conversion or string encoding. Not
-      # required for *nix systems, but I've heard rumors it's helpful for
-      # Windows.
+      # Atomically write payload. Put our underlying Tempfile into binmode to
+      # avoid accidental newline conversion or string encoding. Not required for
+      # *nix systems, but I've heard rumors it's helpful for Windows.
       Tempfile.new(binmode: true).tap do |tmp|
-        Zlib::GzipWriter.new(tmp).tap do |gzip|
-          payload.write(gzip)
-          gzip.close
-        end
+        write_payload(tmp, payload)
         tmp.close
         FileUtils.mv(tmp.path, path)
       end
@@ -73,10 +71,8 @@ module HTTPDisk
       return :force if force?
 
       begin
-        payload = Zlib::GzipReader.open(path, encoding: 'ASCII-8BIT') do
-          Payload.read(_1, peek: peek)
-        end
-      rescue StandardError => e
+        payload = read_payload(path, peek:)
+      rescue => e
         raise "#{path}: #{e}"
       end
       return :force if force_errors? && payload.error?
@@ -88,5 +84,27 @@ module HTTPDisk
     def expired?(path)
       expires && File.stat(path).mtime < Time.now - expires
     end
+
+    def read_payload(path, peek:)
+      if compressed?(path)
+        Zlib::GzipReader.open(path, encoding: "ASCII-8BIT") { Payload.read(_1, peek:) }
+      else
+        File.open(path, "rb") { Payload.read(_1, peek:) }
+      end
+    end
+
+    def write_payload(tmp, payload)
+      if compress?
+        Zlib::GzipWriter.new(tmp).tap do |gzip|
+          payload.write(gzip)
+          gzip.close
+        end
+      else
+        payload.write(tmp)
+      end
+    end
+
+    # check got gz magic
+    def compressed?(path) = File.binread(path, 2) == "\x1f\x8b".b
   end
 end

data/lib/httpdisk/cache_key.rb

@@ -1,66 +1,89 @@
-require 'cgi'
-require 'digest/md5'
-require 'uri'
+require "cgi"
+require "digest/md5"
+require "uri"
 
 module HTTPDisk
   class CacheKey
-    attr_reader :env, :ignore_params
+    PARTS = %i[http_method scheme host port path query body].freeze
+
+    attr_accessor :env, :ignore_params
+    attr_reader(*PARTS)
 
     def initialize(env, ignore_params: [])
       @env, @ignore_params = env, ignore_params
 
+      # setup defaults, user can override
+      @http_method = env.method
+      @scheme = url.scheme
+      @host = url.host
+      @port = default_port? ? nil : url.port
+      @path = (url.path == "/") ? nil : url.path
+      @query = url.query
+      @body = env.request_body ? bodykey : nil
+
       # sanity checks
-      raise InvalidUrl, "http/https required #{env.url.inspect}" if env.url.scheme !~ /^https?$/
-      raise InvalidUrl, "hostname required #{env.url.inspect}" if !env.url.host
+      raise InvalidUrl, "http/https required #{env.url.inspect}" if !/^https?$/.match?(scheme)
+      raise InvalidUrl, "hostname required #{env.url.inspect}" if !host
     end
 
-    def url
-      env.url
-    end
+    def query_params
+      return {} if blank?(query)
 
-    # Cache key (memoized)
-    def key
-      @key ||= calculate_key
+      CGI.parse(query).transform_values do
+        (_1.length == 1) ? _1.first : _1
+      end
     end
 
-    # md5(key) (memoized)
-    def digest
-      @digest ||= Digest::MD5.hexdigest(key)
-    end
+    # one-liners
+    def blank?(value) = value.nil? || value == ""
+    def default_port? = url.default_port == url.port
+    def digest = @digest ||= Digest::MD5.hexdigest(key)
+    def diskpath = @diskpath ||= File.join(hostdir, digest[0, 3], digest[3..])
+    def invalidate! = @digest = @key = @diskpath = nil
+    def key = @key ||= key0
+    def to_s = key
+    def url = env.url
 
-    # Relative path for this cache key based on hostdir & digest.
-    def diskpath
-      @diskpath ||= File.join(hostdir, digest[0, 3], digest[3..])
+    def query_params=(value)
+      self.query = URI.encode_www_form(flatten_query_params(value))
     end
 
-    def to_s
-      key
+    # the parts that feed into `key` invalidate memoized stuff
+    PARTS.each do |part|
+      define_method("#{part}=") do |value|
+        invalidate!
+        instance_variable_set(:"@#{part}", value)
+      end
     end
 
     protected
 
-    # Calculate cache key for url
-    def calculate_key
+    def key0
       key = []
-      key << env.method.upcase
-      key << ' '
-      key << url.scheme
-      key << '://'
-      key << url.host.downcase
-      if !default_port?
-        key << ':'
-        key << url.port
+      key << http_method.to_s.upcase
+      key << " "
+      key << scheme.to_s
+      key << "://"
+      key << host.downcase
+      if port
+        key << ":"
+        key << port
+      end
+      if (path = (blank?(self.path) || self.path == "/") ? nil : self.path)
+        key << path
       end
-      if url.path != '/'
-        key << url.path
+      if (query = canonical_query(self.query))
+        key << "?"
+        key << query
       end
-      if (q = url.query) && q != ''
-        key << '?'
-        key << querykey(q)
+      body = if env.request_headers["Content-Type"] == "application/x-www-form-urlencoded"
+        canonical_query(self.body)
+      else
+        self.body
       end
-      if env.request_body
-        key << ' '
-        key << bodykey
+      if body
+        key << " "
+        key << body
       end
       key.join
     end
@@ -68,8 +91,8 @@ module HTTPDisk
     # Calculate cache key segment for body
     def bodykey
       body = env.request_body.to_s
-      if env.request_headers['Content-Type'] == 'application/x-www-form-urlencoded'
-        querykey(body)
+      if env.request_headers["Content-Type"] == "application/x-www-form-urlencoded"
+        canonical_query(body)
       elsif body.length < 50
         body
       else
@@ -77,32 +100,40 @@ module HTTPDisk
       end
     end
 
-    # Calculate canonical key for a query
-    def querykey(q)
-      parts = q.split('&').sort
+    def canonical_query(q)
+      return if blank?(q)
+
+      parts = q.split("&").sort
       if !ignore_params.empty?
-        parts = parts.map do |part|
-          key, value = part.split('=', 2)
+        parts = parts.filter_map do |part|
+          key, value = part.split("=", 2)
           next if ignore_params.include?(key)
 
           "#{key}=#{value}"
-        end.compact
+        end
       end
-      parts.join('&')
-    end
-
-    def default_port?
-      url.default_port == url.port
+      query = parts.join("&")
+      (query == "") ? nil : query
     end
 
     # Calculate nice directory name from url.host
     def hostdir
-      hostdir = url.host.downcase
-      hostdir = hostdir.gsub(/^www\./, '')
-      hostdir = hostdir.gsub(/[^a-z0-9._-]/, '')
-      hostdir = hostdir.squeeze('.')
-      hostdir = 'any' if hostdir.empty?
+      hostdir = host.downcase
+      hostdir = hostdir.gsub(/^www\./, "")
+      hostdir = hostdir.gsub(/[^a-z0-9._-]/, "")
+      hostdir = hostdir.squeeze(".")
+      hostdir = "any" if hostdir.empty?
       hostdir
     end
+
+    def flatten_query_params(value)
+      value.flat_map do |k, v|
+        if v.is_a?(Array)
+          v.map { [k, _1] }
+        else
+          [[k, v]]
+        end
+      end
+    end
   end
 end

data/lib/httpdisk/cli/args.rb

@@ -1,8 +1,8 @@
 # manually load dependencies here since this is loaded standalone by bin
-require 'httpdisk/error'
-require 'httpdisk/slop_duration'
-require 'httpdisk/version'
-require 'slop'
+require "httpdisk/error"
+require "httpdisk/slop_duration"
+require "httpdisk/version"
+require "slop"
 
 module HTTPDisk
   module Cli
@@ -10,43 +10,43 @@ module HTTPDisk
     module Args
       def self.slop(args)
         slop = Slop.parse(args) do |o|
-          o.banner = 'httpdisk [options] [url]'
+          o.banner = "httpdisk [options] [url]"
 
           # similar to curl
-          o.separator 'Similar to curl:'
-          o.string '-d', '--data', 'HTTP POST data'
-          o.array '-H', '--header', 'pass custom header(s) to server', delimiter: nil
-          o.boolean '-i', '--include', 'include response headers in the output'
-          o.integer '-m', '--max-time', 'maximum time allowed for the transfer'
-          o.string '-o', '--output', 'write to file instead of stdout'
-          o.string '-x', '--proxy', 'use host[:port] as proxy'
-          o.string '-X', '--request', 'HTTP method to use'
-          o.integer '--retry', 'retry request if problems occur'
-          o.boolean '-s', '--silent', "silent mode (don't print errors)"
-          o.string '-A', '--user-agent', 'send User-Agent to server'
+          o.separator "Similar to curl:"
+          o.string "-d", "--data", "HTTP POST data"
+          o.array "-H", "--header", "pass custom header(s) to server", delimiter: nil
+          o.boolean "-i", "--include", "include response headers in the output"
+          o.integer "-m", "--max-time", "maximum time allowed for the transfer"
+          o.string "-o", "--output", "write to file instead of stdout"
+          o.string "-x", "--proxy", "use host[:port] as proxy"
+          o.string "-X", "--request", "HTTP method to use"
+          o.integer "--retry", "retry request if problems occur"
+          o.boolean "-s", "--silent", "silent mode (don't print errors)"
+          o.string "-A", "--user-agent", "send User-Agent to server"
 
           # from httpdisk
-          o.separator 'Specific to httpdisk:'
-          o.string '--dir', 'httpdisk cache directory (defaults to ~/httpdisk)'
-          o.duration '--expires', 'when to expire cached requests (ex: 1h, 2d, 3w)'
-          o.boolean '--force', "don't read anything from cache (but still write)"
-          o.boolean '--force-errors', "don't read errors from cache (but still write)"
-          o.boolean '--status', 'show status for a url in the cache'
+          o.separator "Specific to httpdisk:"
+          o.string "--dir", "httpdisk cache directory (defaults to ~/httpdisk)"
+          o.duration "--expires", "when to expire cached requests (ex: 1h, 2d, 3w)"
+          o.boolean "--force", "don't read anything from cache (but still write)"
+          o.boolean "--force-errors", "don't read errors from cache (but still write)"
+          o.boolean "--status", "show status for a url in the cache"
 
           # generic
-          o.boolean '--version', 'show version' do
+          o.boolean "--version", "show version" do
             puts "httpdisk #{HTTPDisk::VERSION}"
             exit
           end
-          o.on '--help', 'show this help' do
+          o.on "--help", "show this help" do
             puts o
             exit
           end
         end
 
-        raise Slop::Error, '' if args.empty?
-        raise Slop::Error, 'no URL specified' if slop.args.empty?
-        raise Slop::Error, 'more than one URL specified' if slop.args.length > 1
+        raise Slop::Error, "" if args.empty?
+        raise Slop::Error, "no URL specified" if slop.args.empty?
+        raise Slop::Error, "more than one URL specified" if slop.args.length > 1
 
         slop.to_h.tap do
           _1[:url] = slop.args.first

data/lib/httpdisk/cli/main.rb

@@ -1,6 +1,6 @@
-require 'faraday-cookie_jar'
-require 'faraday_middleware'
-require 'ostruct'
+require "faraday-cookie_jar"
+require "faraday/follow_redirects"
+require "ostruct"
 
 module HTTPDisk
   module Cli
@@ -31,7 +31,7 @@ module HTTPDisk
 
         # output
         if options[:output]
-          File.open(options[:output], 'w') { output(response, _1) }
+          File.open(options[:output], "w") { output(response, _1) }
         else
           output(response, $stdout)
         end
@@ -103,9 +103,9 @@ module HTTPDisk
         method = if options[:request]
           options[:request]
         elsif options[:data]
-          'post'
+          "post"
         end
-        method ||= 'get'
+        method ||= "get"
         method = method.downcase.to_sym
 
         if !Faraday::Connection::METHODS.include?(method)
@@ -119,9 +119,9 @@ module HTTPDisk
       def request_url
         url = options[:url]
         # recover from missing http:
-        if url !~ %r{^https?://}i
-          if url =~ %r{^\w+://}
-            raise CliError, 'only http/https supported'
+        if !%r{^https?://}i.match?(url)
+          if %r{^\w+://}.match?(url)
+            raise CliError, "only http/https supported"
           end
 
           url = "http://#{url}"
@@ -140,11 +140,11 @@ module HTTPDisk
       def request_headers
         {}.tap do |headers|
           if options[:user_agent]
-            headers['User-Agent'] = options[:user_agent]
+            headers["User-Agent"] = options[:user_agent]
           end
 
           options[:header].each do |header|
-            key, value = header.split(': ', 2)
+            key, value = header.split(": ", 2)
             if !key || !value || key.empty? || value.empty?
               raise CliError, "invalid --header #{header.inspect}"
             end

data/lib/httpdisk/client.rb

@@ -1,6 +1,6 @@
-require 'content-type'
-require 'faraday'
-require 'logger'
+require "content-type"
+require "faraday"
+require "logger"
 
 module HTTPDisk
   # Middleware and main entry point.
@@ -9,26 +9,28 @@ module HTTPDisk
 
     def initialize(app, options = {})
       options = Sloptions.parse(options) do
-        _1.string :dir, default: File.join(ENV['HOME'], 'httpdisk')
+        _1.string :dir, default: File.join(ENV["HOME"], "httpdisk")
+        _1.boolean :compress, default: true
         _1.integer :expires
         _1.boolean :force
         _1.boolean :force_errors
         _1.array :ignore_params, default: []
+        _1.on :key_transform, type: Proc
         _1.on :logger, type: [:boolean, Logger]
         _1.boolean :utf8
       end
 
-      super(app, options)
+      super
       @cache = Cache.new(options)
     end
 
     def call(env)
-      cache_key = CacheKey.new(env, ignore_params: ignore_params)
+      cache_key = build_cache_key(env)
       logger&.info("#{env.method.upcase} #{env.url} (#{cache.status(cache_key)})")
       env[:httpdisk_diskpath] = cache.diskpath(cache_key)
 
       # check cache, fallback to network
-      if response = read(cache_key, env)
+      if (response = read(cache_key, env))
         response.env[:httpdisk] = true
       else
         response = perform(env)
@@ -42,7 +44,7 @@ module HTTPDisk
 
     # Returns cache status for this request
     def status(env)
-      cache_key = CacheKey.new(env)
+      cache_key = build_cache_key(env)
       {
         url: env.url.to_s,
         status: cache.status(cache_key).to_s,
@@ -90,7 +92,7 @@ module HTTPDisk
     def stuff_999_response(env, err)
       env.tap do
         _1.reason_phrase = "#{err.class} #{err.message}"
-        _1.response_body = ''
+        _1.response_body = ""
         _1.response_headers = Faraday::Utils::Headers.new
         _1.status = HTTPDisk::ERROR_STATUS
       end
@@ -110,11 +112,11 @@ module HTTPDisk
     # network. Not all adapters honor Content-Type (including the default
     # adapter).
     def encode_body(response)
-      body = response.body || ''
+      body = response.body || ""
 
       # parse Content-Type
       begin
-        content_type = response['Content-Type'] && ContentType.parse(response['Content-Type'])
+        content_type = response["Content-Type"] && ContentType.parse(response["Content-Type"])
       rescue Parslet::ParseFailed
         # unparsable
       end
@@ -122,15 +124,14 @@ module HTTPDisk
       # look at charset and set body encoding if necessary
       encoding = encoding_for(content_type)
       if body.encoding != encoding
-        body = body.dup if body.frozen?
-        body.force_encoding(encoding)
+        body = body.dup.force_encoding(encoding)
       end
 
       # if :utf8, force body to UTF-8
       if options[:utf8] && content_type && response_text?(content_type)
         body = body.dup if body.frozen?
         begin
-          body.encode!('UTF-8', invalid: :replace, undef: :replace, replace: '?')
+          body.encode!("UTF-8", invalid: :replace, undef: :replace, replace: "?")
         rescue Encoding::ConverterNotFoundError
           # rare, can't do anything here
           body = "httpdisk could not convert from #{body.encoding.name} to UTF-8"
@@ -152,7 +153,7 @@ module HTTPDisk
     end
 
     def response_text?(content_type)
-      content_type.type == 'text' || content_type.mime_type == 'application/json'
+      content_type.type == "text" || content_type.mime_type == "application/json"
     end
 
     #
@@ -163,6 +164,12 @@ module HTTPDisk
       @ignore_params ||= options[:ignore_params].map { CGI.escape(_1.to_s) }.to_set
     end
 
+    def build_cache_key(env)
+      CacheKey.new(env, ignore_params:).tap do
+        options[:key_transform]&.call(_1)
+      end
+    end
+
     def logger
       return if !options[:logger]
 

data/lib/httpdisk/grep/args.rb

@@ -1,6 +1,6 @@
 # manually load dependencies here since this is loaded standalone by bin
-require 'httpdisk/version'
-require 'slop'
+require "httpdisk/version"
+require "slop"
 
 module HTTPDisk
   module Grep
@@ -8,22 +8,22 @@ module HTTPDisk
       # Slop parsing. This is broken out so we can run without require 'httpdisk'.
       def self.slop(args)
         slop = Slop.parse(args) do |o|
-          o.banner = 'httpdisk-grep [options] pattern [path ...]'
-          o.boolean '-c', '--count', 'suppress normal output and show count'
-          o.boolean '-h', '--head', 'show req headers before each match'
-          o.boolean '-s', '--silent', 'do not print anything to stdout'
-          o.boolean '--version', 'show version' do
+          o.banner = "httpdisk-grep [options] pattern [path ...]"
+          o.boolean "-c", "--count", "suppress normal output and show count"
+          o.boolean "-h", "--head", "show req headers before each match"
+          o.boolean "-s", "--silent", "do not print anything to stdout"
+          o.boolean "--version", "show version" do
             puts "httpdisk-grep #{HTTPDisk::VERSION}"
             exit
           end
-          o.on '--help', 'show this help' do
+          o.on "--help", "show this help" do
             puts o
             exit
           end
         end
 
-        raise Slop::Error, '' if args.empty?
-        raise Slop::Error, 'no PATTERN specified' if slop.args.empty?
+        raise Slop::Error, "" if args.empty?
+        raise Slop::Error, "no PATTERN specified" if slop.args.empty?
 
         slop.to_h.tap do
           _1[:pattern] = slop.args.shift