httpdisk 0.4.0 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7b1e66859e069e390390dd87fac3753ee43501613884bfca2d7f40f70a224274
-  data.tar.gz: 8de194055b6fc7ac858305eb739dc25494b139413d36decbbbaeaa00ce8f1da0
+  metadata.gz: fece53d7b5f7694a0f533797d119495ad2e8bc79667560f458aa23c0fb6eebf5
+  data.tar.gz: 2cb035d54e62cb9195959797fbfb712aabab9fc78db2ddfaf59e714c1da66ba5
 SHA512:
-  metadata.gz: 4594b3eb07cd683a901883484ce88ed3be63f4a65a1c2d9c1a2eb80c9791606ceaaf7287ca74ae35d5fa5015e3218133a7e1199111233841f58c67696edc3f0e
-  data.tar.gz: 5d9f0dd6f3d8407e4b5133ec407d4aa56b6c442d42679c7715f9519b60a791ebb212e11c3c8474ad92bf4dd9e2d5d8daaded2eadcff4b210a6fa5b1e0245677b
+  metadata.gz: '09ace3872dfbe2764e42eff568576f44ee5581c750dfbb5a907bd26a6581e047d1529c8928e2f430d804e3d2669eac6c8ca3307c47e34bab8674e810550a4aa8'
+  data.tar.gz: 70ea93666f7dff7283fc1b6ef3860e52a2309828874a157e7fdd29cc48cef67f814907e2be18c87c9404c33b65b0fed2f9156b31cf747d3bb9398c60e8f11335

data/.rubocop.yml

@@ -17,10 +17,12 @@ Style/Documentation: { Enabled: false }
 Style/DoubleNegation: { Enabled: false }
 Style/EmptyCaseCondition: { Enabled: false }
 Style/FrozenStringLiteralComment: { Enabled: false }
+Style/GuardClause: { Enabled: false }
 Style/IfUnlessModifier: { Enabled: false }
 Style/NegatedIf: { Enabled: false }
 Style/NumericPredicate: { Enabled: false }
 Style/ParallelAssignment: { Enabled: false }
+Style/SoleNestedConditional: { Enabled: false }
 Style/StderrPuts: { Enabled: false }
 Style/TrailingCommaInArrayLiteral: { EnforcedStyleForMultiline: consistent_comma }
 Style/TrailingCommaInHashLiteral: { EnforcedStyleForMultiline: consistent_comma }

data/Gemfile.lock

@@ -1,7 +1,8 @@
 PATH
   remote: .
   specs:
-    httpdisk (0.4.0)
+    httpdisk (0.5.0)
+      content-type (~> 0.0)
       faraday (~> 1.4)
       faraday-cookie_jar (~> 0.0)
       faraday_middleware (~> 1.0)
@@ -14,11 +15,13 @@ GEM
       public_suffix (>= 2.0.2, < 5.0)
     ast (2.4.2)
     coderay (1.1.3)
+    content-type (0.0.1)
+      parslet (~> 1.5)
     crack (0.4.5)
       rexml
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
-    faraday (1.4.2)
+    faraday (1.4.3)
       faraday-em_http (~> 1.0)
       faraday-em_synchrony (~> 1.0)
       faraday-excon (~> 1.1)
@@ -37,7 +40,7 @@ GEM
     faraday_middleware (1.0.0)
       faraday (~> 1.0)
     hashdiff (1.0.1)
-    http-cookie (1.0.3)
+    http-cookie (1.0.4)
       domain_name (~> 0.5)
     method_source (1.0.0)
     minitest (5.14.4)
@@ -46,6 +49,7 @@ GEM
     parallel (1.20.1)
     parser (3.0.1.1)
       ast (~> 2.4.1)
+    parslet (1.8.2)
     pry (0.13.1)
       coderay (~> 1.1)
       method_source (~> 1.0)
@@ -67,7 +71,7 @@ GEM
       parser (>= 3.0.1.1)
     ruby-progressbar (1.11.0)
     ruby2_keywords (0.0.4)
-    slop (4.9.0)
+    slop (4.9.1)
     unf (0.1.4)
       unf_ext
     unf_ext (0.0.7.7)

data/README.md

@@ -65,7 +65,6 @@ faraday = Faraday.new do
   _1.request :url_encoded # auto-encode form bodies
   _1.response :json # auto-decode JSON responses
   _1.response :follow_redirects # follow redirects (should be above httpdisk)
-  _1.response :encoding # set Ruby string encoding based on Content-Type (should be above httpdisk)
   _1.use :httpdisk
   _1.request :retry # retry failed responses (should be below httpdisk)
 end
@@ -120,6 +119,10 @@ httpdisk caches all responses. POST responses are cached, along with 500 respons
 
 In general, if you make a request it will be cached regardless of the outcome.
 
+## String Encoding
+
+httpdisk will honor the `Content-Type` from responses. Unfortunately, it is entirely possible to get invalid bodies if the `Content-Type` doesn't match the bytes. This is a major bummer, so httpdisk provides a `utf8:` option that forces text response bodies to UTF-8.
+
 ## Configuration
 
 httpdisk supports a few options:
@@ -130,6 +133,7 @@ httpdisk supports a few options:
 - `force_errors:` don't read errors from cache (but still write)
 - `ignore_params:` array of query params to ignore when calculating cache_key
 - `logger`: log requests to stderr, or pass your own logger
+- `utf8`: if true, force text response bodies to valid UTF-8
 
 Pass these in when setting up Faraday:
 
@@ -180,6 +184,11 @@ It can be challenging to use grep/ripgrep because cache files are compressed and
 
 ## Changelog
 
+#### 0.5
+
+- honor Content-Type
+- added `:utf8` option to force text-like response bodies to UTF-8
+
 #### 0.4
 
 - added httpdisk-grep for searching cache files

data/httpdisk.gemspec

@@ -21,6 +21,7 @@ Gem::Specification.new do |s|
   s.require_paths = ['lib']
 
   # gem dependencies
+  s.add_dependency 'content-type', '~> 0.0'
   s.add_dependency 'faraday', '~> 1.4'
   s.add_dependency 'faraday-cookie_jar', '~> 0.0'
   s.add_dependency 'faraday_middleware', '~> 1.0'

data/lib/httpdisk/cache.rb

@@ -35,7 +35,19 @@ module HTTPDisk
     def write(cache_key, payload)
       path = diskpath(cache_key)
       FileUtils.mkdir_p(File.dirname(path))
-      Zlib::GzipWriter.open(path) { payload.write(_1) }
+
+      # Atomically write gzipped payload. Put our underlying Tempfile into
+      # binmode to avoid accidental newline conversion or string encoding. Not
+      # required for *nix systems, but I've heard rumors it's helpful for
+      # Windows.
+      Tempfile.new(binmode: true).tap do |tmp|
+        Zlib::GzipWriter.new(tmp).tap do |gzip|
+          payload.write(gzip)
+          gzip.close
+        end
+        tmp.close
+        FileUtils.mv(tmp.path, path)
+      end
     end
 
     # Delete existing response, if any
@@ -59,7 +71,13 @@ module HTTPDisk
       return :stale if expired?(path)
       return :force if force?
 
-      payload = Zlib::GzipReader.open(path) { Payload.read(_1, peek: peek) }
+      begin
+        payload = Zlib::GzipReader.open(path, encoding: 'ASCII-8BIT') do
+          Payload.read(_1, peek: peek)
+        end
+      rescue StandardError => e
+        raise "#{path}: #{e}"
+      end
       return :force if force_errors? && payload.error?
 
       payload

data/lib/httpdisk/cache_key.rb

@@ -10,8 +10,8 @@ module HTTPDisk
       @env, @ignore_params = env, ignore_params
 
       # sanity checks
-      raise 'http/https required' if env.url.scheme !~ /^https?$/
-      raise 'hostname required' if !env.url.host
+      raise InvalidUrl, "http/https required #{env.url.inspect}" if env.url.scheme !~ /^https?$/
+      raise InvalidUrl, "hostname required #{env.url.inspect}" if !env.url.host
     end
 
     def url

data/lib/httpdisk/cli/main.rb

@@ -73,7 +73,8 @@ module HTTPDisk
           _1.method = request_method
           _1.request_body = request_body
           _1.request_headers = request_headers
-          _1.url = request_url
+          # Run the url through Faraday to make sure we see the same stuff as middleware.
+          _1.url = Faraday.new.build_url(request_url)
         end
 
         # now print status
@@ -159,7 +160,9 @@ module HTTPDisk
 
       # Options to HTTPDisk::Client
       def client_options
-        options.slice(:dir, :expires, :force, :force_errors)
+        client_options = options.slice(:dir, :expires, :force, :force_errors)
+        client_options[:utf8] = true
+        client_options
       end
     end
   end

data/lib/httpdisk/client.rb

@@ -1,3 +1,4 @@
+require 'content-type'
 require 'faraday'
 require 'logger'
 
@@ -14,6 +15,7 @@ module HTTPDisk
         _1.boolean :force_errors
         _1.array :ignore_params, default: []
         _1.on :logger, type: [:boolean, Logger]
+        _1.boolean :utf8
       end
 
       super(app, options)
@@ -23,17 +25,19 @@ module HTTPDisk
     def call(env)
       cache_key = CacheKey.new(env, ignore_params: ignore_params)
       logger&.info("#{env.method.upcase} #{env.url} (#{cache.status(cache_key)})")
+      env[:httpdisk_diskpath] = cache.diskpath(cache_key)
 
-      if cached_response = read(cache_key, env)
-        cached_response.env[:httpdisk] = true
-        return cached_response
-      end
-
-      # miss
-      perform(env).tap do |response|
+      # check cache, fallback to network
+      if response = read(cache_key, env)
+        response.env[:httpdisk] = true
+      else
+        response = perform(env)
         response.env[:httpdisk] = false
         write(cache_key, env, response)
       end
+
+      encode_body(response)
+      response
     end
 
     # Returns cache status for this request
@@ -101,6 +105,54 @@ module HTTPDisk
       err.to_s =~ /#{proxy.host}.*#{proxy.port}/
     end
 
+    # Set string encoding for response body. The cache always returns
+    # ASCII-8BIT, but we have no idea what the encoding will be from the
+    # network. Not all adapters honor Content-Type (including the default
+    # adapter).
+    def encode_body(response)
+      body = response.body || ''
+
+      # parse Content-Type
+      begin
+        content_type = response['Content-Type'] && ContentType.parse(response['Content-Type'])
+      rescue Parslet::ParseFailed
+        # unparsable
+      end
+
+      # look at charset and set body encoding if necessary
+      encoding = encoding_for(content_type)
+      if body.encoding != encoding
+        body = body.dup if body.frozen?
+        body.force_encoding(encoding)
+      end
+
+      # if :utf8, force body to UTF-8
+      if options[:utf8] && content_type && response_text?(content_type)
+        body = body.dup if body.frozen?
+        begin
+          body.encode!('UTF-8', invalid: :replace, undef: :replace, replace: '?')
+        rescue Encoding::ConverterNotFoundError
+          # rare, can't do anything here
+          body = "httpdisk could not convert from #{body.encoding.name} to UTF-8"
+        end
+      end
+
+      response.env[:body] = body
+    end
+
+    def encoding_for(content_type)
+      begin
+        return Encoding.find(content_type.charset) if content_type
+      rescue ArgumentError
+        # unknown charset
+      end
+      Encoding::ASCII_8BIT
+    end
+
+    def response_text?(content_type)
+      content_type.type == 'text' || content_type.mime_type == 'application/json'
+    end
+
     #
     # options
     #

data/lib/httpdisk/error.rb

@@ -2,4 +2,6 @@ module HTTPDisk
   ERROR_STATUS = 999
 
   class CliError < StandardError; end
+
+  class InvalidUrl < StandardError; end
 end

data/lib/httpdisk/grep/main.rb

@@ -29,7 +29,9 @@ module HTTPDisk
 
       def run_one(path)
         # read payload & body
-        payload = Zlib::GzipReader.open(path) { Payload.read(_1) }
+        payload = Zlib::GzipReader.open(path, encoding: 'ASCII-8BIT') do
+          Payload.read(_1)
+        end
         body = prepare_body(payload)
 
         # collect all_matches

data/lib/httpdisk/version.rb

@@ -1,3 +1,3 @@
 module HTTPDisk
-  VERSION = '0.4.0'.freeze
+  VERSION = '0.5.0'.freeze
 end

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: httpdisk
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.5.0
 platform: ruby
 authors:
 - Adam Doppelt
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-06-05 00:00:00.000000000 Z
+date: 2021-06-30 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: content-type
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.0'
 - !ruby/object:Gem::Dependency
   name: faraday
   requirement: !ruby/object:Gem::Requirement