httpdisk 0.2.0 → 0.5.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d311812fa4d8d034c9eda6b9b18df36e21eb14ebc9d632cd1fe071268ca77786
-  data.tar.gz: 90e891451c1805d6d8ba5ad268bc2682528954a2a8acd37546f69ab43d493311
+  metadata.gz: 534d055da653551f2dbf23db1c323c6858fcaa0bab09354491c175bffadf6d54
+  data.tar.gz: 825a6779fec3c5ce2828363e9022da231017ae2d13aa8fa88acff9692d8cd7fe
 SHA512:
-  metadata.gz: c61e162e26d8a7b86fe00165e095d72f99415e8e5a4245513758731a009186999d5cf171db45c8b83115dae8c376e8391aa0bfbd00d3a2fd086c0451f77269ad
-  data.tar.gz: 1ad5bdb0a51f2b84822a6164379a724a7b03338b30f9c427b956bbe4263b0326720aea3bcb02b1fa37110a8aa5adc24dc8f3f257733b11667df913b863da0d52
+  metadata.gz: 750eaaca6bba23b1a8af351d916c79e470928c23cb3f6517662d6e99a7278c963cfe9dbbde564282d1367dd0f6bb34c166066b2babdb29d1c29f398fe9498245
+  data.tar.gz: 951f2f668018872bd4a970be506e4fc4c9dce302b4fa03e310eb0269dba428216bdd23a4da1a6fa86ad75f2e75b5d3fbe9988e1018cd328c6bbf6eb790d3406c

data/.rubocop.yml

@@ -2,18 +2,27 @@ AllCops:
   NewCops: enable
   SuggestExtensions: false
 
+# this is buggy in 2.7.0
+Style/HashTransformValues: { Enabled: false }
+
 # minimal personal preference
 Layout/CaseIndentation: { Enabled: false }
 Layout/EndAlignment: { EnforcedStyleAlignWith: variable }
 Lint/AssignmentInCondition: { Enabled: false }
+Lint/NonLocalExitFromIterator: { Enabled: false }
 Metrics: { Enabled: false }
 Naming/MethodParameterName: { Enabled: false }
 Naming/VariableNumber: { Enabled: false }
 Style/Documentation: { Enabled: false }
+Style/DoubleNegation: { Enabled: false }
+Style/EmptyCaseCondition: { Enabled: false }
 Style/FrozenStringLiteralComment: { Enabled: false }
+Style/GuardClause: { Enabled: false }
 Style/IfUnlessModifier: { Enabled: false }
 Style/NegatedIf: { Enabled: false }
+Style/NumericPredicate: { Enabled: false }
 Style/ParallelAssignment: { Enabled: false }
+Style/SoleNestedConditional: { Enabled: false }
 Style/StderrPuts: { Enabled: false }
 Style/TrailingCommaInArrayLiteral: { EnforcedStyleForMultiline: consistent_comma }
 Style/TrailingCommaInHashLiteral: { EnforcedStyleForMultiline: consistent_comma }

data/Gemfile.lock

@@ -1,7 +1,8 @@
 PATH
   remote: .
   specs:
-    httpdisk (0.2.0)
+    httpdisk (0.5.2)
+      content-type (~> 0.0)
       faraday (~> 1.4)
       faraday-cookie_jar (~> 0.0)
       faraday_middleware (~> 1.0)
@@ -14,26 +15,36 @@ GEM
       public_suffix (>= 2.0.2, < 5.0)
     ast (2.4.2)
     coderay (1.1.3)
+    content-type (0.0.1)
+      parslet (~> 1.5)
     crack (0.4.5)
       rexml
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
-    faraday (1.4.1)
+    faraday (1.5.0)
+      faraday-em_http (~> 1.0)
+      faraday-em_synchrony (~> 1.0)
       faraday-excon (~> 1.1)
+      faraday-httpclient (~> 1.0.1)
       faraday-net_http (~> 1.0)
       faraday-net_http_persistent (~> 1.1)
+      faraday-patron (~> 1.0)
       multipart-post (>= 1.2, < 3)
       ruby2_keywords (>= 0.0.4)
     faraday-cookie_jar (0.0.7)
       faraday (>= 0.8.0)
       http-cookie (~> 1.0.0)
+    faraday-em_http (1.0.0)
+    faraday-em_synchrony (1.0.0)
     faraday-excon (1.1.0)
+    faraday-httpclient (1.0.1)
     faraday-net_http (1.0.1)
     faraday-net_http_persistent (1.1.0)
+    faraday-patron (1.0.0)
     faraday_middleware (1.0.0)
       faraday (~> 1.0)
     hashdiff (1.0.1)
-    http-cookie (1.0.3)
+    http-cookie (1.0.4)
       domain_name (~> 0.5)
     method_source (1.0.0)
     minitest (5.14.4)
@@ -42,6 +53,7 @@ GEM
     parallel (1.20.1)
     parser (3.0.1.1)
       ast (~> 2.4.1)
+    parslet (1.8.2)
     pry (0.13.1)
       coderay (~> 1.1)
       method_source (~> 1.0)
@@ -63,7 +75,7 @@ GEM
       parser (>= 3.0.1.1)
     ruby-progressbar (1.11.0)
     ruby2_keywords (0.0.4)
-    slop (4.8.2)
+    slop (4.9.1)
     unf (0.1.4)
       unf_ext
     unf_ext (0.0.7.7)

data/README.md

@@ -65,7 +65,6 @@ faraday = Faraday.new do
   _1.request :url_encoded # auto-encode form bodies
   _1.response :json # auto-decode JSON responses
   _1.response :follow_redirects # follow redirects (should be above httpdisk)
-  _1.response :encoding # set Ruby string encoding based on Content-Type (should be above httpdisk)
   _1.use :httpdisk
   _1.request :retry # retry failed responses (should be below httpdisk)
 end
@@ -120,21 +119,27 @@ httpdisk caches all responses. POST responses are cached, along with 500 respons
 
 In general, if you make a request it will be cached regardless of the outcome.
 
+## String Encoding
+
+httpdisk will honor the `Content-Type` from responses. Unfortunately, it is entirely possible to get invalid bodies if the `Content-Type` doesn't match the bytes. This is a major bummer, so httpdisk provides a `utf8:` option that forces text response bodies to UTF-8.
+
 ## Configuration
 
 httpdisk supports a few options:
 
 - `dir:` location for disk cache, defaults to `~/httpdisk`
-- `expires_in:` when to expire cached requests, default is nil (never expire)
+- `expires:` when to expire cached requests, default is nil (never expire)
 - `force:` don't read anything from cache (but still write)
 - `force_errors:` don't read errors from cache (but still write)
+- `ignore_params:` array of query params to ignore when calculating cache_key
 - `logger`: log requests to stderr, or pass your own logger
+- `utf8`: if true, force text response bodies to valid UTF-8
 
 Pass these in when setting up Faraday:
 
 ```ruby
 faraday = Faraday.new do
-  _1.use :httpdisk, expires_in: 7*24*60*60, force: true
+  _1.use :httpdisk, expires: 7*24*60*60, force: true
 end
 ```
 
@@ -162,10 +167,13 @@ Specific to httpdisk:
     --force           don't read anything from cache (but still write)
     --force-errors    don't read errors from cache (but still write)
     --status          show status for a url in the cache
-    --version         show version
-    --help            show this help
 ```
 
+## Goodies: httpdisk-grep
+
+The `httpdisk-grep` command makes it easy to search your cache directory.
+It can be challenging to use grep/ripgrep because cache files are compressed and JSON bodies often lack newlines. httpdisk-grep is the right tool for the job. See `httpdisk-grep --help`.
+
 ## Limitations & Gotchas
 
 - Transient errors are cached. This is appropriate for many uses cases (like crawling) but can be confusing. Use `httpdisk --status` to debug.
@@ -176,10 +184,28 @@ Specific to httpdisk:
 
 ## Changelog
 
+#### 0.5
+
+- honor Content-Type
+- added `:utf8` option to force text-like response bodies to UTF-8
+
+#### 0.4
+
+- added httpdisk-grep for searching cache files
+- added HTTPDisk::Cache#delete
+- rename `:expires_in` to `:expires`
+
+#### 0.3
+
+- added :ignore_params, for ignoring query params when generating cache keys
+- HTTP 40x & 50x responses return :error status (and respond to `force_error`)
+
 #### 0.2 - May 2020
+
 - added `response.env[:httpdisk]`, which will be true if the response came from the cache
-- `:logger` option
+- added `:logger` option
 - rake rubocop
 
 #### 0.1 - April 2020
+
 - Original release

data/Rakefile

@@ -10,12 +10,15 @@ spec = Gem::Specification.load('httpdisk.gemspec')
 #
 
 # test (default)
-Rake::TestTask.new { _1.libs << 'test' }
+Rake::TestTask.new do
+  _1.libs << 'test'
+  _1.warning = false # https://github.com/lostisland/faraday/issues/1285
+end
 task default: :test
 
-# Watch files, run tests whenever something changes
+# Watch rb files, run tests whenever something changes
 task :watch do
-  system('find . | entr -c rake test')
+  sh "find . -name '*.rb' | entr -c rake"
 end
 
 #
@@ -23,7 +26,7 @@ end
 #
 
 task :pry do
-  system 'pry -I lib -r httpdisk.rb'
+  sh 'pry -I lib -r httpdisk.rb'
 end
 
 #
@@ -31,7 +34,7 @@ end
 #
 
 task :rubocop do
-  system('bundle exec rubocop -A .', exception: true)
+  sh 'bundle exec rubocop -A .'
 end
 
 #
@@ -39,17 +42,17 @@ end
 #
 
 task :build do
-  system('gem build --quiet httpdisk.gemspec', exception: true)
+  sh 'gem build --quiet httpdisk.gemspec'
 end
 
 task install: :build do
-  system("gem install --quiet httpdisk-#{spec.version}.gem", exception: true)
+  sh "gem install --quiet httpdisk-#{spec.version}.gem"
 end
 
-task release: %i[test build] do
+task release: %i[rubocop test build] do
   raise "looks like git isn't clean" unless `git status --porcelain`.empty?
 
-  system("git tag -a #{spec.version} -m 'Tagging #{spec.version}'", exception: true)
-  system('git push --tags', exception: true)
-  system("gem push httpdisk-#{spec.version}.gem", exception: true)
+  sh "git tag -a #{spec.version} -m 'Tagging #{spec.version}'"
+  sh 'git push --tags'
+  sh "gem push httpdisk-#{spec.version}.gem"
 end

data/bin/httpdisk

@@ -6,20 +6,22 @@
 
 $LOAD_PATH.unshift(File.join(__dir__, '../lib'))
 
+BIN = File.basename($PROGRAM_NAME)
+
 def puts_error(s)
-  $stderr.puts "httpdisk: #{s}"
+  $stderr.puts "#{BIN}: #{s}"
 end
 
 #
 # Load the bare minimum and parse args with slop. We do this separately for speed.
 #
 
-require 'httpdisk/cli_slop'
+require 'httpdisk/cli/args'
 begin
-  slop = HTTPDisk::CliSlop.slop(ARGV)
+  slop = HTTPDisk::Cli::Args.slop(ARGV)
 rescue Slop::Error => e
   puts_error(e) if e.message != ''
-  puts_error("try 'httpdisk --help' for more information")
+  puts_error("try '#{BIN} --help' for more information")
   exit 1
 end
 
@@ -28,11 +30,11 @@ end
 #
 
 require 'httpdisk'
-cli = HTTPDisk::Cli.new(slop)
+main = HTTPDisk::Cli::Main.new(slop)
 begin
-  cli.run
+  main.run
 rescue StandardError => e
-  puts_error(e) if !cli.options[:silent]
+  puts_error(e) if !main.options[:silent]
   if ENV['HTTPDISK_DEBUG']
     $stderr.puts
     $stderr.puts e.backtrace.join("\n")

data/bin/httpdisk-grep

@@ -0,0 +1,46 @@
+#!/usr/bin/env ruby
+
+#
+# Search an HTTPDisk cache, similar to grep.
+#
+
+$LOAD_PATH.unshift(File.join(__dir__, '../lib'))
+
+BIN = File.basename($PROGRAM_NAME)
+
+def puts_error(s)
+  $stderr.puts "#{BIN}: #{s}"
+end
+
+#
+# Load the bare minimum and parse args with slop. We do this separately for speed.
+#
+
+require 'httpdisk/grep/args'
+begin
+  slop = HTTPDisk::Grep::Args.slop(ARGV)
+rescue Slop::Error => e
+  puts_error(e) if e.message != ''
+  puts_error("try '#{BIN} --help' for more information")
+  exit 1
+end
+
+#
+# now load everything and run
+#
+
+require 'httpdisk'
+
+main = HTTPDisk::Grep::Main.new(slop)
+begin
+  success = main.run
+  exit 1 if !success
+rescue StandardError => e
+  puts_error(e)
+  if ENV['HTTPDISK_DEBUG']
+    $stderr.puts
+    $stderr.puts e.class
+    $stderr.puts e.backtrace.join("\n")
+  end
+  exit 2
+end

data/httpdisk.gemspec

@@ -21,6 +21,7 @@ Gem::Specification.new do |s|
   s.require_paths = ['lib']
 
   # gem dependencies
+  s.add_dependency 'content-type', '~> 0.0'
   s.add_dependency 'faraday', '~> 1.4'
   s.add_dependency 'faraday-cookie_jar', '~> 0.0'
   s.add_dependency 'faraday_middleware', '~> 1.0'

data/lib/httpdisk.rb

@@ -1,12 +1,17 @@
 require 'httpdisk/cache_key'
 require 'httpdisk/cache'
-require 'httpdisk/cli_slop'
-require 'httpdisk/cli'
 require 'httpdisk/client'
 require 'httpdisk/error'
 require 'httpdisk/payload'
+require 'httpdisk/slop_duration'
+require 'httpdisk/sloptions'
 require 'httpdisk/version'
 
-module HTTPDisk
-  ERROR_STATUS = 999
-end
+# cli
+require 'httpdisk/cli/args'
+require 'httpdisk/cli/main'
+
+# grep
+require 'httpdisk/grep/args'
+require 'httpdisk/grep/main'
+require 'httpdisk/grep/printer'

data/lib/httpdisk/cache.rb

@@ -1,4 +1,5 @@
 require 'fileutils'
+require 'tempfile'
 
 module HTTPDisk
   # Disk cache for cache_keys => response. Files are compressed.
@@ -7,24 +8,9 @@ module HTTPDisk
 
     def initialize(options)
       @options = options
-
-      # heavy sanity checking on arguments here
-      if !dir.is_a?(String)
-        raise ArgumentError, "expected :dir to be a string, not #{dir.inspect}"
-      end
-      if expires_in && !expires_in.is_a?(Integer)
-        raise ArgumentError, "expected :expires_in to be an integer, not #{expires_in.inspect}"
-      end
-
-      %i[force force_errors].each do
-        value = send(_1)
-        if ![nil, true, false].include?(value)
-          raise ArgumentError, "expected #{_1} to be a boolean, not #{value.inspect}"
-        end
-      end
     end
 
-    %i[dir expires_in force force_errors].each do |method|
+    %i[dir expires force force_errors].each do |method|
       define_method(method) do
         options[method]
       end
@@ -43,14 +29,32 @@ module HTTPDisk
       payload_or_status = read0(cache_key, peek: true)
       return payload_or_status if payload_or_status.is_a?(Symbol)
 
-      payload_or_status.error_999? ? :error : :hit
+      payload_or_status.error? ? :error : :hit
     end
 
     # Write response to the disk cache
     def write(cache_key, payload)
       path = diskpath(cache_key)
       FileUtils.mkdir_p(File.dirname(path))
-      Zlib::GzipWriter.open(path) { payload.write(_1) }
+
+      # Atomically write gzipped payload. Put our underlying Tempfile into
+      # binmode to avoid accidental newline conversion or string encoding. Not
+      # required for *nix systems, but I've heard rumors it's helpful for
+      # Windows.
+      Tempfile.new(binmode: true).tap do |tmp|
+        Zlib::GzipWriter.new(tmp).tap do |gzip|
+          payload.write(gzip)
+          gzip.close
+        end
+        tmp.close
+        FileUtils.mv(tmp.path, path)
+      end
+    end
+
+    # Delete existing response, if any
+    def delete(cache_key)
+      path = diskpath(cache_key)
+      FileUtils.rm(path) if File.exist?(path)
     end
 
     # Relative path for this cache_key based on the cache key
@@ -68,15 +72,21 @@ module HTTPDisk
       return :stale if expired?(path)
       return :force if force?
 
-      payload = Zlib::GzipReader.open(path) { Payload.read(_1, peek: peek) }
-      return :force if force_errors? && payload.error_999?
+      begin
+        payload = Zlib::GzipReader.open(path, encoding: 'ASCII-8BIT') do
+          Payload.read(_1, peek: peek)
+        end
+      rescue StandardError => e
+        raise "#{path}: #{e}"
+      end
+      return :force if force_errors? && payload.error?
 
       payload
     end
 
     # Is this path expired?
     def expired?(path)
-      expires_in && File.stat(path).mtime < Time.now - expires_in
+      expires && File.stat(path).mtime < Time.now - expires
     end
   end
 end