httpdisk 0.1.0 → 0.5.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0c47ec4fda68047f57e8348746cf7e08467151a4b3d193cce16652980b5b8a47
-  data.tar.gz: 158e71dc98ba8a954eb3e744140a6e1f468de06e54a482690e7347d4b8750153
+  metadata.gz: 994e5e02b7c4cdbfbfcef40099558c8fa86a97538da74065a76321c8ee25e129
+  data.tar.gz: 97ae05c7a08e2c9b3bdccc01a9679d63ab35ce4fabc6307087fbfa7946fbecef
 SHA512:
-  metadata.gz: 4e261b58f3c1246dec8ab9ab5732c06c595af86b27c14866045fb9992983b1f8c89d93f788272c2331ded4f5424e424ebdf5ae0bdc886e95537e5696ad2ea04b
-  data.tar.gz: 9947c8fd27c4f7dbb98b9eb9c3aca086d7c0041ec51353b8163371d89b98c745592f764cfb05074daff0c8e9a6e9500863bf16d5f200b54e92e61ccc9006eaca
+  metadata.gz: '09d938bd3b32d743aab858883c94a760a27c40762a98d792b4135df29625995582aac5571aca0b510d0e85adc8b99436f2ede87196581544ae5759de35df9cfb'
+  data.tar.gz: 18c4f281814896c8d8a9e8d513416538770263b4b17dea5fbc5a5e5612af9f3b0f39d86b5e54919df6466a2e18e73f52e125ad9d167b1912c4f3acc2cb8b8a27

data/.rubocop.yml

@@ -0,0 +1,28 @@
+AllCops:
+  NewCops: enable
+  SuggestExtensions: false
+
+# this is buggy in 2.7.0
+Style/HashTransformValues: { Enabled: false }
+
+# minimal personal preference
+Layout/CaseIndentation: { Enabled: false }
+Layout/EndAlignment: { EnforcedStyleAlignWith: variable }
+Lint/AssignmentInCondition: { Enabled: false }
+Lint/NonLocalExitFromIterator: { Enabled: false }
+Metrics: { Enabled: false }
+Naming/MethodParameterName: { Enabled: false }
+Naming/VariableNumber: { Enabled: false }
+Style/Documentation: { Enabled: false }
+Style/DoubleNegation: { Enabled: false }
+Style/EmptyCaseCondition: { Enabled: false }
+Style/FrozenStringLiteralComment: { Enabled: false }
+Style/GuardClause: { Enabled: false }
+Style/IfUnlessModifier: { Enabled: false }
+Style/NegatedIf: { Enabled: false }
+Style/NumericPredicate: { Enabled: false }
+Style/ParallelAssignment: { Enabled: false }
+Style/SoleNestedConditional: { Enabled: false }
+Style/StderrPuts: { Enabled: false }
+Style/TrailingCommaInArrayLiteral: { EnforcedStyleForMultiline: consistent_comma }
+Style/TrailingCommaInHashLiteral: { EnforcedStyleForMultiline: consistent_comma }

data/Gemfile

@@ -6,5 +6,6 @@ group :development, :test do
   gem 'mocha'
   gem 'pry'
   gem 'rake'
+  gem 'rubocop', '~> 1.13.0'
   gem 'webmock'
 end

data/Gemfile.lock

@@ -1,7 +1,8 @@
 PATH
   remote: .
   specs:
-    httpdisk (0.1.0)
+    httpdisk (0.5.1)
+      content-type (~> 0.0)
       faraday (~> 1.4)
       faraday-cookie_jar (~> 0.0)
       faraday_middleware (~> 1.0)
@@ -12,43 +13,73 @@ GEM
   specs:
     addressable (2.7.0)
       public_suffix (>= 2.0.2, < 5.0)
+    ast (2.4.2)
     coderay (1.1.3)
+    content-type (0.0.1)
+      parslet (~> 1.5)
     crack (0.4.5)
       rexml
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
-    faraday (1.4.1)
+    faraday (1.5.0)
+      faraday-em_http (~> 1.0)
+      faraday-em_synchrony (~> 1.0)
       faraday-excon (~> 1.1)
+      faraday-httpclient (~> 1.0.1)
       faraday-net_http (~> 1.0)
       faraday-net_http_persistent (~> 1.1)
+      faraday-patron (~> 1.0)
       multipart-post (>= 1.2, < 3)
       ruby2_keywords (>= 0.0.4)
     faraday-cookie_jar (0.0.7)
       faraday (>= 0.8.0)
       http-cookie (~> 1.0.0)
+    faraday-em_http (1.0.0)
+    faraday-em_synchrony (1.0.0)
     faraday-excon (1.1.0)
+    faraday-httpclient (1.0.1)
     faraday-net_http (1.0.1)
     faraday-net_http_persistent (1.1.0)
+    faraday-patron (1.0.0)
     faraday_middleware (1.0.0)
       faraday (~> 1.0)
     hashdiff (1.0.1)
-    http-cookie (1.0.3)
+    http-cookie (1.0.4)
       domain_name (~> 0.5)
     method_source (1.0.0)
     minitest (5.14.4)
     mocha (1.11.2)
     multipart-post (2.1.1)
+    parallel (1.20.1)
+    parser (3.0.1.1)
+      ast (~> 2.4.1)
+    parslet (1.8.2)
     pry (0.13.1)
       coderay (~> 1.1)
       method_source (~> 1.0)
     public_suffix (4.0.6)
+    rainbow (3.0.0)
     rake (13.0.3)
+    regexp_parser (2.1.1)
     rexml (3.2.5)
+    rubocop (1.13.0)
+      parallel (~> 1.10)
+      parser (>= 3.0.0.0)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml
+      rubocop-ast (>= 1.2.0, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 3.0)
+    rubocop-ast (1.5.0)
+      parser (>= 3.0.1.1)
+    ruby-progressbar (1.11.0)
     ruby2_keywords (0.0.4)
-    slop (4.8.2)
+    slop (4.9.1)
     unf (0.1.4)
       unf_ext
     unf_ext (0.0.7.7)
+    unicode-display_width (2.0.0)
     webmock (3.12.2)
       addressable (>= 2.3.6)
       crack (>= 0.3.2)
@@ -63,6 +94,7 @@ DEPENDENCIES
   mocha
   pry
   rake
+  rubocop (~> 1.13.0)
   webmock
 
 BUNDLED WITH

data/README.md

@@ -119,20 +119,27 @@ httpdisk caches all responses. POST responses are cached, along with 500 respons
 
 In general, if you make a request it will be cached regardless of the outcome.
 
+## String Encoding
+
+httpdisk will honor the `Content-Type` from responses. Unfortunately, it is entirely possible to get invalid bodies if the `Content-Type` doesn't match the bytes. This is a major bummer, so httpdisk provides a `utf8:` option that forces text response bodies to UTF-8.
+
 ## Configuration
 
 httpdisk supports a few options:
 
 - `dir:` location for disk cache, defaults to `~/httpdisk`
-- `expires_in:` when to expire cached requests, default is nil (never expire)
+- `expires:` when to expire cached requests, default is nil (never expire)
 - `force:` don't read anything from cache (but still write)
 - `force_errors:` don't read errors from cache (but still write)
+- `ignore_params:` array of query params to ignore when calculating cache_key
+- `logger`: log requests to stderr, or pass your own logger
+- `utf8`: if true, force text response bodies to valid UTF-8
 
 Pass these in when setting up Faraday:
 
 ```ruby
 faraday = Faraday.new do
-  _1.use :httpdisk, expires_in: 7*24*60*60, force: true
+  _1.use :httpdisk, expires: 7*24*60*60, force: true
 end
 ```
 
@@ -160,10 +167,13 @@ Specific to httpdisk:
     --force           don't read anything from cache (but still write)
     --force-errors    don't read errors from cache (but still write)
     --status          show status for a url in the cache
-    --version         show version
-    --help            show this help
 ```
 
+## Goodies: httpdisk-grep
+
+The `httpdisk-grep` command makes it easy to search your cache directory.
+It can be challenging to use grep/ripgrep because cache files are compressed and JSON bodies often lack newlines. httpdisk-grep is the right tool for the job. See `httpdisk-grep --help`.
+
 ## Limitations & Gotchas
 
 - Transient errors are cached. This is appropriate for many uses cases (like crawling) but can be confusing. Use `httpdisk --status` to debug.
@@ -174,6 +184,28 @@ Specific to httpdisk:
 
 ## Changelog
 
+#### 0.5
+
+- honor Content-Type
+- added `:utf8` option to force text-like response bodies to UTF-8
+
+#### 0.4
+
+- added httpdisk-grep for searching cache files
+- added HTTPDisk::Cache#delete
+- rename `:expires_in` to `:expires`
+
+#### 0.3
+
+- added :ignore_params, for ignoring query params when generating cache keys
+- HTTP 40x & 50x responses return :error status (and respond to `force_error`)
+
+#### 0.2 - May 2020
+
+- added `response.env[:httpdisk]`, which will be true if the response came from the cache
+- added `:logger` option
+- rake rubocop
+
 #### 0.1 - April 2020
 
 - Original release

data/Rakefile

@@ -10,12 +10,15 @@ spec = Gem::Specification.load('httpdisk.gemspec')
 #
 
 # test (default)
-Rake::TestTask.new { _1.libs << 'test' }
+Rake::TestTask.new do
+  _1.libs << 'test'
+  _1.warning = false # https://github.com/lostisland/faraday/issues/1285
+end
 task default: :test
 
-# Watch files, run tests whenever something changes
+# Watch rb files, run tests whenever something changes
 task :watch do
-  system('find . | entr -c rake test')
+  sh "find . -name '*.rb' | entr -c rake"
 end
 
 #
@@ -23,7 +26,15 @@ end
 #
 
 task :pry do
-  system 'pry -I lib -r httpdisk.rb'
+  sh 'pry -I lib -r httpdisk.rb'
+end
+
+#
+# rubocop
+#
+
+task :rubocop do
+  sh 'bundle exec rubocop -A .'
 end
 
 #
@@ -31,17 +42,17 @@ end
 #
 
 task :build do
-  system('gem build --quiet httpdisk.gemspec', exception: true)
+  sh 'gem build --quiet httpdisk.gemspec'
 end
 
 task install: :build do
-  system("gem install --quiet httpdisk-#{spec.version}.gem", exception: true)
+  sh "gem install --quiet httpdisk-#{spec.version}.gem"
 end
 
-task release: :build do
+task release: %i[rubocop test build] do
   raise "looks like git isn't clean" unless `git status --porcelain`.empty?
 
-  system("git tag -a #{spec.version} -m 'Tagging #{spec.version}'", exception: true)
-  system('git push --tags', exception: true)
-  system("gem push httpdisk-#{spec.version}.gem", exception: true)
+  sh "git tag -a #{spec.version} -m 'Tagging #{spec.version}'"
+  sh 'git push --tags'
+  sh "gem push httpdisk-#{spec.version}.gem"
 end

data/bin/httpdisk

@@ -6,20 +6,22 @@
 
 $LOAD_PATH.unshift(File.join(__dir__, '../lib'))
 
-def puts_error(s, exit: false)
-  $stderr.puts "httpdisk: #{s}"
+BIN = File.basename($PROGRAM_NAME)
+
+def puts_error(s)
+  $stderr.puts "#{BIN}: #{s}"
 end
 
 #
 # Load the bare minimum and parse args with slop. We do this separately for speed.
 #
 
-require 'httpdisk/cli_slop'
+require 'httpdisk/cli/args'
 begin
-  slop = HTTPDisk::CliSlop.slop(ARGV)
+  slop = HTTPDisk::Cli::Args.slop(ARGV)
 rescue Slop::Error => e
   puts_error(e) if e.message != ''
-  puts_error("try 'httpdisk --help' for more information")
+  puts_error("try '#{BIN} --help' for more information")
   exit 1
 end
 
@@ -28,11 +30,11 @@ end
 #
 
 require 'httpdisk'
-cli = HTTPDisk::Cli.new(slop)
+main = HTTPDisk::Cli::Main.new(slop)
 begin
-  cli.run
+  main.run
 rescue StandardError => e
-  puts_error(e) if !cli.options[:silent]
+  puts_error(e) if !main.options[:silent]
   if ENV['HTTPDISK_DEBUG']
     $stderr.puts
     $stderr.puts e.backtrace.join("\n")

data/bin/httpdisk-grep

@@ -0,0 +1,46 @@
+#!/usr/bin/env ruby
+
+#
+# Search an HTTPDisk cache, similar to grep.
+#
+
+$LOAD_PATH.unshift(File.join(__dir__, '../lib'))
+
+BIN = File.basename($PROGRAM_NAME)
+
+def puts_error(s)
+  $stderr.puts "#{BIN}: #{s}"
+end
+
+#
+# Load the bare minimum and parse args with slop. We do this separately for speed.
+#
+
+require 'httpdisk/grep/args'
+begin
+  slop = HTTPDisk::Grep::Args.slop(ARGV)
+rescue Slop::Error => e
+  puts_error(e) if e.message != ''
+  puts_error("try '#{BIN} --help' for more information")
+  exit 1
+end
+
+#
+# now load everything and run
+#
+
+require 'httpdisk'
+
+main = HTTPDisk::Grep::Main.new(slop)
+begin
+  success = main.run
+  exit 1 if !success
+rescue StandardError => e
+  puts_error(e)
+  if ENV['HTTPDISK_DEBUG']
+    $stderr.puts
+    $stderr.puts e.class
+    $stderr.puts e.backtrace.join("\n")
+  end
+  exit 2
+end

data/examples.rb

@@ -18,7 +18,6 @@ class Examples
     faraday.get('http://www.google.com', nil, { "User-Agent": 'test-agent' })
     faraday.get('http://www.google.com', { q: 'ruby' })
     faraday.post('http://httpbin.org/post', 'name=hello')
-    exit
 
     3.times { puts }
     response = faraday.get('http://httpbingo.org/get')
@@ -106,7 +105,7 @@ class Examples
     end
 
     3.times { puts }
-    response = faraday.post('http://httpbingo.org/post', { this_is: [ 'json' ] })
+    response = faraday.post('http://httpbingo.org/post', { this_is: ['json'] })
     puts response.env.url
     puts JSON.pretty_generate(response.body)
   end

data/httpdisk.gemspec

@@ -3,7 +3,7 @@ require_relative 'lib/httpdisk/version'
 Gem::Specification.new do |s|
   s.name = 'httpdisk'
   s.version = HTTPDisk::VERSION
-  s.authors = [ 'Adam Doppelt' ]
+  s.authors = ['Adam Doppelt']
   s.email = 'amd@gurge.com'
 
   s.summary = 'httpdisk - disk cache for faraday'
@@ -18,9 +18,10 @@ Gem::Specification.new do |s|
   end
   s.bindir = 'bin'
   s.executables = s.files.grep(%r{^#{s.bindir}/}) { File.basename(_1) }
-  s.require_paths = [ 'lib' ]
+  s.require_paths = ['lib']
 
   # gem dependencies
+  s.add_dependency 'content-type', '~> 0.0'
   s.add_dependency 'faraday', '~> 1.4'
   s.add_dependency 'faraday-cookie_jar', '~> 0.0'
   s.add_dependency 'faraday_middleware', '~> 1.0'

data/lib/httpdisk.rb

@@ -1,12 +1,17 @@
 require 'httpdisk/cache_key'
 require 'httpdisk/cache'
-require 'httpdisk/cli_slop'
-require 'httpdisk/cli'
 require 'httpdisk/client'
 require 'httpdisk/error'
 require 'httpdisk/payload'
+require 'httpdisk/slop_duration'
+require 'httpdisk/sloptions'
 require 'httpdisk/version'
 
-module HTTPDisk
-  ERROR_STATUS = 999
-end
+# cli
+require 'httpdisk/cli/args'
+require 'httpdisk/cli/main'
+
+# grep
+require 'httpdisk/grep/args'
+require 'httpdisk/grep/main'
+require 'httpdisk/grep/printer'

data/lib/httpdisk/cache.rb

@@ -7,23 +7,9 @@ module HTTPDisk
 
     def initialize(options)
       @options = options
-
-      # heavy sanity checking on arguments here
-      if !dir.is_a?(String)
-        raise ArgumentError, "expected :dir to be a string, not #{dir.inspect}"
-      end
-      if expires_in && !expires_in.is_a?(Integer)
-        raise ArgumentError, "expected :expires_in to be an integer, not #{expires_in.inspect}"
-      end
-      %i[force force_errors].each do
-        value = send(_1)
-        if ![ nil, true, false ].include?(value)
-          raise ArgumentError, "expected #{_1} to be a boolean, not #{value.inspect}"
-        end
-      end
     end
 
-    %i[dir expires_in force force_errors].each do |method|
+    %i[dir expires force force_errors].each do |method|
       define_method(method) do
         options[method]
       end
@@ -39,16 +25,35 @@ module HTTPDisk
 
     # Cache status for a cache_key, %i[error force hit miss stale]
     def status(cache_key)
-      payload_or_status = read0(cache_key)
+      payload_or_status = read0(cache_key, peek: true)
       return payload_or_status if payload_or_status.is_a?(Symbol)
-      payload_or_status.error_999? ? :error : :hit
+
+      payload_or_status.error? ? :error : :hit
     end
 
     # Write response to the disk cache
     def write(cache_key, payload)
       path = diskpath(cache_key)
       FileUtils.mkdir_p(File.dirname(path))
-      Zlib::GzipWriter.open(path) { payload.write(_1) }
+
+      # Atomically write gzipped payload. Put our underlying Tempfile into
+      # binmode to avoid accidental newline conversion or string encoding. Not
+      # required for *nix systems, but I've heard rumors it's helpful for
+      # Windows.
+      Tempfile.new(binmode: true).tap do |tmp|
+        Zlib::GzipWriter.new(tmp).tap do |gzip|
+          payload.write(gzip)
+          gzip.close
+        end
+        tmp.close
+        FileUtils.mv(tmp.path, path)
+      end
+    end
+
+    # Delete existing response, if any
+    def delete(cache_key)
+      path = diskpath(cache_key)
+      FileUtils.rm(path) if File.exist?(path)
     end
 
     # Relative path for this cache_key based on the cache key
@@ -59,22 +64,28 @@ module HTTPDisk
     protected
 
     # low level read, returns payload or status
-    def read0(cache_key)
+    def read0(cache_key, peek: false)
       path = diskpath(cache_key)
 
       return :miss if !File.exist?(path)
       return :stale if expired?(path)
       return :force if force?
 
-      payload = Zlib::GzipReader.open(path) { Payload.read(_1) }
-      return :force if force_errors? && payload.error_999?
+      begin
+        payload = Zlib::GzipReader.open(path, encoding: 'ASCII-8BIT') do
+          Payload.read(_1, peek: peek)
+        end
+      rescue StandardError => e
+        raise "#{path}: #{e}"
+      end
+      return :force if force_errors? && payload.error?
 
       payload
     end
 
     # Is this path expired?
     def expired?(path)
-      expires_in && File.stat(path).mtime < Time.now - expires_in
+      expires && File.stat(path).mtime < Time.now - expires
     end
   end
 end