httpdisk 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0e3b6af20bf5a6b76a23a6d170328c040cf3f0715e012c392739b70ab6911bee
-  data.tar.gz: 46385301b89bc19f2e8862bcc9bf48b0f1b14fafbd2fa4fc8542b7bf4dcfe930
+  metadata.gz: 7b1e66859e069e390390dd87fac3753ee43501613884bfca2d7f40f70a224274
+  data.tar.gz: 8de194055b6fc7ac858305eb739dc25494b139413d36decbbbaeaa00ce8f1da0
 SHA512:
-  metadata.gz: c4b163648a7adf00ae2ae103ccef899f5a3b958d8e38eeb56fc0d609f0bae73a19eed4fb7d5921c583b4f6d2755bd70d5cf1da748a6d8e858f5195da1683d009
-  data.tar.gz: 3270ce507204023e25451048fd0ddecc713ce6c637abfe795c2f159d619328892393bb8fa73249f08199a30e9db895fb2e2b6c440bfae0dcf07abd277851d050
+  metadata.gz: 4594b3eb07cd683a901883484ce88ed3be63f4a65a1c2d9c1a2eb80c9791606ceaaf7287ca74ae35d5fa5015e3218133a7e1199111233841f58c67696edc3f0e
+  data.tar.gz: 5d9f0dd6f3d8407e4b5133ec407d4aa56b6c442d42679c7715f9519b60a791ebb212e11c3c8474ad92bf4dd9e2d5d8daaded2eadcff4b210a6fa5b1e0245677b

data/.rubocop.yml

@@ -15,9 +15,11 @@ Naming/MethodParameterName: { Enabled: false }
 Naming/VariableNumber: { Enabled: false }
 Style/Documentation: { Enabled: false }
 Style/DoubleNegation: { Enabled: false }
+Style/EmptyCaseCondition: { Enabled: false }
 Style/FrozenStringLiteralComment: { Enabled: false }
 Style/IfUnlessModifier: { Enabled: false }
 Style/NegatedIf: { Enabled: false }
+Style/NumericPredicate: { Enabled: false }
 Style/ParallelAssignment: { Enabled: false }
 Style/StderrPuts: { Enabled: false }
 Style/TrailingCommaInArrayLiteral: { EnforcedStyleForMultiline: consistent_comma }

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    httpdisk (0.3.0)
+    httpdisk (0.4.0)
       faraday (~> 1.4)
       faraday-cookie_jar (~> 0.0)
       faraday_middleware (~> 1.0)
@@ -18,7 +18,9 @@ GEM
       rexml
     domain_name (0.5.20190701)
       unf (>= 0.0.5, < 1.0.0)
-    faraday (1.4.1)
+    faraday (1.4.2)
+      faraday-em_http (~> 1.0)
+      faraday-em_synchrony (~> 1.0)
       faraday-excon (~> 1.1)
       faraday-net_http (~> 1.0)
       faraday-net_http_persistent (~> 1.1)
@@ -27,6 +29,8 @@ GEM
     faraday-cookie_jar (0.0.7)
       faraday (>= 0.8.0)
       http-cookie (~> 1.0.0)
+    faraday-em_http (1.0.0)
+    faraday-em_synchrony (1.0.0)
     faraday-excon (1.1.0)
     faraday-net_http (1.0.1)
     faraday-net_http_persistent (1.1.0)

data/README.md

@@ -125,7 +125,7 @@ In general, if you make a request it will be cached regardless of the outcome.
 httpdisk supports a few options:
 
 - `dir:` location for disk cache, defaults to `~/httpdisk`
-- `expires_in:` when to expire cached requests, default is nil (never expire)
+- `expires:` when to expire cached requests, default is nil (never expire)
 - `force:` don't read anything from cache (but still write)
 - `force_errors:` don't read errors from cache (but still write)
 - `ignore_params:` array of query params to ignore when calculating cache_key
@@ -135,7 +135,7 @@ Pass these in when setting up Faraday:
 
 ```ruby
 faraday = Faraday.new do
-  _1.use :httpdisk, expires_in: 7*24*60*60, force: true
+  _1.use :httpdisk, expires: 7*24*60*60, force: true
 end
 ```
 
@@ -163,10 +163,13 @@ Specific to httpdisk:
     --force           don't read anything from cache (but still write)
     --force-errors    don't read errors from cache (but still write)
     --status          show status for a url in the cache
-    --version         show version
-    --help            show this help
 ```
 
+## Goodies: httpdisk-grep
+
+The `httpdisk-grep` command makes it easy to search your cache directory.
+It can be challenging to use grep/ripgrep because cache files are compressed and JSON bodies often lack newlines. httpdisk-grep is the right tool for the job. See `httpdisk-grep --help`.
+
 ## Limitations & Gotchas
 
 - Transient errors are cached. This is appropriate for many uses cases (like crawling) but can be confusing. Use `httpdisk --status` to debug.
@@ -177,7 +180,13 @@ Specific to httpdisk:
 
 ## Changelog
 
-#### 0.3 (unreleased)
+#### 0.4
+
+- added httpdisk-grep for searching cache files
+- added HTTPDisk::Cache#delete
+- rename `:expires_in` to `:expires`
+
+#### 0.3
 
 - added :ignore_params, for ignoring query params when generating cache keys
 - HTTP 40x & 50x responses return :error status (and respond to `force_error`)

data/Rakefile

@@ -10,13 +10,15 @@ spec = Gem::Specification.load('httpdisk.gemspec')
 #
 
 # test (default)
-Rake::TestTask.new { _1.libs << 'test' }
+Rake::TestTask.new do
+  _1.libs << 'test'
+  _1.warning = false # https://github.com/lostisland/faraday/issues/1285
+end
 task default: :test
 
 # Watch rb files, run tests whenever something changes
 task :watch do
-  # https://superuser.com/a/665208 / https://unix.stackexchange.com/a/42288
-  system("while true; do find . -name '*.rb' | entr -c -d rake; test $? -gt 128 && break; done")
+  sh "find . -name '*.rb' | entr -c rake"
 end
 
 #
@@ -24,7 +26,7 @@ end
 #
 
 task :pry do
-  system 'pry -I lib -r httpdisk.rb'
+  sh 'pry -I lib -r httpdisk.rb'
 end
 
 #

data/bin/httpdisk

@@ -6,20 +6,22 @@
 
 $LOAD_PATH.unshift(File.join(__dir__, '../lib'))
 
+BIN = File.basename($PROGRAM_NAME)
+
 def puts_error(s)
-  $stderr.puts "httpdisk: #{s}"
+  $stderr.puts "#{BIN}: #{s}"
 end
 
 #
 # Load the bare minimum and parse args with slop. We do this separately for speed.
 #
 
-require 'httpdisk/cli_slop'
+require 'httpdisk/cli/args'
 begin
-  slop = HTTPDisk::CliSlop.slop(ARGV)
+  slop = HTTPDisk::Cli::Args.slop(ARGV)
 rescue Slop::Error => e
   puts_error(e) if e.message != ''
-  puts_error("try 'httpdisk --help' for more information")
+  puts_error("try '#{BIN} --help' for more information")
   exit 1
 end
 
@@ -28,11 +30,11 @@ end
 #
 
 require 'httpdisk'
-cli = HTTPDisk::Cli.new(slop)
+main = HTTPDisk::Cli::Main.new(slop)
 begin
-  cli.run
+  main.run
 rescue StandardError => e
-  puts_error(e) if !cli.options[:silent]
+  puts_error(e) if !main.options[:silent]
   if ENV['HTTPDISK_DEBUG']
     $stderr.puts
     $stderr.puts e.backtrace.join("\n")

data/bin/httpdisk-grep

@@ -0,0 +1,46 @@
+#!/usr/bin/env ruby
+
+#
+# Search an HTTPDisk cache, similar to grep.
+#
+
+$LOAD_PATH.unshift(File.join(__dir__, '../lib'))
+
+BIN = File.basename($PROGRAM_NAME)
+
+def puts_error(s)
+  $stderr.puts "#{BIN}: #{s}"
+end
+
+#
+# Load the bare minimum and parse args with slop. We do this separately for speed.
+#
+
+require 'httpdisk/grep/args'
+begin
+  slop = HTTPDisk::Grep::Args.slop(ARGV)
+rescue Slop::Error => e
+  puts_error(e) if e.message != ''
+  puts_error("try '#{BIN} --help' for more information")
+  exit 1
+end
+
+#
+# now load everything and run
+#
+
+require 'httpdisk'
+
+main = HTTPDisk::Grep::Main.new(slop)
+begin
+  success = main.run
+  exit 1 if !success
+rescue StandardError => e
+  puts_error(e)
+  if ENV['HTTPDISK_DEBUG']
+    $stderr.puts
+    $stderr.puts e.class
+    $stderr.puts e.backtrace.join("\n")
+  end
+  exit 2
+end

data/lib/httpdisk.rb

@@ -1,13 +1,17 @@
 require 'httpdisk/cache_key'
 require 'httpdisk/cache'
-require 'httpdisk/cli_slop'
-require 'httpdisk/cli'
 require 'httpdisk/client'
 require 'httpdisk/error'
 require 'httpdisk/payload'
+require 'httpdisk/slop_duration'
 require 'httpdisk/sloptions'
 require 'httpdisk/version'
 
-module HTTPDisk
-  ERROR_STATUS = 999
-end
+# cli
+require 'httpdisk/cli/args'
+require 'httpdisk/cli/main'
+
+# grep
+require 'httpdisk/grep/args'
+require 'httpdisk/grep/main'
+require 'httpdisk/grep/printer'

data/lib/httpdisk/cache.rb

@@ -9,7 +9,7 @@ module HTTPDisk
       @options = options
     end
 
-    %i[dir expires_in force force_errors].each do |method|
+    %i[dir expires force force_errors].each do |method|
       define_method(method) do
         options[method]
       end
@@ -38,6 +38,12 @@ module HTTPDisk
       Zlib::GzipWriter.open(path) { payload.write(_1) }
     end
 
+    # Delete existing response, if any
+    def delete(cache_key)
+      path = diskpath(cache_key)
+      FileUtils.rm(path) if File.exist?(path)
+    end
+
     # Relative path for this cache_key based on the cache key
     def diskpath(cache_key)
       File.join(dir, cache_key.diskpath)
@@ -61,7 +67,7 @@ module HTTPDisk
 
     # Is this path expired?
     def expired?(path)
-      expires_in && File.stat(path).mtime < Time.now - expires_in
+      expires && File.stat(path).mtime < Time.now - expires
     end
   end
 end

data/lib/httpdisk/cli/args.rb

@@ -0,0 +1,57 @@
+# manually load dependencies here since this is loaded standalone by bin
+require 'httpdisk/error'
+require 'httpdisk/slop_duration'
+require 'httpdisk/version'
+require 'slop'
+
+module HTTPDisk
+  module Cli
+    # Slop parsing. This is broken out so we can run without require 'httpdisk'.
+    module Args
+      def self.slop(args)
+        slop = Slop.parse(args) do |o|
+          o.banner = 'httpdisk [options] [url]'
+
+          # similar to curl
+          o.separator 'Similar to curl:'
+          o.string '-d', '--data', 'HTTP POST data'
+          o.array '-H', '--header', 'pass custom header(s) to server', delimiter: nil
+          o.boolean '-i', '--include', 'include response headers in the output'
+          o.integer '-m', '--max-time', 'maximum time allowed for the transfer'
+          o.string '-o', '--output', 'write to file instead of stdout'
+          o.string '-x', '--proxy', 'use host[:port] as proxy'
+          o.string '-X', '--request', 'HTTP method to use'
+          o.integer '--retry', 'retry request if problems occur'
+          o.boolean '-s', '--silent', "silent mode (don't print errors)"
+          o.string '-A', '--user-agent', 'send User-Agent to server'
+
+          # from httpdisk
+          o.separator 'Specific to httpdisk:'
+          o.string '--dir', 'httpdisk cache directory (defaults to ~/httpdisk)'
+          o.duration '--expires', 'when to expire cached requests (ex: 1h, 2d, 3w)'
+          o.boolean '--force', "don't read anything from cache (but still write)"
+          o.boolean '--force-errors', "don't read errors from cache (but still write)"
+          o.boolean '--status', 'show status for a url in the cache'
+
+          # generic
+          o.boolean '--version', 'show version' do
+            puts "httpdisk #{HTTPDisk::VERSION}"
+            exit
+          end
+          o.on '--help', 'show this help' do
+            puts o
+            exit
+          end
+        end
+
+        raise Slop::Error, '' if args.empty?
+        raise Slop::Error, 'no URL specified' if slop.args.empty?
+        raise Slop::Error, 'more than one URL specified' if slop.args.length > 1
+
+        slop.to_h.tap do
+          _1[:url] = slop.args.first
+        end
+      end
+    end
+  end
+end

data/lib/httpdisk/cli/main.rb

@@ -0,0 +1,166 @@
+require 'faraday-cookie_jar'
+require 'faraday_middleware'
+require 'ostruct'
+
+module HTTPDisk
+  module Cli
+    # Command line httpdisk command.
+    class Main
+      attr_reader :options
+
+      def initialize(options)
+        @options = options
+      end
+
+      # Make the request (or print status)
+      def run
+        # short circuit --status
+        if options[:status]
+          status
+          return
+        end
+
+        # create Faraday client
+        faraday = create_faraday
+
+        # run request
+        response = faraday.run_request(request_method, request_url, request_body, request_headers)
+        if response.status >= 400
+          raise CliError, "the requested URL returned error: #{response.status} #{response.reason_phrase}"
+        end
+
+        # output
+        if options[:output]
+          File.open(options[:output], 'w') { output(response, _1) }
+        else
+          output(response, $stdout)
+        end
+      end
+
+      def create_faraday
+        Faraday.new do
+          # connection settings
+          _1.proxy = options[:proxy] if options[:proxy]
+          _1.options.timeout = options[:max_time] if options[:max_time]
+
+          # cookie middleware
+          _1.use :cookie_jar
+
+          # BEFORE httpdisk so each redirect segment is cached
+          _1.response :follow_redirects
+
+          # httpdisk
+          _1.use :httpdisk, client_options
+
+          # AFTER httpdisk so transient failures are not cached
+          if options[:retry]
+            # we have a very liberal retry policy
+            retry_options = {
+              max: options[:retry],
+              methods: %w[delete get head options patch post put trace],
+              retry_statuses: (500..600).to_a,
+              retry_if: ->(_env, _err) { true },
+            }
+            _1.request :retry, retry_options
+          end
+        end
+      end
+
+      # Support for --status
+      def status
+        # build env
+        env = Faraday::Env.new.tap do
+          _1.method = request_method
+          _1.request_body = request_body
+          _1.request_headers = request_headers
+          _1.url = request_url
+        end
+
+        # now print status
+        client = HTTPDisk::Client.new(nil, client_options)
+        client.status(env).each do
+          puts "#{_1}: #{_2.inspect}"
+        end
+      end
+
+      # Output response to f
+      def output(response, f)
+        if options[:include]
+          f.puts "HTTPDISK #{response.status} #{response.reason_phrase}"
+          response.headers.each { f.puts("#{_1}: #{_2}") }
+          f.puts
+        end
+        f.write(response.body)
+      end
+
+      #
+      # request_XXX
+      #
+
+      # HTTP method (get, post, etc.)
+      def request_method
+        method = if options[:request]
+          options[:request]
+        elsif options[:data]
+          'post'
+        end
+        method ||= 'get'
+        method = method.downcase.to_sym
+
+        if !Faraday::Connection::METHODS.include?(method)
+          raise CliError, "invalid --request #{method.inspect}"
+        end
+
+        method
+      end
+
+      # Request url
+      def request_url
+        url = options[:url]
+        # recover from missing http:
+        if url !~ %r{^https?://}i
+          if url =~ %r{^\w+://}
+            raise CliError, 'only http/https supported'
+          end
+
+          url = "http://#{url}"
+        end
+        URI.parse(url)
+      rescue URI::InvalidURIError
+        raise CliError, "invalid url #{url.inspect}"
+      end
+
+      # Request body
+      def request_body
+        options[:data]
+      end
+
+      # Request headers
+      def request_headers
+        {}.tap do |headers|
+          if options[:user_agent]
+            headers['User-Agent'] = options[:user_agent]
+          end
+
+          options[:header].each do |header|
+            key, value = header.split(': ', 2)
+            if !key || !value || key.empty? || value.empty?
+              raise CliError, "invalid --header #{header.inspect}"
+            end
+
+            headers[key] = value
+          end
+        end
+      end
+
+      #
+      # helpers
+      #
+
+      # Options to HTTPDisk::Client
+      def client_options
+        options.slice(:dir, :expires, :force, :force_errors)
+      end
+    end
+  end
+end

data/lib/httpdisk/client.rb

@@ -9,7 +9,7 @@ module HTTPDisk
     def initialize(app, options = {})
       options = Sloptions.parse(options) do
         _1.string :dir, default: File.join(ENV['HOME'], 'httpdisk')
-        _1.integer :expires_in
+        _1.integer :expires
         _1.boolean :force
         _1.boolean :force_errors
         _1.array :ignore_params, default: []

data/lib/httpdisk/error.rb

@@ -1,3 +1,5 @@
 module HTTPDisk
+  ERROR_STATUS = 999
+
   class CliError < StandardError; end
 end

data/lib/httpdisk/grep/args.rb

@@ -0,0 +1,35 @@
+# manually load dependencies here since this is loaded standalone by bin
+require 'httpdisk/version'
+require 'slop'
+
+module HTTPDisk
+  module Grep
+    module Args
+      # Slop parsing. This is broken out so we can run without require 'httpdisk'.
+      def self.slop(args)
+        slop = Slop.parse(args) do |o|
+          o.banner = 'httpdisk-grep [options] pattern [path ...]'
+          o.boolean '-c', '--count', 'suppress normal output and show count'
+          o.boolean '-h', '--head', 'show req headers before each match'
+          o.boolean '-s', '--silent', 'do not print anything to stdout'
+          o.boolean '--version', 'show version' do
+            puts "httpdisk-grep #{HTTPDisk::VERSION}"
+            exit
+          end
+          o.on '--help', 'show this help' do
+            puts o
+            exit
+          end
+        end
+
+        raise Slop::Error, '' if args.empty?
+        raise Slop::Error, 'no PATTERN specified' if slop.args.empty?
+
+        slop.to_h.tap do
+          _1[:pattern] = slop.args.shift
+          _1[:roots] = slop.args
+        end
+      end
+    end
+  end
+end

data/lib/httpdisk/grep/main.rb

@@ -0,0 +1,110 @@
+require 'find'
+require 'json'
+
+module HTTPDisk
+  module Grep
+    class Main
+      attr_reader :options, :success
+
+      def initialize(options)
+        @options = options
+      end
+
+      # Enumerate file paths one at a time. Returns true if matches were found.
+      def run
+        paths.each do
+          begin
+            run_one(_1)
+          rescue StandardError => e
+            if ENV['HTTPDISK_DEBUG']
+              $stderr.puts
+              $stderr.puts e.class
+              $stderr.puts e.backtrace.join("\n")
+            end
+            raise CliError, "#{e.message[0, 70]} (#{_1})"
+          end
+        end
+        success
+      end
+
+      def run_one(path)
+        # read payload & body
+        payload = Zlib::GzipReader.open(path) { Payload.read(_1) }
+        body = prepare_body(payload)
+
+        # collect all_matches
+        all_matches = body.each_line.map do |line|
+          [].tap do |matches|
+            line.scan(pattern) { matches << Regexp.last_match }
+          end
+        end.reject(&:empty?)
+        return if all_matches.empty?
+
+        # print
+        @success = true
+        printer.print(path, payload, all_matches)
+      end
+
+      # file paths to be searched
+      def paths
+        # roots
+        roots = options[:roots]
+        roots = ['.'] if roots.empty?
+
+        # find files in roots
+        paths = roots.flat_map { Find.find(_1).to_a }.sort
+        paths = paths.select { File.file?(_1) }
+
+        # strip default './'
+        paths = paths.map { _1.gsub(%r{^\./}, '') } if options[:roots].empty?
+        paths
+      end
+
+      # convert raw body into something palatable for pattern matching
+      def prepare_body(payload)
+        body = payload.body
+
+        if content_type = payload.headers['Content-Type']
+          # Mismatches between Content-Type and body.encoding are fatal, so make
+          # an effort to align them.
+          if charset = content_type[/charset=([^;]+)/, 1]
+            encoding = begin
+              Encoding.find(charset)
+            rescue StandardError
+              nil
+            end
+            if encoding && body.encoding != encoding
+              body.force_encoding(encoding)
+            end
+          end
+
+          # pretty print json for easier searching
+          if content_type =~ /\bjson\b/
+            body = JSON.pretty_generate(JSON.parse(body))
+          end
+        end
+
+        body
+      end
+
+      # regex pattern from options
+      def pattern
+        @pattern ||= Regexp.new(options[:pattern], Regexp::IGNORECASE)
+      end
+
+      # printer for output
+      def printer
+        @printer ||= case
+        when options[:silent]
+          Grep::SilentPrinter.new
+        when options[:count]
+          Grep::CountPrinter.new($stdout)
+        when options[:head] || $stdout.tty?
+          Grep::HeaderPrinter.new($stdout, options[:head])
+        else
+          Grep::TersePrinter.new($stdout)
+        end
+      end
+    end
+  end
+end

data/lib/httpdisk/grep/printer.rb

@@ -0,0 +1,99 @@
+module HTTPDisk
+  module Grep
+    class Printer
+      GREP_COLOR = '37;45'.freeze
+
+      attr_reader :output
+
+      def initialize(output)
+        @output = output
+      end
+
+      def print(path, payload, all_matches); end
+
+      protected
+
+      #
+      # helpers for subclasses
+      #
+
+      def grep_color
+        @grep_color ||= (ENV['GREP_COLOR'] || GREP_COLOR)
+      end
+
+      def print_matches(matches)
+        s = matches.first.string
+        if output.tty?
+          s = [].tap do |result|
+            ii = 0
+            matches.each do
+              result << s[ii..._1.begin(0)]
+              result << "\e["
+              result << grep_color
+              result << 'm'
+              result << _1[0]
+              result << "\e[0m"
+              ii = _1.end(0)
+            end
+            result << s[ii..]
+          end.join
+        end
+        output.puts s
+      end
+    end
+
+    #
+    # subclasses
+    #
+
+    # path:count
+    class CountPrinter < Printer
+      def print(path, _payload, all_matches)
+        output.puts "#{path}:#{all_matches.length}"
+      end
+    end
+
+    # header, then each match
+    class HeaderPrinter < Printer
+      attr_reader :head, :printed
+
+      def initialize(output, head)
+        super(output)
+        @head = head
+        @printed = 0
+      end
+
+      def print(path, payload, all_matches)
+        # separator & filename
+        output.puts if (@printed += 1) > 1
+        output.puts path
+
+        # --head
+        if head
+          io = StringIO.new
+          payload.write_header(io)
+          io.string.lines.each { output.puts "< #{_1}" }
+        end
+
+        # matches
+        all_matches.each { print_matches(_1) }
+      end
+    end
+
+    class SilentPrinter < Printer
+      def initialize
+        super(nil)
+      end
+    end
+
+    # each match as path:match
+    class TersePrinter < Printer
+      def print(path, _payload, all_matches)
+        all_matches.each do
+          output.write("#{path}:")
+          print_matches(_1)
+        end
+      end
+    end
+  end
+end

data/lib/httpdisk/payload.rb

@@ -43,7 +43,7 @@ module HTTPDisk
       status >= 400
     end
 
-    def write(f)
+    def write_header(f)
       # comment
       f.puts "# #{comment}"
 
@@ -52,9 +52,11 @@ module HTTPDisk
 
       # headers
       headers.each { f.puts("#{_1}: #{_2}") }
-      f.puts
+    end
 
-      # body
+    def write(f)
+      write_header(f)
+      f.puts
       f.write(body)
     end
   end

data/lib/httpdisk/slop_duration.rb

@@ -0,0 +1,24 @@
+require 'slop'
+
+module Slop
+  # Custom duration type for Slop, used for --expires. Raises aggressively
+  # because this is a tricky and lightly documented option.
+  class DurationOption < Option
+    UNITS = {
+      s: 1,
+      m: 60,
+      h: 60 * 60,
+      d: 24 * 60 * 60,
+      w: 7 * 24 * 60 * 60,
+      y: 365 * 7 * 24 * 60 * 60,
+    }.freeze
+
+    def call(value)
+      m = value.match(/^(\d+)([smhdwy])?$/)
+      raise Slop::Error, "invalid --expires #{value.inspect}" if !m
+
+      num, unit = m[1].to_i, (m[2] || 's').to_sym
+      num * UNITS[unit]
+    end
+  end
+end

data/lib/httpdisk/version.rb

@@ -1,3 +1,3 @@
 module HTTPDisk
-  VERSION = '0.3.0'.freeze
+  VERSION = '0.4.0'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: httpdisk
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Adam Doppelt
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-05-20 00:00:00.000000000 Z
+date: 2021-06-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: faraday
@@ -70,6 +70,7 @@ description: httpdisk works with faraday to aggressively cache responses on disk
 email: amd@gurge.com
 executables:
 - httpdisk
+- httpdisk-grep
 extensions: []
 extra_rdoc_files: []
 files:
@@ -82,16 +83,21 @@ files:
 - README.md
 - Rakefile
 - bin/httpdisk
+- bin/httpdisk-grep
 - examples.rb
 - httpdisk.gemspec
 - lib/httpdisk.rb
 - lib/httpdisk/cache.rb
 - lib/httpdisk/cache_key.rb
-- lib/httpdisk/cli.rb
-- lib/httpdisk/cli_slop.rb
+- lib/httpdisk/cli/args.rb
+- lib/httpdisk/cli/main.rb
 - lib/httpdisk/client.rb
 - lib/httpdisk/error.rb
+- lib/httpdisk/grep/args.rb
+- lib/httpdisk/grep/main.rb
+- lib/httpdisk/grep/printer.rb
 - lib/httpdisk/payload.rb
+- lib/httpdisk/slop_duration.rb
 - lib/httpdisk/sloptions.rb
 - lib/httpdisk/version.rb
 - logo.svg

data/lib/httpdisk/cli.rb

@@ -1,223 +0,0 @@
-require 'faraday-cookie_jar'
-require 'faraday_middleware'
-require 'ostruct'
-
-module HTTPDisk
-  # Command line httpdisk command.
-  class Cli
-    attr_reader :options
-
-    # for --expires
-    UNITS = {
-      s: 1,
-      m: 60,
-      h: 60 * 60,
-      d: 24 * 60 * 60,
-      w: 7 * 24 * 60 * 60,
-      y: 365 * 7 * 24 * 60 * 60,
-    }.freeze
-
-    def initialize(options)
-      @options = options
-    end
-
-    # we have a very liberal retry policy
-    RETRY_OPTIONS = {
-      methods: %w[delete get head options patch post put trace],
-      retry_statuses: (400..600).to_a,
-      retry_if: ->(_env, _err) { true },
-    }.freeze
-
-    # Make the request (or print status)
-    def run
-      # short circuit --status
-      if options[:status]
-        status
-        return
-      end
-
-      # create Faraday client
-      faraday = create_faraday
-
-      # run request
-      response = faraday.run_request(request_method, request_url, request_body, request_headers)
-      if response.status >= 400
-        raise CliError, "the requested URL returned error: #{response.status} #{response.reason_phrase}"
-      end
-
-      # output
-      if options[:output]
-        File.open(options[:output], 'w') { output(response, _1) }
-      else
-        output(response, $stdout)
-      end
-    end
-
-    def create_faraday
-      Faraday.new do
-        # connection settings
-        _1.proxy = proxy if options[:proxy]
-        _1.options.timeout = options[:max_time] if options[:max_time]
-
-        # cookie middleware
-        _1.use :cookie_jar
-
-        # BEFORE httpdisk so each redirect segment is cached
-        _1.response :follow_redirects
-
-        # httpdisk
-        _1.use :httpdisk, client_options
-
-        # AFTER httpdisk so transient failures are not cached
-        if options[:retry]
-          _1.request :retry, RETRY_OPTIONS.merge(max: options[:retry])
-        end
-      end
-    end
-
-    # Support for --status
-    def status
-      # build env
-      env = Faraday::Env.new.tap do
-        _1.method = request_method
-        _1.request_body = request_body
-        _1.request_headers = request_headers
-        _1.url = request_url
-      end
-
-      # now print status
-      client = HTTPDisk::Client.new(nil, client_options)
-      client.status(env).each do
-        puts "#{_1}: #{_2.inspect}"
-      end
-    end
-
-    # Output response to f
-    def output(response, f)
-      if options[:include]
-        f.puts "HTTPDISK #{response.status} #{response.reason_phrase}"
-        response.headers.each { f.puts("#{_1}: #{_2}") }
-        f.puts
-      end
-      f.write(response.body)
-    end
-
-    #
-    # request_XXX
-    #
-
-    # HTTP method (get, post, etc.)
-    def request_method
-      method = if options[:request]
-        options[:request]
-      elsif options[:data]
-        'post'
-      end
-      method ||= 'get'
-      method = method.downcase.to_sym
-
-      if !Faraday::Connection::METHODS.include?(method)
-        raise CliError, "invalid --request #{method.inspect}"
-      end
-
-      method
-    end
-
-    # Request url
-    def request_url
-      url = options[:url]
-      # recover from missing http:
-      if url !~ %r{^https?://}i
-        if url =~ %r{^\w+://}
-          raise CliError, 'only http/https supported'
-        end
-
-        url = "http://#{url}"
-      end
-      URI.parse(url)
-    rescue URI::InvalidURIError
-      raise CliError, "invalid url #{url.inspect}"
-    end
-
-    # Request body
-    def request_body
-      options[:data]
-    end
-
-    # Request headers
-    def request_headers
-      {}.tap do |headers|
-        if options[:user_agent]
-          headers['User-Agent'] = options[:user_agent]
-        end
-
-        options[:header].each do |header|
-          key, value = header.split(': ', 2)
-          if !key || !value || key.empty? || value.empty?
-            raise CliError, "invalid --header #{header.inspect}"
-          end
-
-          headers[key] = value
-        end
-      end
-    end
-
-    #
-    # helpers
-    #
-
-    # Options to HTTPDisk::Client
-    def client_options
-      {}.tap do |client_options|
-        client_options[:dir] = options[:dir]
-        if options[:expires]
-          seconds = parse_expires(options[:expires])
-          if !seconds
-            raise CliError, "invalid --expires #{options[:expires].inspect}"
-          end
-
-          client_options[:expires_in] = seconds
-        end
-        client_options[:force] = options[:force]
-        client_options[:force_errors] = options[:force_errors]
-      end
-    end
-
-    # Return validated --proxy flag if present
-    def proxy
-      return if !options[:proxy]
-
-      proxy = parse_proxy(options[:proxy])
-      raise CliError, "--proxy should be host[:port], not #{options[:proxy].inspect}" if !proxy
-
-      proxy
-    end
-
-    # Parse --expires flag
-    def parse_expires(s)
-      m = s.match(/^(\d+)([smhdwy])?$/)
-      return if !m
-
-      num, unit = m[1].to_i, (m[2] || 's').to_sym
-      return if !UNITS.key?(unit)
-
-      num * UNITS[unit]
-    end
-
-    # Parse --proxy flag
-    def parse_proxy(proxy_flag)
-      host, port = proxy_flag.split(':', 2)
-      return if !host || host.empty?
-      return if port&.empty?
-
-      URI.parse('http://placeholder').tap do
-        begin
-          _1.host = host
-          _1.port = port if port
-        rescue URI::InvalidComponentError
-          return
-        end
-      end.to_s
-    end
-  end
-end

data/lib/httpdisk/cli_slop.rb

@@ -1,54 +0,0 @@
-# manually load dependencies here since this is loaded standalone by bin
-require 'httpdisk/error'
-require 'httpdisk/version'
-require 'slop'
-
-module HTTPDisk
-  # Slop parsing. This is broken out so we can run without require 'httpdisk'.
-  module CliSlop
-    def self.slop(args)
-      slop = Slop.parse(args) do |o|
-        o.banner = 'httpdisk [options] [url]'
-
-        # similar to curl
-        o.separator 'Similar to curl:'
-        o.string '-d', '--data', 'HTTP POST data'
-        o.array '-H', '--header', 'pass custom header(s) to server', delimiter: nil
-        o.boolean '-i', '--include', 'include response headers in the output'
-        o.integer '-m', '--max-time', 'maximum time allowed for the transfer'
-        o.string '-o', '--output', 'write to file instead of stdout'
-        o.string '-x', '--proxy', 'use host[:port] as proxy'
-        o.string '-X', '--request', 'HTTP method to use'
-        o.integer '--retry', 'retry request if problems occur'
-        o.boolean '-s', '--silent', "silent mode (don't print errors)"
-        o.string '-A', '--user-agent', 'send User-Agent to server'
-
-        # from httpdisk
-        o.separator 'Specific to httpdisk:'
-        o.string '--dir', 'httpdisk cache directory (defaults to ~/httpdisk)'
-        o.string '--expires', 'when to expire cached requests (ex: 1h, 2d, 3w)'
-        o.boolean '--force', "don't read anything from cache (but still write)"
-        o.boolean '--force-errors', "don't read errors from cache (but still write)"
-        o.boolean '--status', 'show status for a url in the cache'
-
-        # generic
-        o.boolean '--version', 'show version' do
-          puts "httpdisk #{HTTPDisk::VERSION}"
-          exit
-        end
-        o.on '--help', 'show this help' do
-          puts o
-          exit
-        end
-      end
-
-      raise Slop::Error, '' if args.empty?
-      raise Slop::Error, 'no URL specified' if slop.args.empty?
-      raise Slop::Error, 'more than one URL specified' if slop.args.length > 1
-
-      slop.to_h.tap do
-        _1[:url] = slop.args.first
-      end
-    end
-  end
-end