httpdisk 0.2.0 → 0.5.2

data/lib/httpdisk/cache_key.rb

@@ -4,14 +4,14 @@ require 'uri'
 
 module HTTPDisk
   class CacheKey
-    attr_reader :env
+    attr_reader :env, :ignore_params
 
-    def initialize(env)
-      @env = env
+    def initialize(env, ignore_params: [])
+      @env, @ignore_params = env, ignore_params
 
       # sanity checks
-      raise 'http/https required' if env.url.scheme !~ /^https?$/
-      raise 'hostname required' if !env.url.host
+      raise InvalidUrl, "http/https required #{env.url.inspect}" if env.url.scheme !~ /^https?$/
+      raise InvalidUrl, "hostname required #{env.url.inspect}" if !env.url.host
     end
 
     def url
@@ -79,7 +79,16 @@ module HTTPDisk
 
     # Calculate canonical key for a query
     def querykey(q)
-      q.split('&').sort.join('&')
+      parts = q.split('&').sort
+      if !ignore_params.empty?
+        parts = parts.map do |part|
+          key, value = part.split('=', 2)
+          next if ignore_params.include?(key)
+
+          "#{key}=#{value}"
+        end.compact
+      end
+      parts.join('&')
     end
 
     def default_port?

data/lib/httpdisk/cli/args.rb

@@ -0,0 +1,57 @@
+# manually load dependencies here since this is loaded standalone by bin
+require 'httpdisk/error'
+require 'httpdisk/slop_duration'
+require 'httpdisk/version'
+require 'slop'
+
+module HTTPDisk
+  module Cli
+    # Slop parsing. This is broken out so we can run without require 'httpdisk'.
+    module Args
+      def self.slop(args)
+        slop = Slop.parse(args) do |o|
+          o.banner = 'httpdisk [options] [url]'
+
+          # similar to curl
+          o.separator 'Similar to curl:'
+          o.string '-d', '--data', 'HTTP POST data'
+          o.array '-H', '--header', 'pass custom header(s) to server', delimiter: nil
+          o.boolean '-i', '--include', 'include response headers in the output'
+          o.integer '-m', '--max-time', 'maximum time allowed for the transfer'
+          o.string '-o', '--output', 'write to file instead of stdout'
+          o.string '-x', '--proxy', 'use host[:port] as proxy'
+          o.string '-X', '--request', 'HTTP method to use'
+          o.integer '--retry', 'retry request if problems occur'
+          o.boolean '-s', '--silent', "silent mode (don't print errors)"
+          o.string '-A', '--user-agent', 'send User-Agent to server'
+
+          # from httpdisk
+          o.separator 'Specific to httpdisk:'
+          o.string '--dir', 'httpdisk cache directory (defaults to ~/httpdisk)'
+          o.duration '--expires', 'when to expire cached requests (ex: 1h, 2d, 3w)'
+          o.boolean '--force', "don't read anything from cache (but still write)"
+          o.boolean '--force-errors', "don't read errors from cache (but still write)"
+          o.boolean '--status', 'show status for a url in the cache'
+
+          # generic
+          o.boolean '--version', 'show version' do
+            puts "httpdisk #{HTTPDisk::VERSION}"
+            exit
+          end
+          o.on '--help', 'show this help' do
+            puts o
+            exit
+          end
+        end
+
+        raise Slop::Error, '' if args.empty?
+        raise Slop::Error, 'no URL specified' if slop.args.empty?
+        raise Slop::Error, 'more than one URL specified' if slop.args.length > 1
+
+        slop.to_h.tap do
+          _1[:url] = slop.args.first
+        end
+      end
+    end
+  end
+end

data/lib/httpdisk/cli/main.rb

@@ -0,0 +1,169 @@
+require 'faraday-cookie_jar'
+require 'faraday_middleware'
+require 'ostruct'
+
+module HTTPDisk
+  module Cli
+    # Command line httpdisk command.
+    class Main
+      attr_reader :options
+
+      def initialize(options)
+        @options = options
+      end
+
+      # Make the request (or print status)
+      def run
+        # short circuit --status
+        if options[:status]
+          status
+          return
+        end
+
+        # create Faraday client
+        faraday = create_faraday
+
+        # run request
+        response = faraday.run_request(request_method, request_url, request_body, request_headers)
+        if response.status >= 400
+          raise CliError, "the requested URL returned error: #{response.status} #{response.reason_phrase}"
+        end
+
+        # output
+        if options[:output]
+          File.open(options[:output], 'w') { output(response, _1) }
+        else
+          output(response, $stdout)
+        end
+      end
+
+      def create_faraday
+        Faraday.new do
+          # connection settings
+          _1.proxy = options[:proxy] if options[:proxy]
+          _1.options.timeout = options[:max_time] if options[:max_time]
+
+          # cookie middleware
+          _1.use :cookie_jar
+
+          # BEFORE httpdisk so each redirect segment is cached
+          _1.response :follow_redirects
+
+          # httpdisk
+          _1.use :httpdisk, client_options
+
+          # AFTER httpdisk so transient failures are not cached
+          if options[:retry]
+            # we have a very liberal retry policy
+            retry_options = {
+              max: options[:retry],
+              methods: %w[delete get head options patch post put trace],
+              retry_statuses: (500..600).to_a,
+              retry_if: ->(_env, _err) { true },
+            }
+            _1.request :retry, retry_options
+          end
+        end
+      end
+
+      # Support for --status
+      def status
+        # build env
+        env = Faraday::Env.new.tap do
+          _1.method = request_method
+          _1.request_body = request_body
+          _1.request_headers = request_headers
+          # Run the url through Faraday to make sure we see the same stuff as middleware.
+          _1.url = Faraday.new.build_url(request_url)
+        end
+
+        # now print status
+        client = HTTPDisk::Client.new(nil, client_options)
+        client.status(env).each do
+          puts "#{_1}: #{_2.inspect}"
+        end
+      end
+
+      # Output response to f
+      def output(response, f)
+        if options[:include]
+          f.puts "HTTPDISK #{response.status} #{response.reason_phrase}"
+          response.headers.each { f.puts("#{_1}: #{_2}") }
+          f.puts
+        end
+        f.write(response.body)
+      end
+
+      #
+      # request_XXX
+      #
+
+      # HTTP method (get, post, etc.)
+      def request_method
+        method = if options[:request]
+          options[:request]
+        elsif options[:data]
+          'post'
+        end
+        method ||= 'get'
+        method = method.downcase.to_sym
+
+        if !Faraday::Connection::METHODS.include?(method)
+          raise CliError, "invalid --request #{method.inspect}"
+        end
+
+        method
+      end
+
+      # Request url
+      def request_url
+        url = options[:url]
+        # recover from missing http:
+        if url !~ %r{^https?://}i
+          if url =~ %r{^\w+://}
+            raise CliError, 'only http/https supported'
+          end
+
+          url = "http://#{url}"
+        end
+        URI.parse(url)
+      rescue URI::InvalidURIError
+        raise CliError, "invalid url #{url.inspect}"
+      end
+
+      # Request body
+      def request_body
+        options[:data]
+      end
+
+      # Request headers
+      def request_headers
+        {}.tap do |headers|
+          if options[:user_agent]
+            headers['User-Agent'] = options[:user_agent]
+          end
+
+          options[:header].each do |header|
+            key, value = header.split(': ', 2)
+            if !key || !value || key.empty? || value.empty?
+              raise CliError, "invalid --header #{header.inspect}"
+            end
+
+            headers[key] = value
+          end
+        end
+      end
+
+      #
+      # helpers
+      #
+
+      # Options to HTTPDisk::Client
+      def client_options
+        client_options = options.slice(:dir, :expires, :force, :force_errors)
+        client_options[:utf8] = true
+        client_options
+      end
+    end
+  end
+end

data/lib/httpdisk/client.rb

@@ -1,38 +1,43 @@
+require 'content-type'
 require 'faraday'
 require 'logger'
 
 module HTTPDisk
-  OPTIONS = {
-    dir: File.join(ENV['HOME'], 'httpdisk'),
-    expires_in: nil,
-    force: false,
-    force_errors: false,
-    logger: false,
-  }.freeze
-
   # Middleware and main entry point.
   class Client < Faraday::Middleware
     attr_reader :cache, :options
 
     def initialize(app, options = {})
-      super(app, options = OPTIONS.merge(options.compact))
+      options = Sloptions.parse(options) do
+        _1.string :dir, default: File.join(ENV['HOME'], 'httpdisk')
+        _1.integer :expires
+        _1.boolean :force
+        _1.boolean :force_errors
+        _1.array :ignore_params, default: []
+        _1.on :logger, type: [:boolean, Logger]
+        _1.boolean :utf8
+      end
+
+      super(app, options)
       @cache = Cache.new(options)
     end
 
     def call(env)
-      cache_key = CacheKey.new(env)
+      cache_key = CacheKey.new(env, ignore_params: ignore_params)
       logger&.info("#{env.method.upcase} #{env.url} (#{cache.status(cache_key)})")
+      env[:httpdisk_diskpath] = cache.diskpath(cache_key)
 
-      if cached_response = read(cache_key, env)
-        cached_response.env[:httpdisk] = true
-        return cached_response
-      end
-
-      # miss
-      perform(env).tap do |response|
+      # check cache, fallback to network
+      if response = read(cache_key, env)
+        response.env[:httpdisk] = true
+      else
+        response = perform(env)
         response.env[:httpdisk] = false
         write(cache_key, env, response)
       end
+
+      encode_body(response)
+      response
     end
 
     # Returns cache status for this request
@@ -100,10 +105,68 @@ module HTTPDisk
       err.to_s =~ /#{proxy.host}.*#{proxy.port}/
     end
 
+    # Set string encoding for response body. The cache always returns
+    # ASCII-8BIT, but we have no idea what the encoding will be from the
+    # network. Not all adapters honor Content-Type (including the default
+    # adapter).
+    def encode_body(response)
+      body = response.body || ''
+
+      # parse Content-Type
+      begin
+        content_type = response['Content-Type'] && ContentType.parse(response['Content-Type'])
+      rescue Parslet::ParseFailed
+        # unparsable
+      end
+
+      # look at charset and set body encoding if necessary
+      encoding = encoding_for(content_type)
+      if body.encoding != encoding
+        body = body.dup if body.frozen?
+        body.force_encoding(encoding)
+      end
+
+      # if :utf8, force body to UTF-8
+      if options[:utf8] && content_type && response_text?(content_type)
+        body = body.dup if body.frozen?
+        begin
+          body.encode!('UTF-8', invalid: :replace, undef: :replace, replace: '?')
+        rescue Encoding::ConverterNotFoundError
+          # rare, can't do anything here
+          body = "httpdisk could not convert from #{body.encoding.name} to UTF-8"
+        end
+      end
+
+      response.env[:body] = body
+    end
+
+    def encoding_for(content_type)
+      if content_type&.charset
+        begin
+          return Encoding.find(content_type.charset)
+        rescue ArgumentError
+          # unknown charset
+        end
+      end
+      Encoding::ASCII_8BIT
+    end
+
+    def response_text?(content_type)
+      content_type.type == 'text' || content_type.mime_type == 'application/json'
+    end
+
+    #
+    # options
+    #
+
+    def ignore_params
+      @ignore_params ||= options[:ignore_params].map { CGI.escape(_1.to_s) }.to_set
+    end
+
     def logger
-      return @logger if defined?(@logger)
+      return if !options[:logger]
 
-      @logger = case options[:logger]
+      @logger ||= case options[:logger]
       when true then Logger.new($stderr)
       when Logger then options[:logger]
       end

data/lib/httpdisk/error.rb

@@ -1,3 +1,7 @@
 module HTTPDisk
+  ERROR_STATUS = 999
+
   class CliError < StandardError; end
+
+  class InvalidUrl < StandardError; end
 end

data/lib/httpdisk/grep/args.rb

@@ -0,0 +1,35 @@
+# manually load dependencies here since this is loaded standalone by bin
+require 'httpdisk/version'
+require 'slop'
+
+module HTTPDisk
+  module Grep
+    module Args
+      # Slop parsing. This is broken out so we can run without require 'httpdisk'.
+      def self.slop(args)
+        slop = Slop.parse(args) do |o|
+          o.banner = 'httpdisk-grep [options] pattern [path ...]'
+          o.boolean '-c', '--count', 'suppress normal output and show count'
+          o.boolean '-h', '--head', 'show req headers before each match'
+          o.boolean '-s', '--silent', 'do not print anything to stdout'
+          o.boolean '--version', 'show version' do
+            puts "httpdisk-grep #{HTTPDisk::VERSION}"
+            exit
+          end
+          o.on '--help', 'show this help' do
+            puts o
+            exit
+          end
+        end
+
+        raise Slop::Error, '' if args.empty?
+        raise Slop::Error, 'no PATTERN specified' if slop.args.empty?
+
+        slop.to_h.tap do
+          _1[:pattern] = slop.args.shift
+          _1[:roots] = slop.args
+        end
+      end
+    end
+  end
+end

data/lib/httpdisk/grep/main.rb

@@ -0,0 +1,112 @@
+require 'find'
+require 'json'
+
+module HTTPDisk
+  module Grep
+    class Main
+      attr_reader :options, :success
+
+      def initialize(options)
+        @options = options
+      end
+
+      # Enumerate file paths one at a time. Returns true if matches were found.
+      def run
+        paths.each do
+          begin
+            run_one(_1)
+          rescue StandardError => e
+            if ENV['HTTPDISK_DEBUG']
+              $stderr.puts
+              $stderr.puts e.class
+              $stderr.puts e.backtrace.join("\n")
+            end
+            raise CliError, "#{e.message[0, 70]} (#{_1})"
+          end
+        end
+        success
+      end
+
+      def run_one(path)
+        # read payload & body
+        payload = Zlib::GzipReader.open(path, encoding: 'ASCII-8BIT') do
+          Payload.read(_1)
+        end
+        body = prepare_body(payload)
+
+        # collect all_matches
+        all_matches = body.each_line.map do |line|
+          [].tap do |matches|
+            line.scan(pattern) { matches << Regexp.last_match }
+          end
+        end.reject(&:empty?)
+        return if all_matches.empty?
+
+        # print
+        @success = true
+        printer.print(path, payload, all_matches)
+      end
+
+      # file paths to be searched
+      def paths
+        # roots
+        roots = options[:roots]
+        roots = ['.'] if roots.empty?
+
+        # find files in roots
+        paths = roots.flat_map { Find.find(_1).to_a }.sort
+        paths = paths.select { File.file?(_1) }
+
+        # strip default './'
+        paths = paths.map { _1.gsub(%r{^\./}, '') } if options[:roots].empty?
+        paths
+      end
+
+      # convert raw body into something palatable for pattern matching
+      def prepare_body(payload)
+        body = payload.body
+
+        if content_type = payload.headers['Content-Type']
+          # Mismatches between Content-Type and body.encoding are fatal, so make
+          # an effort to align them.
+          if charset = content_type[/charset=([^;]+)/, 1]
+            encoding = begin
+              Encoding.find(charset)
+            rescue StandardError
+              nil
+            end
+            if encoding && body.encoding != encoding
+              body.force_encoding(encoding)
+            end
+          end
+
+          # pretty print json for easier searching
+          if content_type =~ /\bjson\b/
+            body = JSON.pretty_generate(JSON.parse(body))
+          end
+        end
+
+        body
+      end
+
+      # regex pattern from options
+      def pattern
+        @pattern ||= Regexp.new(options[:pattern], Regexp::IGNORECASE)
+      end
+
+      # printer for output
+      def printer
+        @printer ||= case
+        when options[:silent]
+          Grep::SilentPrinter.new
+        when options[:count]
+          Grep::CountPrinter.new($stdout)
+        when options[:head] || $stdout.tty?
+          Grep::HeaderPrinter.new($stdout, options[:head])
+        else
+          Grep::TersePrinter.new($stdout)
+        end
+      end
+    end
+  end
+end