httpdisk 0.1.0 → 0.5.1

data/lib/httpdisk/cache_key.rb

@@ -4,14 +4,14 @@ require 'uri'
 
 module HTTPDisk
   class CacheKey
-    attr_reader :env
+    attr_reader :env, :ignore_params
 
-    def initialize(env)
-      @env = env
+    def initialize(env, ignore_params: [])
+      @env, @ignore_params = env, ignore_params
 
       # sanity checks
-      raise 'http/https required' if env.url.scheme !~ /^https?$/
-      raise 'hostname required' if !env.url.host
+      raise InvalidUrl, "http/https required #{env.url.inspect}" if env.url.scheme !~ /^https?$/
+      raise InvalidUrl, "hostname required #{env.url.inspect}" if !env.url.host
     end
 
     def url
@@ -68,10 +68,9 @@ module HTTPDisk
     # Calculate cache key segment for body
     def bodykey
       body = env.request_body.to_s
-      case
-      when env.request_headers['Content-Type'] == 'application/x-www-form-urlencoded'
+      if env.request_headers['Content-Type'] == 'application/x-www-form-urlencoded'
         querykey(body)
-      when body.length < 50
+      elsif body.length < 50
         body
       else
         Digest::MD5.hexdigest(body)
@@ -80,7 +79,16 @@ module HTTPDisk
 
     # Calculate canonical key for a query
     def querykey(q)
-      q.split('&').sort.join('&')
+      parts = q.split('&').sort
+      if !ignore_params.empty?
+        parts = parts.map do |part|
+          key, value = part.split('=', 2)
+          next if ignore_params.include?(key)
+
+          "#{key}=#{value}"
+        end.compact
+      end
+      parts.join('&')
     end
 
     def default_port?

data/lib/httpdisk/cli/args.rb

@@ -0,0 +1,57 @@
+# manually load dependencies here since this is loaded standalone by bin
+require 'httpdisk/error'
+require 'httpdisk/slop_duration'
+require 'httpdisk/version'
+require 'slop'
+
+module HTTPDisk
+  module Cli
+    # Slop parsing. This is broken out so we can run without require 'httpdisk'.
+    module Args
+      def self.slop(args)
+        slop = Slop.parse(args) do |o|
+          o.banner = 'httpdisk [options] [url]'
+
+          # similar to curl
+          o.separator 'Similar to curl:'
+          o.string '-d', '--data', 'HTTP POST data'
+          o.array '-H', '--header', 'pass custom header(s) to server', delimiter: nil
+          o.boolean '-i', '--include', 'include response headers in the output'
+          o.integer '-m', '--max-time', 'maximum time allowed for the transfer'
+          o.string '-o', '--output', 'write to file instead of stdout'
+          o.string '-x', '--proxy', 'use host[:port] as proxy'
+          o.string '-X', '--request', 'HTTP method to use'
+          o.integer '--retry', 'retry request if problems occur'
+          o.boolean '-s', '--silent', "silent mode (don't print errors)"
+          o.string '-A', '--user-agent', 'send User-Agent to server'
+
+          # from httpdisk
+          o.separator 'Specific to httpdisk:'
+          o.string '--dir', 'httpdisk cache directory (defaults to ~/httpdisk)'
+          o.duration '--expires', 'when to expire cached requests (ex: 1h, 2d, 3w)'
+          o.boolean '--force', "don't read anything from cache (but still write)"
+          o.boolean '--force-errors', "don't read errors from cache (but still write)"
+          o.boolean '--status', 'show status for a url in the cache'
+
+          # generic
+          o.boolean '--version', 'show version' do
+            puts "httpdisk #{HTTPDisk::VERSION}"
+            exit
+          end
+          o.on '--help', 'show this help' do
+            puts o
+            exit
+          end
+        end
+
+        raise Slop::Error, '' if args.empty?
+        raise Slop::Error, 'no URL specified' if slop.args.empty?
+        raise Slop::Error, 'more than one URL specified' if slop.args.length > 1
+
+        slop.to_h.tap do
+          _1[:url] = slop.args.first
+        end
+      end
+    end
+  end
+end

data/lib/httpdisk/cli/main.rb

@@ -0,0 +1,169 @@
+require 'faraday-cookie_jar'
+require 'faraday_middleware'
+require 'ostruct'
+
+module HTTPDisk
+  module Cli
+    # Command line httpdisk command.
+    class Main
+      attr_reader :options
+
+      def initialize(options)
+        @options = options
+      end
+
+      # Make the request (or print status)
+      def run
+        # short circuit --status
+        if options[:status]
+          status
+          return
+        end
+
+        # create Faraday client
+        faraday = create_faraday
+
+        # run request
+        response = faraday.run_request(request_method, request_url, request_body, request_headers)
+        if response.status >= 400
+          raise CliError, "the requested URL returned error: #{response.status} #{response.reason_phrase}"
+        end
+
+        # output
+        if options[:output]
+          File.open(options[:output], 'w') { output(response, _1) }
+        else
+          output(response, $stdout)
+        end
+      end
+
+      def create_faraday
+        Faraday.new do
+          # connection settings
+          _1.proxy = options[:proxy] if options[:proxy]
+          _1.options.timeout = options[:max_time] if options[:max_time]
+
+          # cookie middleware
+          _1.use :cookie_jar
+
+          # BEFORE httpdisk so each redirect segment is cached
+          _1.response :follow_redirects
+
+          # httpdisk
+          _1.use :httpdisk, client_options
+
+          # AFTER httpdisk so transient failures are not cached
+          if options[:retry]
+            # we have a very liberal retry policy
+            retry_options = {
+              max: options[:retry],
+              methods: %w[delete get head options patch post put trace],
+              retry_statuses: (500..600).to_a,
+              retry_if: ->(_env, _err) { true },
+            }
+            _1.request :retry, retry_options
+          end
+        end
+      end
+
+      # Support for --status
+      def status
+        # build env
+        env = Faraday::Env.new.tap do
+          _1.method = request_method
+          _1.request_body = request_body
+          _1.request_headers = request_headers
+          # Run the url through Faraday to make sure we see the same stuff as middleware.
+          _1.url = Faraday.new.build_url(request_url)
+        end
+
+        # now print status
+        client = HTTPDisk::Client.new(nil, client_options)
+        client.status(env).each do
+          puts "#{_1}: #{_2.inspect}"
+        end
+      end
+
+      # Output response to f
+      def output(response, f)
+        if options[:include]
+          f.puts "HTTPDISK #{response.status} #{response.reason_phrase}"
+          response.headers.each { f.puts("#{_1}: #{_2}") }
+          f.puts
+        end
+        f.write(response.body)
+      end
+
+      #
+      # request_XXX
+      #
+
+      # HTTP method (get, post, etc.)
+      def request_method
+        method = if options[:request]
+          options[:request]
+        elsif options[:data]
+          'post'
+        end
+        method ||= 'get'
+        method = method.downcase.to_sym
+
+        if !Faraday::Connection::METHODS.include?(method)
+          raise CliError, "invalid --request #{method.inspect}"
+        end
+
+        method
+      end
+
+      # Request url
+      def request_url
+        url = options[:url]
+        # recover from missing http:
+        if url !~ %r{^https?://}i
+          if url =~ %r{^\w+://}
+            raise CliError, 'only http/https supported'
+          end
+
+          url = "http://#{url}"
+        end
+        URI.parse(url)
+      rescue URI::InvalidURIError
+        raise CliError, "invalid url #{url.inspect}"
+      end
+
+      # Request body
+      def request_body
+        options[:data]
+      end
+
+      # Request headers
+      def request_headers
+        {}.tap do |headers|
+          if options[:user_agent]
+            headers['User-Agent'] = options[:user_agent]
+          end
+
+          options[:header].each do |header|
+            key, value = header.split(': ', 2)
+            if !key || !value || key.empty? || value.empty?
+              raise CliError, "invalid --header #{header.inspect}"
+            end
+
+            headers[key] = value
+          end
+        end
+      end
+
+      #
+      # helpers
+      #
+
+      # Options to HTTPDisk::Client
+      def client_options
+        client_options = options.slice(:dir, :expires, :force, :force_errors)
+        client_options[:utf8] = true
+        client_options
+      end
+    end
+  end
+end

data/lib/httpdisk/client.rb

@@ -1,34 +1,43 @@
+require 'content-type'
 require 'faraday'
+require 'logger'
 
 module HTTPDisk
-  OPTIONS = {
-    dir: File.join(ENV['HOME'], 'httpdisk'),
-    expires_in: nil,
-    force: false,
-    force_errors: false,
-  }.freeze
-
   # Middleware and main entry point.
   class Client < Faraday::Middleware
     attr_reader :cache, :options
 
     def initialize(app, options = {})
-      super(app, options = OPTIONS.merge(options.compact))
+      options = Sloptions.parse(options) do
+        _1.string :dir, default: File.join(ENV['HOME'], 'httpdisk')
+        _1.integer :expires
+        _1.boolean :force
+        _1.boolean :force_errors
+        _1.array :ignore_params, default: []
+        _1.on :logger, type: [:boolean, Logger]
+        _1.boolean :utf8
+      end
+
+      super(app, options)
       @cache = Cache.new(options)
     end
 
     def call(env)
-      cache_key = CacheKey.new(env)
-
-      # hit?
-      if cached_response = read(cache_key, env)
-        return cached_response
-      end
-
-      # miss
-      perform(cache_key, env).tap do |response|
+      cache_key = CacheKey.new(env, ignore_params: ignore_params)
+      logger&.info("#{env.method.upcase} #{env.url} (#{cache.status(cache_key)})")
+      env[:httpdisk_diskpath] = cache.diskpath(cache_key)
+
+      # check cache, fallback to network
+      if response = read(cache_key, env)
+        response.env[:httpdisk] = true
+      else
+        response = perform(env)
+        response.env[:httpdisk] = false
         write(cache_key, env, response)
       end
+
+      encode_body(response)
+      response
     end
 
     # Returns cache status for this request
@@ -46,7 +55,7 @@ module HTTPDisk
     protected
 
     # perform the request, return Faraday::Response
-    def perform(cache_key, env)
+    def perform(env)
       app.call(env)
     rescue Faraday::ConnectionFailed, Faraday::SSLError, Faraday::TimeoutError => e
       # try to avoid caching proxy errors
@@ -95,6 +104,73 @@ module HTTPDisk
 
       err.to_s =~ /#{proxy.host}.*#{proxy.port}/
     end
+
+    # Set string encoding for response body. The cache always returns
+    # ASCII-8BIT, but we have no idea what the encoding will be from the
+    # network. Not all adapters honor Content-Type (including the default
+    # adapter).
+    def encode_body(response)
+      body = response.body || ''
+
+      # parse Content-Type
+      begin
+        content_type = response['Content-Type'] && ContentType.parse(response['Content-Type'])
+      rescue Parslet::ParseFailed
+        # unparsable
+      end
+
+      # look at charset and set body encoding if necessary
+      encoding = encoding_for(content_type)
+      if body.encoding != encoding
+        body = body.dup if body.frozen?
+        body.force_encoding(encoding)
+      end
+
+      # if :utf8, force body to UTF-8
+      if options[:utf8] && content_type && response_text?(content_type)
+        body = body.dup if body.frozen?
+        begin
+          body.encode!('UTF-8', invalid: :replace, undef: :replace, replace: '?')
+        rescue Encoding::ConverterNotFoundError
+          # rare, can't do anything here
+          body = "httpdisk could not convert from #{body.encoding.name} to UTF-8"
+        end
+      end
+
+      response.env[:body] = body
+    end
+
+    def encoding_for(content_type)
+      if content_type&.charset
+        begin
+          return Encoding.find(content_type.charset)
+        rescue ArgumentError
+          # unknown charset
+        end
+      end
+      Encoding::ASCII_8BIT
+    end
+
+    def response_text?(content_type)
+      content_type.type == 'text' || content_type.mime_type == 'application/json'
+    end
+
+    #
+    # options
+    #
+
+    def ignore_params
+      @ignore_params ||= options[:ignore_params].map { CGI.escape(_1.to_s) }.to_set
+    end
+
+    def logger
+      return if !options[:logger]
+
+      @logger ||= case options[:logger]
+      when true then Logger.new($stderr)
+      when Logger then options[:logger]
+      end
+    end
   end
 end
 

data/lib/httpdisk/error.rb

@@ -1,3 +1,7 @@
 module HTTPDisk
+  ERROR_STATUS = 999
+
   class CliError < StandardError; end
+
+  class InvalidUrl < StandardError; end
 end

data/lib/httpdisk/grep/args.rb

@@ -0,0 +1,35 @@
+# manually load dependencies here since this is loaded standalone by bin
+require 'httpdisk/version'
+require 'slop'
+
+module HTTPDisk
+  module Grep
+    module Args
+      # Slop parsing. This is broken out so we can run without require 'httpdisk'.
+      def self.slop(args)
+        slop = Slop.parse(args) do |o|
+          o.banner = 'httpdisk-grep [options] pattern [path ...]'
+          o.boolean '-c', '--count', 'suppress normal output and show count'
+          o.boolean '-h', '--head', 'show req headers before each match'
+          o.boolean '-s', '--silent', 'do not print anything to stdout'
+          o.boolean '--version', 'show version' do
+            puts "httpdisk-grep #{HTTPDisk::VERSION}"
+            exit
+          end
+          o.on '--help', 'show this help' do
+            puts o
+            exit
+          end
+        end
+
+        raise Slop::Error, '' if args.empty?
+        raise Slop::Error, 'no PATTERN specified' if slop.args.empty?
+
+        slop.to_h.tap do
+          _1[:pattern] = slop.args.shift
+          _1[:roots] = slop.args
+        end
+      end
+    end
+  end
+end