httpdisk 0.1.0

data/httpdisk.gemspec

@@ -0,0 +1,28 @@
+require_relative 'lib/httpdisk/version'
+
+Gem::Specification.new do |s|
+  s.name = 'httpdisk'
+  s.version = HTTPDisk::VERSION
+  s.authors = [ 'Adam Doppelt' ]
+  s.email = 'amd@gurge.com'
+
+  s.summary = 'httpdisk - disk cache for faraday'
+  s.description = 'httpdisk works with faraday to aggressively cache responses on disk.'
+  s.homepage = 'http://github.com/gurgeous/httpdisk'
+  s.license = 'MIT'
+  s.required_ruby_version = '>= 2.7.0'
+
+  # what's in the gem?
+  s.files = Dir.chdir(File.expand_path(__dir__)) do
+    `git ls-files -z`.split("\x0").reject { _1.match(%r{^test/}) }
+  end
+  s.bindir = 'bin'
+  s.executables = s.files.grep(%r{^#{s.bindir}/}) { File.basename(_1) }
+  s.require_paths = [ 'lib' ]
+
+  # gem dependencies
+  s.add_dependency 'faraday', '~> 1.4'
+  s.add_dependency 'faraday-cookie_jar', '~> 0.0'
+  s.add_dependency 'faraday_middleware', '~> 1.0'
+  s.add_dependency 'slop', '~> 4.8'
+end

data/lib/httpdisk.rb

@@ -0,0 +1,12 @@
+require 'httpdisk/cache_key'
+require 'httpdisk/cache'
+require 'httpdisk/cli_slop'
+require 'httpdisk/cli'
+require 'httpdisk/client'
+require 'httpdisk/error'
+require 'httpdisk/payload'
+require 'httpdisk/version'
+
+module HTTPDisk
+  ERROR_STATUS = 999
+end

data/lib/httpdisk/cache.rb

@@ -0,0 +1,80 @@
+require 'fileutils'
+
+module HTTPDisk
+  # Disk cache for cache_keys => response. Files are compressed.
+  class Cache
+    attr_reader :options
+
+    def initialize(options)
+      @options = options
+
+      # heavy sanity checking on arguments here
+      if !dir.is_a?(String)
+        raise ArgumentError, "expected :dir to be a string, not #{dir.inspect}"
+      end
+      if expires_in && !expires_in.is_a?(Integer)
+        raise ArgumentError, "expected :expires_in to be an integer, not #{expires_in.inspect}"
+      end
+      %i[force force_errors].each do
+        value = send(_1)
+        if ![ nil, true, false ].include?(value)
+          raise ArgumentError, "expected #{_1} to be a boolean, not #{value.inspect}"
+        end
+      end
+    end
+
+    %i[dir expires_in force force_errors].each do |method|
+      define_method(method) do
+        options[method]
+      end
+    end
+    alias force? force
+    alias force_errors? force_errors
+
+    # Get cached response. If there is a cached error it will be raised.
+    def read(cache_key)
+      payload_or_status = read0(cache_key)
+      payload_or_status.is_a?(Symbol) ? nil : payload_or_status
+    end
+
+    # Cache status for a cache_key, %i[error force hit miss stale]
+    def status(cache_key)
+      payload_or_status = read0(cache_key)
+      return payload_or_status if payload_or_status.is_a?(Symbol)
+      payload_or_status.error_999? ? :error : :hit
+    end
+
+    # Write response to the disk cache
+    def write(cache_key, payload)
+      path = diskpath(cache_key)
+      FileUtils.mkdir_p(File.dirname(path))
+      Zlib::GzipWriter.open(path) { payload.write(_1) }
+    end
+
+    # Relative path for this cache_key based on the cache key
+    def diskpath(cache_key)
+      File.join(dir, cache_key.diskpath)
+    end
+
+    protected
+
+    # low level read, returns payload or status
+    def read0(cache_key)
+      path = diskpath(cache_key)
+
+      return :miss if !File.exist?(path)
+      return :stale if expired?(path)
+      return :force if force?
+
+      payload = Zlib::GzipReader.open(path) { Payload.read(_1) }
+      return :force if force_errors? && payload.error_999?
+
+      payload
+    end
+
+    # Is this path expired?
+    def expired?(path)
+      expires_in && File.stat(path).mtime < Time.now - expires_in
+    end
+  end
+end

data/lib/httpdisk/cache_key.rb

@@ -0,0 +1,100 @@
+require 'cgi'
+require 'digest/md5'
+require 'uri'
+
+module HTTPDisk
+  class CacheKey
+    attr_reader :env
+
+    def initialize(env)
+      @env = env
+
+      # sanity checks
+      raise 'http/https required' if env.url.scheme !~ /^https?$/
+      raise 'hostname required' if !env.url.host
+    end
+
+    def url
+      env.url
+    end
+
+    # Cache key (memoized)
+    def key
+      @key ||= calculate_key
+    end
+
+    # md5(key) (memoized)
+    def digest
+      @digest ||= Digest::MD5.hexdigest(key)
+    end
+
+    # Relative path for this cache key based on hostdir & digest.
+    def diskpath
+      @diskpath ||= File.join(hostdir, digest[0, 3], digest[3..])
+    end
+
+    def to_s
+      key
+    end
+
+    protected
+
+    # Calculate cache key for url
+    def calculate_key
+      key = []
+      key << env.method.upcase
+      key << ' '
+      key << url.scheme
+      key << '://'
+      key << url.host.downcase
+      if !default_port?
+        key << ':'
+        key << url.port
+      end
+      if url.path != '/'
+        key << url.path
+      end
+      if (q = url.query) && q != ''
+        key << '?'
+        key << querykey(q)
+      end
+      if env.request_body
+        key << ' '
+        key << bodykey
+      end
+      key.join
+    end
+
+    # Calculate cache key segment for body
+    def bodykey
+      body = env.request_body.to_s
+      case
+      when env.request_headers['Content-Type'] == 'application/x-www-form-urlencoded'
+        querykey(body)
+      when body.length < 50
+        body
+      else
+        Digest::MD5.hexdigest(body)
+      end
+    end
+
+    # Calculate canonical key for a query
+    def querykey(q)
+      q.split('&').sort.join('&')
+    end
+
+    def default_port?
+      url.default_port == url.port
+    end
+
+    # Calculate nice directory name from url.host
+    def hostdir
+      hostdir = url.host.downcase
+      hostdir = hostdir.gsub(/^www\./, '')
+      hostdir = hostdir.gsub(/[^a-z0-9._-]/, '')
+      hostdir = hostdir.squeeze('.')
+      hostdir = 'any' if hostdir.empty?
+      hostdir
+    end
+  end
+end

data/lib/httpdisk/cli.rb

@@ -0,0 +1,218 @@
+require 'faraday-cookie_jar'
+require 'faraday_middleware'
+require 'ostruct'
+
+module HTTPDisk
+  # Command line httpdisk command.
+  class Cli
+    attr_reader :options
+
+    # for --expires
+    UNITS = {
+      s: 1,
+      m: 60,
+      h: 60 * 60,
+      d: 24 * 60 * 60,
+      w: 7 * 24 * 60 * 60,
+      y: 365 * 7 * 24 * 60 * 60,
+    }.freeze
+
+    def initialize(options)
+      @options = options
+    end
+
+    # we have a very liberal retry policy
+    RETRY_OPTIONS = {
+      methods: %w[delete get head options patch post put trace],
+      retry_statuses: (400..600).to_a,
+      retry_if: ->(_env, _err) { true },
+    }.freeze
+
+    # Make the request (or print status)
+    def run
+      # short circuit --status
+      if options[:status]
+        status
+        return
+      end
+
+      # create Faraday client
+      faraday = create_faraday
+
+      # run request
+      response = faraday.run_request(request_method, request_url, request_body, request_headers)
+      if response.status >= 400
+        raise CliError, "the requested URL returned error: #{response.status} #{response.reason_phrase}"
+      end
+
+      # output
+      if options[:output]
+        File.open(options[:output], 'w') { output(response, _1) }
+      else
+        output(response, $stdout)
+      end
+    end
+
+    def create_faraday
+      Faraday.new do
+        # connection settings
+        _1.proxy = proxy if options[:proxy]
+        _1.options.timeout = options[:max_time] if options[:max_time]
+
+        # cookie middleware
+        _1.use :cookie_jar
+
+        # BEFORE httpdisk so each redirect segment is cached
+        _1.response :follow_redirects
+
+        # httpdisk
+        _1.use :httpdisk, client_options
+
+        # AFTER httpdisk so transient failures are not cached
+        if options[:retry]
+          _1.request :retry, RETRY_OPTIONS.merge(max: options[:retry])
+        end
+      end
+    end
+
+    # Support for --status
+    def status
+      # build env
+      env = Faraday::Env.new.tap do
+        _1.method = request_method
+        _1.request_body = request_body
+        _1.request_headers = request_headers
+        _1.url = request_url
+      end
+
+      # now print status
+      client = HTTPDisk::Client.new(nil, client_options)
+      client.status(env).each do
+        puts "#{_1}: #{_2.inspect}"
+      end
+    end
+
+    # Output response to f
+    def output(response, f)
+      if options[:include]
+        f.puts "HTTPDISK #{response.status} #{response.reason_phrase}"
+        response.headers.each { f.puts("#{_1}: #{_2}") }
+        f.puts
+      end
+      f.write(response.body)
+    end
+
+    #
+    # request_XXX
+    #
+
+    # HTTP method (get, post, etc.)
+    def request_method
+      method = if options[:request]
+        options[:request]
+      elsif options[:data]
+        'post'
+      end
+      method ||= 'get'
+      method = method.downcase.to_sym
+
+      if !Faraday::Connection::METHODS.include?(method)
+        raise CliError, "invalid --request #{method.inspect}"
+      end
+      method
+    end
+
+    # Request url
+    def request_url
+      url = options[:url]
+      # recover from missing http:
+      if url !~ %r{^https?://}i
+        if url =~ %r{^\w+://}
+          raise CliError, 'only http/https supported'
+        end
+        url = "http://#{url}"
+      end
+      URI.parse(url)
+    rescue URI::InvalidURIError
+      raise CliError, "invalid url #{url.inspect}"
+    end
+
+    # Request body
+    def request_body
+      options[:data]
+    end
+
+    # Request headers
+    def request_headers
+      {}.tap do |headers|
+        if options[:user_agent]
+          headers['User-Agent'] = options[:user_agent]
+        end
+
+        options[:header].each do |header|
+          key, value = header.split(': ', 2)
+          if !key || !value || key.empty? || value.empty?
+            raise CliError, "invalid --header #{header.inspect}"
+          end
+          headers[key] = value
+        end
+      end
+    end
+
+    #
+    # helpers
+    #
+
+    # Options to HTTPDisk::Client
+    def client_options
+      {}.tap do |client_options|
+        client_options[:dir] = options[:dir]
+        if options[:expires]
+          seconds = parse_expires(options[:expires])
+          if !seconds
+            raise CliError, "invalid --expires #{options[:expires].inspect}"
+          end
+          client_options[:expires_in] = seconds
+        end
+        client_options[:force] = options[:force]
+        client_options[:force_errors] = options[:force_errors]
+      end
+    end
+
+    # Return validated --proxy flag if present
+    def proxy
+      return if !options[:proxy]
+
+      proxy = parse_proxy(options[:proxy])
+      raise CliError, "--proxy should be host[:port], not #{options[:proxy].inspect}" if !proxy
+      proxy
+    end
+
+    # Parse --expires flag
+    def parse_expires(s)
+      m = s.match(/^(\d+)([smhdwy])?$/)
+      return if !m
+
+      num, unit = m[1].to_i, (m[2] || 's').to_sym
+      return if !UNITS.key?(unit)
+
+      num * UNITS[unit]
+    end
+
+    # Parse --proxy flag
+    def parse_proxy(proxy_flag)
+      host, port = proxy_flag.split(':', 2)
+      return if !host || host.empty?
+      return if port&.empty?
+
+      URI.parse('http://placeholder').tap do
+        begin
+          _1.host = host
+          _1.port = port if port
+        rescue URI::InvalidComponentError
+          return
+        end
+      end.to_s
+    end
+  end
+end

data/lib/httpdisk/cli_slop.rb

@@ -0,0 +1,54 @@
+# manually load dependencies here since this is loaded standalone by bin
+require 'httpdisk/error'
+require 'httpdisk/version'
+require 'slop'
+
+module HTTPDisk
+  # Slop parsing. This is broken out so we can run without require 'httpdisk'.
+  module CliSlop
+    def self.slop(args)
+      slop = Slop.parse(args) do |o|
+        o.banner = 'httpdisk [options] [url]'
+
+        # similar to curl
+        o.separator 'Similar to curl:'
+        o.string '-d', '--data', 'HTTP POST data'
+        o.array '-H', '--header', 'pass custom header(s) to server', delimiter: nil
+        o.boolean '-i', '--include', 'include response headers in the output'
+        o.integer '-m', '--max-time', 'maximum time allowed for the transfer'
+        o.string '-o', '--output', 'write to file instead of stdout'
+        o.string '-x', '--proxy', 'use host[:port] as proxy'
+        o.string '-X', '--request', 'HTTP method to use'
+        o.integer '--retry', 'retry request if problems occur'
+        o.boolean '-s', '--silent', "silent mode (don't print errors)"
+        o.string '-A', '--user-agent', 'send User-Agent to server'
+
+        # from httpdisk
+        o.separator 'Specific to httpdisk:'
+        o.string '--dir', 'httpdisk cache directory (defaults to ~/httpdisk)'
+        o.string '--expires', 'when to expire cached requests (ex: 1h, 2d, 3w)'
+        o.boolean '--force', "don't read anything from cache (but still write)"
+        o.boolean '--force-errors', "don't read errors from cache (but still write)"
+        o.boolean '--status', 'show status for a url in the cache'
+
+        # generic
+        o.boolean '--version', 'show version' do
+          puts "httpdisk #{HTTPDisk::VERSION}"
+          exit
+        end
+        o.on '--help', 'show this help' do
+          puts o
+          exit
+        end
+      end
+
+      raise Slop::Error, '' if args.empty?
+      raise Slop::Error, 'no URL specified' if slop.args.empty?
+      raise Slop::Error, 'more than one URL specified' if slop.args.length > 1
+
+      slop.to_h.tap do
+        _1[:url] = slop.args.first
+      end
+    end
+  end
+end