httpdisk 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 0c47ec4fda68047f57e8348746cf7e08467151a4b3d193cce16652980b5b8a47
+  data.tar.gz: 158e71dc98ba8a954eb3e744140a6e1f468de06e54a482690e7347d4b8750153
+SHA512:
+  metadata.gz: 4e261b58f3c1246dec8ab9ab5732c06c595af86b27c14866045fb9992983b1f8c89d93f788272c2331ded4f5424e424ebdf5ae0bdc886e95537e5696ad2ea04b
+  data.tar.gz: 9947c8fd27c4f7dbb98b9eb9c3aca086d7c0041ec51353b8163371d89b98c745592f764cfb05074daff0c8e9a6e9500863bf16d5f200b54e92e61ccc9006eaca

data/.github/workflows/test.yml

@@ -0,0 +1,26 @@
+name: test
+
+on:
+  push:
+    paths-ignore:
+      - '**.md'
+  pull_request:
+    paths-ignore:
+      - '**.md'
+  workflow_dispatch:
+
+jobs:
+  test:
+    strategy:
+      max-parallel: 3
+      matrix:
+        os: [ubuntu, macos]
+        ruby-version: [head, 3.0, 2.7]
+    runs-on: ${{ matrix.os }}-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: ruby/setup-ruby@v1
+        with:
+          ruby-version: ${{ matrix.ruby-version }}
+      - run: bundle install
+      - run: bundle exec rake test

data/.gitignore

@@ -0,0 +1,3 @@
+.ruby-version
+.vscode
+*.gem

data/Gemfile

@@ -0,0 +1,10 @@
+source 'https://rubygems.org'
+gemspec
+
+group :development, :test do
+  gem 'minitest'
+  gem 'mocha'
+  gem 'pry'
+  gem 'rake'
+  gem 'webmock'
+end

data/Gemfile.lock

@@ -0,0 +1,69 @@
+PATH
+  remote: .
+  specs:
+    httpdisk (0.1.0)
+      faraday (~> 1.4)
+      faraday-cookie_jar (~> 0.0)
+      faraday_middleware (~> 1.0)
+      slop (~> 4.8)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.7.0)
+      public_suffix (>= 2.0.2, < 5.0)
+    coderay (1.1.3)
+    crack (0.4.5)
+      rexml
+    domain_name (0.5.20190701)
+      unf (>= 0.0.5, < 1.0.0)
+    faraday (1.4.1)
+      faraday-excon (~> 1.1)
+      faraday-net_http (~> 1.0)
+      faraday-net_http_persistent (~> 1.1)
+      multipart-post (>= 1.2, < 3)
+      ruby2_keywords (>= 0.0.4)
+    faraday-cookie_jar (0.0.7)
+      faraday (>= 0.8.0)
+      http-cookie (~> 1.0.0)
+    faraday-excon (1.1.0)
+    faraday-net_http (1.0.1)
+    faraday-net_http_persistent (1.1.0)
+    faraday_middleware (1.0.0)
+      faraday (~> 1.0)
+    hashdiff (1.0.1)
+    http-cookie (1.0.3)
+      domain_name (~> 0.5)
+    method_source (1.0.0)
+    minitest (5.14.4)
+    mocha (1.11.2)
+    multipart-post (2.1.1)
+    pry (0.13.1)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    public_suffix (4.0.6)
+    rake (13.0.3)
+    rexml (3.2.5)
+    ruby2_keywords (0.0.4)
+    slop (4.8.2)
+    unf (0.1.4)
+      unf_ext
+    unf_ext (0.0.7.7)
+    webmock (3.12.2)
+      addressable (>= 2.3.6)
+      crack (>= 0.3.2)
+      hashdiff (>= 0.4.0, < 2.0.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  httpdisk!
+  minitest
+  mocha
+  pry
+  rake
+  webmock
+
+BUNDLED WITH
+   2.1.4

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2021 gurgeous
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,179 @@
+[![Build Status](https://github.com/gurgeous/httpdisk/workflows/test/badge.svg?branch=main)](https://github.com/gurgeous/httpdisk/actions)
+
+![logo](logo.svg)
+
+# httpdisk
+
+httpdisk is an aggressive disk cache built on top of [Faraday](https://lostisland.github.io/faraday/). It's primarily used for crawling, and will aggressively cache all requests including POSTs and transient errors.
+
+## Installation
+
+```sh
+# install gem
+$ gem install httpdisk
+
+# or add to your Gemfile
+gem 'httpdisk'
+```
+
+## Quick Start
+
+```ruby
+require 'httpdisk'
+
+# create a new Faraday client
+faraday = Faraday.new do
+  _1.use :httpdisk
+end
+
+response = faraday.get('https://google.com') # read from network
+response = faraday.get('https://google.com') # read from ~/httpdisk/google.com/...
+```
+
+httpdisk includes a handy command that works like `curl`:
+
+```sh
+# cache miss, read from network
+$ httpdisk google.com
+
+# cache hit, read from ~/httpdisk/google.com/...
+$ httpdisk google.com
+
+# supports many curl flags
+$ httpdisk -A test-agent --proxy localhost:8080 --output tmp.html twitter.com
+```
+
+## Faraday & httpdisk
+
+[Faraday](https://lostisland.github.io/faraday/) is a popular Ruby HTTP client. Faraday uses a stack of middleware to process each request, similar to the way Rack works deep inside Rails or Sinatra. httpdisk is Faraday middleware - it processes requests to look for cached responses on disk. Faraday's [usage page](https://lostisland.github.io/faraday/usage/) is a good place to learn more about Faraday.
+
+The simplest possible setup for httpdisk looks like this:
+
+```ruby
+faraday = Faraday.new do
+  _1.use :httpdisk
+end
+faraday.get(...)
+```
+
+For serious crawling, you probably want a more robust middleware stack:
+
+```ruby
+faraday = Faraday.new do
+  _1.options.timeout = 10 # lower the timeout
+  _1.use :cookie_jar # cookie support
+  _1.request :url_encoded # auto-encode form bodies
+  _1.response :json # auto-decode JSON responses
+  _1.response :follow_redirects # follow redirects (should be above httpdisk)
+  _1.use :httpdisk
+  _1.request :retry # retry failed responses (should be below httpdisk)
+end
+faraday.get(...)
+```
+
+You may want to experiment with the options for [:retry](https://lostisland.github.io/faraday/middleware/retry), to retry a
+broader set of transient errors. See [examples.rb](https://github.com/gurgeous/httpdisk/blob/main/examples.rb) for more ideas.
+
+## Disk Cache
+
+httpdisk calculates a canonical cache key for each request. The key consists of the http method, url, sorted query, and sorted body if possible. We use md5(key) as the path for each file in the cache. Try `httpdisk --status` to see it in action:
+
+```sh
+$ httpdisk --status "google.com?q=ruby"
+url: "http://google.com/?q=ruby"
+status: "miss"
+key: "GET http://google.com?q=ruby"
+digest: "0e37f96800a55958fa6029283c78f672"
+path: "httpdisk/google.com/0e3/7f96800a55958fa6029283c78f672"
+```
+
+EVERY response will be cached on disk, including POSTs. By default, the cache will be placed at `~/httpdisk` and cached responses never expire. Some examples:
+
+```ruby
+faraday.get("http://www.google.com", nil, { "User-Agent": "test-agent" })
+faraday.get("http://www.google.com", { "q": "ruby" })
+faraday.post("http://httpbin.org/post", "name=hello")
+```
+
+This will populate the cache:
+
+```sh
+$ cd ~/httpdisk
+$ find . -type f
+./google.com/5eb/fc70198242876f5e83a67253663e9
+./google.com/6d0/52ac9a33d25065fc9f405100f3741
+./httpbin.org/88f/7b2bc35cc3759c9905c4de1dbf981
+
+$ gzcat google.com/5eb/fc70198242876f5e83a67253663e9
+# GET http://www.google.com
+HTTPDISK 200 OK
+date: Mon, 19 Apr 2021 18:40:01 GMT
+expires: -1
+cache-control: private, max-age=0
+...
+```
+
+## Aggressive Caching
+
+httpdisk caches all responses. POST responses are cached, along with 500 responses and other HTTP errors. HTTP response headers that typically control caching are completely ignored. We also cache many exceptions like connection refused, timeout, ssl error, etc. These are returned as responses with HTTP status code 999.
+
+In general, if you make a request it will be cached regardless of the outcome.
+
+## Configuration
+
+httpdisk supports a few options:
+
+- `dir:` location for disk cache, defaults to `~/httpdisk`
+- `expires_in:` when to expire cached requests, default is nil (never expire)
+- `force:` don't read anything from cache (but still write)
+- `force_errors:` don't read errors from cache (but still write)
+
+Pass these in when setting up Faraday:
+
+```ruby
+faraday = Faraday.new do
+  _1.use :httpdisk, expires_in: 7*24*60*60, force: true
+end
+```
+
+## Command Line
+
+The `httpdisk` command works like `curl` and supports some of curl's popular flags. Exit code 1 indicates an HTTP response code >= 400 or a failed request.
+
+```
+$ httpdisk --help
+httpdisk [options] [url]
+Similar to curl:
+    -d, --data        HTTP POST data
+    -H, --header      pass custom header(s) to server
+    -i, --include     include response headers in the output
+    -m, --max-time    maximum time allowed for the transfer
+    -o, --output      write to file instead of stdout
+    -x, --proxy       use host[:port] as proxy
+    -X, --request     HTTP method to use
+    --retry           retry request if problems occur
+    -s, --silent      silent mode (don't print errors)
+    -A, --user-agent  send User-Agent to server
+Specific to httpdisk:
+    --dir             httpdisk cache directory (defaults to ~/httpdisk)
+    --expires         when to expire cached requests (ex: 1h, 2d, 3w)
+    --force           don't read anything from cache (but still write)
+    --force-errors    don't read errors from cache (but still write)
+    --status          show status for a url in the cache
+    --version         show version
+    --help            show this help
+```
+
+## Limitations & Gotchas
+
+- Transient errors are cached. This is appropriate for many uses cases (like crawling) but can be confusing. Use `httpdisk --status` to debug.
+- There are no builtin mechanisms to cleanup or limit the size of the cache. Use `rm`
+- For best results the `:follow_redirects` middleware should be listed _above_ httpdisk. That way each redirect request will be cached.
+- For best results the `:retry` middleware should be listed _below_ httpdisk. That way retries will complete before we cache.
+- httpdisk does not work with Faraday's parallel mode or `on_complete`.
+
+## Changelog
+
+#### 0.1 - April 2020
+
+- Original release

data/Rakefile

@@ -0,0 +1,47 @@
+require 'bundler/setup'
+require 'rake/testtask'
+
+# load the spec, we use it below
+spec = Gem::Specification.load('httpdisk.gemspec')
+
+#
+# testing
+# don't forget about TESTOPTS="--verbose" rake
+#
+
+# test (default)
+Rake::TestTask.new { _1.libs << 'test' }
+task default: :test
+
+# Watch files, run tests whenever something changes
+task :watch do
+  system('find . | entr -c rake test')
+end
+
+#
+# pry
+#
+
+task :pry do
+  system 'pry -I lib -r httpdisk.rb'
+end
+
+#
+# gem
+#
+
+task :build do
+  system('gem build --quiet httpdisk.gemspec', exception: true)
+end
+
+task install: :build do
+  system("gem install --quiet httpdisk-#{spec.version}.gem", exception: true)
+end
+
+task release: :build do
+  raise "looks like git isn't clean" unless `git status --porcelain`.empty?
+
+  system("git tag -a #{spec.version} -m 'Tagging #{spec.version}'", exception: true)
+  system('git push --tags', exception: true)
+  system("gem push httpdisk-#{spec.version}.gem", exception: true)
+end

data/bin/httpdisk

@@ -0,0 +1,41 @@
+#!/usr/bin/env ruby
+
+#
+# Main bin. Most of the interesting stuff is in HTTPDisk, for ease of testing.
+#
+
+$LOAD_PATH.unshift(File.join(__dir__, '../lib'))
+
+def puts_error(s, exit: false)
+  $stderr.puts "httpdisk: #{s}"
+end
+
+#
+# Load the bare minimum and parse args with slop. We do this separately for speed.
+#
+
+require 'httpdisk/cli_slop'
+begin
+  slop = HTTPDisk::CliSlop.slop(ARGV)
+rescue Slop::Error => e
+  puts_error(e) if e.message != ''
+  puts_error("try 'httpdisk --help' for more information")
+  exit 1
+end
+
+#
+# now load everything and run
+#
+
+require 'httpdisk'
+cli = HTTPDisk::Cli.new(slop)
+begin
+  cli.run
+rescue StandardError => e
+  puts_error(e) if !cli.options[:silent]
+  if ENV['HTTPDISK_DEBUG']
+    $stderr.puts
+    $stderr.puts e.backtrace.join("\n")
+  end
+  exit 1
+end

data/examples.rb

@@ -0,0 +1,117 @@
+#!/usr/bin/env ruby
+
+$LOAD_PATH.unshift(File.join(__dir__, 'lib'))
+
+require 'httpdisk'
+require 'json'
+
+class Examples
+  #
+  # Very simple example. The only middleware is httpdisk.
+  #
+
+  def simple
+    faraday = Faraday.new do
+      _1.use :httpdisk, force: true
+    end
+
+    faraday.get('http://www.google.com', nil, { "User-Agent": 'test-agent' })
+    faraday.get('http://www.google.com', { q: 'ruby' })
+    faraday.post('http://httpbin.org/post', 'name=hello')
+    exit
+
+    3.times { puts }
+    response = faraday.get('http://httpbingo.org/get')
+    puts response.env.url
+    puts JSON.pretty_generate(JSON.parse(response.body))
+  end
+
+  #
+  # Complete Faraday stack with cookies, redirects, retries, form encoding &
+  # JSON response parsing.
+  #
+
+  def better
+    faraday = Faraday.new do
+      # options
+      _1.headers['User-Agent'] = 'HTTPDisk'
+      _1.params.update(hello: 'world')
+      _1.options.timeout = 10
+
+      # middleware
+      _1.use :cookie_jar
+      _1.request :url_encoded
+      _1.response :json
+      _1.response :follow_redirects # must come before httpdisk
+
+      # httpdisk
+      _1.use :httpdisk
+
+      # retries (must come after httpdisk)
+      retry_options = {
+        methods: %w[delete get head options patch post put trace],
+        retry_statuses: (400..600).to_a,
+        retry_if: ->(_env, _err) { true },
+      }.freeze
+      _1.request :retry, retry_options
+    end
+
+    # get w/ params
+    3.times { puts }
+    response = faraday.get('http://httpbingo.org/get', { q: 'query' })
+    puts response.env.url
+    puts JSON.pretty_generate(response.body)
+
+    # post w/ encoded form body
+    3.times { puts }
+    response = faraday.post('http://httpbingo.org/post', 'a=1&b=2')
+    puts response.env.url
+    puts JSON.pretty_generate(response.body)
+
+    # post w/ auto-encoded form hash
+    3.times { puts }
+    response = faraday.post('http://httpbingo.org/post', { input: 'body' })
+    puts response.env.url
+    puts JSON.pretty_generate(response.body)
+  end
+
+  #
+  # Complete Faraday stack with cookies, redirects, retries, JSON encoding &
+  # JSON response parsing.
+  #
+
+  def json
+    faraday = Faraday.new do
+      # options
+      _1.headers['User-Agent'] = 'HTTPDisk'
+      _1.params.update(hello: 'world')
+      _1.options.timeout = 10
+
+      # middleware
+      _1.use :cookie_jar
+      _1.request :json
+      _1.response :json
+      _1.response :follow_redirects # must come before httpdisk
+
+      # httpdisk
+      _1.use :httpdisk
+
+      # retries (must come after httpdisk)
+      retry_options = {
+        methods: %w[delete get head options patch post put trace],
+        retry_statuses: (400..600).to_a,
+        retry_if: ->(_env, _err) { true },
+      }.freeze
+      _1.request :retry, retry_options
+    end
+
+    3.times { puts }
+    response = faraday.post('http://httpbingo.org/post', { this_is: [ 'json' ] })
+    puts response.env.url
+    puts JSON.pretty_generate(response.body)
+  end
+end
+
+Examples.new.simple
+Examples.new.better
+Examples.new.json