chuckle 1.0.0

data/.gitignore

@@ -0,0 +1,6 @@
+*.gem
+.bundle
+.rake-complete-cache
+Gemfile.lock
+pkg/*
+rdoc

data/Gemfile

@@ -0,0 +1,2 @@
+source "http://rubygems.org"
+gemspec

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2013 Adam Doppelt
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,64 @@
+# Welcome to chuckle
+
+Chuckle is an http client with a disk-based cache. HTTP responses are cached on disk in a way that makes them easy to find and debug. The cache can be shared between machines. Features:
+
+* Disk caching, with expiration
+* Cookies (also cached)
+* Robust support for timeouts and retries (also cached)
+
+This gem is an extraction from [Dwellable](http://dwellable.com). We use it for tasks such as:
+
+* Crawling content from partner web sites, including sites that require cookies
+* Downloading photo sets from partner web sites
+* Broken link detection
+* Wrapping slow or rate-limited APIs with caching
+* Sharing cached http responses between developers or machines
+
+## Install
+
+```ruby
+gem install chuckle
+```
+
+## Example Usage
+
+```ruby
+require "chuckle"
+
+client = Chuckle::Client.new
+
+# This makes a network request. The response will be cached in
+# ~/.chuckle/www.google.com/_root_
+#
+# => Chuckle::Response http://www.google.com code=200
+p client.get("http://www.google.com")
+
+# Since the response is now cached, no network is required here.
+#
+# => Chuckle::Response http://www.google.com code=200
+p client.get("http://www.google.com")
+```
+
+## Options
+
+Pass these `Chuckle::Client.new`:
+
+* **cache_dir** (~/.chuckle) - where chuckle should cache files. If HOME doesn't exist or isn't writable, it'll use `/tmp/chuckle` instead
+* **cache_errors** (true) - false to not cache errors on disk (timeouts, http status >= 400, etc.)
+* **cookies** (false) - true to turn on cookie support
+* **expires_in** (:never) - time in seconds after which cache files should expire, or `:never` to never expire
+* **nretries** (2) - number of times to retry a failing request
+* **rate_limit** (1) - number of seconds between requests
+* **timeout** (30) - timeout per request. Note that if `nretries` is 2 and `timeout` is 30, a failing request could take 90 seconds to truly fail.
+* **user_agent** - the user agent. Defaults to the IE9 user agent.
+* **verbose** (false) - if true, prints each request before fetching. Only prints network requests.
+
+
+
+
+## Limitations
+
+* Only supports GET and POST.
+* Cookies aren't intended to be cached, but we do it anyway. This works fine for our use case, where we crawl a site all at once and then clear the cache before re-crawling. Also, cookies are cached on a per-host basis so subdomains and wildcard cookies will be a problem. Use caution!
+* The cache naming scheme isn't perfect. It's theoretically possible for two URLs to map to the same cache filename, though we haven't seen this happen in the wild.
+* Chuckle shells out to [curl](http://curl.haxx.se/) for each request. Curl is rock solid and has great support for cookies, timeouts, compression, retries, etc. That makes Chuckle slower than other http clients, though network speed and rate limiting dwarf all such considerations.

data/Rakefile

@@ -0,0 +1,42 @@
+require "bundler"
+require "bundler/setup"
+require "rake/testtask"
+require "rdoc/task"
+
+#
+# gem
+#
+
+task gem: :build
+task :build do
+  system "gem build --quiet chuckle.gemspec"
+end
+
+task install: :build do
+  system "sudo gem install --quiet chuckle-#{Chuckle::VERSION}.gem"
+end
+
+task release: :build do
+  system "git tag -a #{Chuckle::VERSION} -m 'Tagging #{Chuckle::VERSION}'"
+  system "git push --tags"
+  system "gem push chuckle-#{Chuckle::VERSION}.gem"
+end
+
+#
+# test
+#
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << "test"
+end
+task default: :test
+
+#
+# rdoc
+#
+
+RDoc::Task.new do |rdoc|
+  rdoc.rdoc_dir = "rdoc"
+  rdoc.title = "chuckle #{Chuckle::VERSION}"
+  rdoc.rdoc_files.include("lib/**/*.rb")
+end

data/chuckle.gemspec

@@ -0,0 +1,28 @@
+$LOAD_PATH << File.expand_path("../lib", __FILE__)
+
+require "chuckle/version"
+
+Gem::Specification.new do |s|
+  s.name        = "chuckle"
+  s.version     = Chuckle::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.required_ruby_version = ">= 1.9.0"
+  s.authors     = ["Adam Doppelt"]
+  s.email       = ["amd@gurge.com"]
+  s.homepage    = "http://github.com/gurgeous/chuckle"
+  s.summary     = "Chuckle - caching http client that wraps curl."
+  s.description = "A caching http client that wraps curl."
+
+  s.rubyforge_project = "chuckle"
+
+  s.add_development_dependency "awesome_print"
+  s.add_development_dependency "json"
+  s.add_development_dependency "minitest", "~> 5.0"
+  s.add_development_dependency "rake"
+  s.add_development_dependency "rdoc"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- test/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map { |i| File.basename(i) }
+  s.require_paths = ["lib"]
+end

data/lib/chuckle.rb

@@ -0,0 +1,12 @@
+require "chuckle/cache"
+require "chuckle/cookie_jar"
+require "chuckle/error"
+require "chuckle/options"
+require "chuckle/request"
+require "chuckle/response"
+require "chuckle/util"
+require "chuckle/version"
+
+# these last
+require "chuckle/curl"
+require "chuckle/client"

data/lib/chuckle/cache.rb

@@ -0,0 +1,89 @@
+require "fileutils"
+
+module Chuckle
+  class Cache
+    attr_accessor :hits, :misses
+
+    def initialize(client)
+      @client = client
+
+      self.hits = self.misses = 0
+    end
+
+    def get(request)
+      if !exists?(request) || expired?(request)
+        self.misses += 1
+        return
+      end
+      self.hits += 1
+      Response.new(request)
+    end
+
+    def set(request, curl)
+      %w(body headers).each do |i|
+        src, dst = curl.send("#{i}_path"), request.send("#{i}_path")
+        FileUtils.mkdir_p(File.dirname(dst))
+        FileUtils.mv(src, dst)
+      end
+      Response.new(request)
+    end
+
+    def clear(request)
+      Util.rm_if_necessary(request.headers_path)
+      Util.rm_if_necessary(request.body_path)
+    end
+
+    def exists?(request)
+      File.exists?(request.body_path)
+    end
+
+    def expired?(request)
+      return false if @client.expires_in == :never
+      return false if !exists?(request)
+      if File.stat(request.body_path).mtime + @client.expires_in < Time.now
+        clear(request)
+        true
+      end
+    end
+
+    def body_path(request)
+      uri = request.uri
+
+      # calculate body_path
+      s = @client.cache_dir
+      s = "#{s}/#{pathify(uri.host)}"
+      s = "#{s}/#{pathify(uri.path)}"
+      if uri.query
+        q = "?#{uri.query}"
+        s = "#{s}#{pathify(q)}"
+      end
+      if body = request.body
+        s = "#{s},#{pathify(body)}"
+      end
+
+      # shorten long paths to md5 checksum
+      if s.length > 250
+        dir, base = File.dirname(s), File.basename(s)
+        s = "#{dir}/#{Util.md5(base)}"
+      end
+
+      s
+    end
+
+    protected
+
+    # turn s into a string that can be a path
+    def pathify(s)
+      s = s.gsub(/^\//, "")
+      s = s.gsub("..", ",")
+      s = s.gsub(/[?\/&]/, ",")
+      s = s.gsub(/[^A-Za-z0-9_.,=%-]/) do |i|
+        hex = i.unpack("H2").first
+        "%#{hex}"
+      end
+      s = s.downcase
+      s = "_root_" if s.empty?
+      s
+    end
+  end
+end

data/lib/chuckle/client.rb

@@ -0,0 +1,89 @@
+require "uri"
+
+module Chuckle
+  class Client
+    include Chuckle::Options
+
+    attr_accessor :options, :cache
+
+    def initialize(options = {})
+      self.options = DEFAULT_OPTIONS.merge(options)
+      self.cache = Cache.new(self)
+      sanity!
+    end
+
+    #
+    # main entry points
+    #
+
+    def create_request(uri, body = nil)
+      uri = URI.parse(uri.to_s) if !uri.is_a?(URI)
+      Request.new(self, uri, body)
+    end
+
+    def get(uri)
+      run(create_request(uri))
+    end
+
+    def post(uri, body)
+      body = case body
+      when Hash
+        Util.hash_to_query(body)
+      else
+        body.to_s
+      end
+      run(create_request(uri, body))
+    end
+
+    def run(request)
+      response = cache.get(request) || curl(request)
+      raise_errors(response)
+      response
+    end
+
+    def inspect #:nodoc:
+      self.class.name
+    end
+
+    protected
+
+    # make sure curl command exists
+    def sanity!
+      system("which curl > /dev/null")
+      raise "Chuckle requires curl. Please install it." if $? != 0
+    end
+
+    def curl(request)
+      vputs request.uri
+      rate_limit!(request)
+      cache.set(request, Curl.new(request))
+    end
+
+    def raise_errors(response)
+      # raise errors if necessary
+      error = if response.curl_exit_code
+        "curl exit code #{response.curl_exit_code}"
+      elsif response.code >= 400
+        "http status #{response.code}"
+      end
+      return if !error
+
+      if !cache_errors?
+        cache.clear(response.request)
+      end
+      raise Error.new("#{Error.class}, #{error}", response)
+    end
+
+    def vputs(s)
+      puts "chuckle: #{s}" if verbose?
+    end
+
+    def rate_limit!(request)
+      return if !request.uri.host
+      @last_request ||= Time.at(0)
+      sleep = (@last_request + rate_limit) - Time.now
+      sleep(sleep) if sleep > 0
+      @last_request = Time.now
+    end
+  end
+end

data/lib/chuckle/cookie_jar.rb

@@ -0,0 +1,24 @@
+module Chuckle
+  class CookieJar
+    PATH = "/_chuckle_cookies.txt"
+
+    def initialize(request)
+      @request = request
+    end
+
+    def bogus_request
+      @bogus_request ||= Request.new(@request.client, @request.uri + PATH)
+    end
+
+    def path
+      bogus_request.body_path
+    end
+
+    def preflight
+      # expire the cookie jar if necessary
+      bogus_request.client.cache.expired?(bogus_request)
+      # mkdir
+      FileUtils.mkdir_p(File.dirname(path))
+    end
+  end
+end

data/lib/chuckle/curl.rb

@@ -0,0 +1,90 @@
+require "fileutils"
+
+module Chuckle
+  class Curl
+    def initialize(request)
+      @request = request
+      run
+    end
+
+    # tmp path for response headers
+    def headers_path
+      @headers_path ||= Util.tmp_path
+    end
+
+    # tmp path for response body
+    def body_path
+      @body_path ||= Util.tmp_path
+    end
+
+    def run
+      # note: explicitly use Kernel.system to allow for mocking
+      command = command(@request)
+      Kernel.system(*command)
+
+      # capture exit code, bail on INT
+      exit_code = $?.to_i / 256
+      if exit_code != 0 && $?.termsig == Signal.list["INT"]
+        Process.kill(:INT, $$)
+      end
+
+      # create tmp files if there were errors
+      if !File.exists?(body_path)
+        FileUtils.touch(body_path)
+      end
+      if exit_code != 0
+        IO.write(headers_path, Curl.exit_code_to_headers(exit_code))
+      end
+    end
+
+    def self.exit_code_to_headers(exit_code)
+      "exit_code #{exit_code}"
+    end
+
+    def self.exit_code_from_headers(headers)
+      if exit_code = headers[/^exit_code (\d+)/, 1]
+        exit_code.to_i
+      end
+    end
+
+    protected
+
+    # the command line for this request, based on the request and the
+    # options from client
+    def command(request)
+      client = request.client
+
+      command = ["curl"]
+      command << "--silent"
+      command << "--compressed"
+
+      command += [ "--user-agent", client.user_agent]
+      command += ["--max-time", client.timeout]
+      command += ["--retry", client.nretries]
+      command += ["--location", "--max-redirs", 3]
+
+      if request.body
+        command += ["--data-binary", request.body]
+        command += ["--header", "Content-Type: application/x-www-form-urlencoded"]
+      end
+
+      if client.cookies?
+        cookie_jar.preflight
+        command += ["--cookie", cookie_jar.path]
+        command += ["--cookie-jar", cookie_jar.path]
+      end
+
+      command += ["--dump-header", headers_path]
+      command += ["--output", body_path]
+
+      command << request.uri
+
+      command = command.map(&:to_s)
+      command
+    end
+
+    def cookie_jar
+      @cookie_jar ||= CookieJar.new(@request)
+    end
+  end
+end

data/lib/chuckle/error.rb

@@ -0,0 +1,16 @@
+module Chuckle
+  class Error < StandardError
+    CURL_TIMEOUT = 28
+
+    attr_accessor :response
+
+    def initialize(msg, response)
+      super(msg)
+      self.response = response
+    end
+
+    def timeout?
+      response.curl_exit_code == CURL_TIMEOUT
+    end
+  end
+end

data/lib/chuckle/options.rb

@@ -0,0 +1,71 @@
+module Chuckle
+  module Options
+    DEFAULT_OPTIONS = {
+      cache_dir: nil,
+      cache_errors: true,
+      cookies: false,
+      expires_in: :never,
+      nretries: 2,
+      rate_limit: 1,
+      timeout: 30,
+      user_agent: "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0",
+      verbose: false,
+    }
+
+    # cache root directory
+    def cache_dir
+      @cache_dir ||= begin
+        dir = options[:cache_dir]
+        dir ||= begin
+          if home = ENV["HOME"]
+            if File.exists?(home) && File.stat(home).writable?
+              "#{home}/.chuckle"
+            end
+          end
+        end
+        dir ||= "/tmp/chuckle"
+        dir
+      end
+    end
+
+    # should errors be cached?
+    def cache_errors?
+      options[:cache_errors]
+    end
+
+    # are cookies enabled?
+    def cookies?
+      options[:cookies]
+    end
+
+    # number of seconds to cache responses and cookies, or :never
+    def expires_in
+      options[:expires_in]
+    end
+
+    # number of retries to attempt
+    def nretries
+      options[:nretries]
+    end
+
+    # number of seconds between requests
+    def rate_limit
+      options[:rate_limit]
+    end
+
+    # timeout per retry
+    def timeout
+      options[:timeout]
+    end
+
+    # user agent
+    def user_agent
+      options[:user_agent]
+    end
+
+    # verbose output?
+    def verbose?
+      options[:verbose]
+    end
+  end
+end

data/lib/chuckle/request.rb

@@ -0,0 +1,34 @@
+module Chuckle
+  class Request
+    attr_accessor :client, :uri, :body
+
+    def initialize(client, uri, body = nil)
+      self.client = client
+      self.uri = uri
+      self.body = body
+    end
+
+    def headers_path
+      @headers_path ||= begin
+        dir, base = File.dirname(body_path), File.basename(body_path)
+        "#{dir}/head/#{base}"
+      end
+    end
+
+    def body_path
+      @body_path ||= client.cache.body_path(self)
+    end
+
+    def to_s #:nodoc:
+      inspect
+    end
+
+    def inspect #:nodoc:
+      s = "#{self.class} #{uri}"
+      if body
+        s = "#{s} (#{body.length} bytes)"
+      end
+      s
+    end
+  end
+end

data/lib/chuckle/response.rb

@@ -0,0 +1,45 @@
+module Chuckle
+  class Response
+    attr_accessor :request, :curl_exit_code, :uri, :code
+
+    def initialize(request)
+      self.request = request
+      self.uri = request.uri
+      parse
+    end
+
+    def headers
+      @headers ||= File.read(request.headers_path)
+    end
+
+    def body
+      @body ||= File.read(request.body_path)
+    end
+
+    def to_s #:nodoc:
+      inspect
+    end
+
+    def inspect #:nodoc:
+      "#{self.class} #{uri} code=#{code}"
+    end
+
+    protected
+
+    def parse
+      self.curl_exit_code = Curl.exit_code_from_headers(headers)
+
+      locations = headers.scan(/^Location: ([^\r\n]+)/m).flatten
+      if !locations.empty?
+        location = locations.last
+        # some buggy servers do this. sigh.
+        location = location.gsub(" ", "%20")
+        self.uri = URI.parse(location)
+      end
+
+      codes = headers.scan(/^HTTP\/\d\.\d (\d+).*?\r\n\r\n/m).flatten
+      codes = codes.map(&:to_i)
+      self.code = codes.last
+    end
+  end
+end

data/lib/chuckle/util.rb

@@ -0,0 +1,33 @@
+require "digest/md5"
+require "tempfile"
+
+module Chuckle
+  module Util
+    extend self
+
+    def hash_to_query(hash)
+      q = hash.map do |key, value|
+        key = CGI.escape(key.to_s)
+        value = CGI.escape(value.to_s)
+        "#{key}=#{value}"
+      end
+      q.sort.join("&")
+    end
+
+    def md5(s)
+      Digest::MD5.hexdigest(s.to_s)
+    end
+
+    def rm_if_necessary(path)
+      File.unlink(path) if File.exists?(path)
+    end
+
+    def tmp_path
+      Tempfile.open("chuckle") do |f|
+        path = f.path
+        f.unlink
+        path
+      end
+    end
+  end
+end

data/lib/chuckle/version.rb

@@ -0,0 +1,4 @@
+module Chuckle
+  # Gem version
+  VERSION = "1.0.0"
+end

data/test/helper.rb

@@ -0,0 +1,93 @@
+require "awesome_print"
+require "chuckle"
+require "json"
+require "minitest/autorun"
+require "minitest/pride"
+
+module Helper
+  CACHE_DIR = "/tmp/_chuckle_tests"
+  URL = "http://chuckle"
+  QUERY = { "b" => "12", "a" => "34", "x y" => "56" }
+
+  #
+  # fake responses
+  #
+
+  HTTP_200 = <<-EOF.gsub(/(^|\n) +/, "\\1")
+    HTTP/1.1 200 OK
+
+    hello
+  EOF
+
+  HTTP_200_ALTERNATE = <<-EOF.gsub(/(^|\n) +/, "\\1")
+    HTTP/1.1 200 OK
+
+    alternate
+  EOF
+
+  HTTP_302 = <<-EOF.gsub(/(^|\n) +/, "\\1")
+    HTTP/1.1 302 FOUND
+    Location: http://one
+
+    HTTP/1.0 200 OK
+
+    hello
+  EOF
+
+  HTTP_302_2 = <<-EOF.gsub(/(^|\n) +/, "\\1")
+    HTTP/1.1 302 FOUND
+    Location: http://one
+
+    HTTP/1.1 302 FOUND
+    Location: http://two
+
+    HTTP/1.0 200 OK
+
+    hello
+  EOF
+
+  HTTP_404 = <<-EOF.gsub(/(^|\n) +/, "\\1")
+    HTTP/1.1 404 Not Found
+
+  EOF
+
+  def setup
+    # clear the cache before each test
+    FileUtils.rm_rf(CACHE_DIR)
+  end
+
+  # create a new client, with options
+  def client(options = {})
+    @client ||= begin
+      options = options.merge(cache_dir: CACHE_DIR)
+      Chuckle::Client.new(options)
+    end
+  end
+
+  # pretend to be curl by stubbing Kernel.system
+  def mcurl(response, exit_code = 0, &block)
+    # divide response into headers/body
+    sep = response.rindex("\n\n") + "\n\n".length
+    body = response[sep..-1]
+    headers = response[0, sep].gsub("\n", "\r\n")
+
+    # a lambda that pretends to be curl
+    fake_system = lambda do |*command|
+      tmp_headers = command[command.index("--dump-header") + 1]
+      tmp_body    = command[command.index("--output") + 1]
+      IO.write(tmp_headers, headers)
+      IO.write(tmp_body, body)
+      `(exit #{exit_code})`
+    end
+
+    # stub out Kernel.system
+    Kernel.stub(:system, fake_system) { yield }
+  end
+
+  def assert_if_system(&block)
+    fake_system = lambda do |*command|
+      assert false, "system called with #{command.inspect}"
+    end
+    Kernel.stub(:system, fake_system) { yield }
+  end
+end

data/test/test_cache.rb

@@ -0,0 +1,184 @@
+require "helper"
+
+class TestCache < Minitest::Test
+  include Helper
+
+  def after_setup
+    client(expires_in: 10)
+  end
+
+  # exists? and expired? (and clear)
+  def test_predicates
+    request = client.create_request(URL)
+    assert !client.cache.exists?(request), "uncache! uri said it was cached"
+
+    mcurl(HTTP_200) do
+      client.run(request)
+    end
+    assert client.cache.exists?(request), "cache said it wasn't cached"
+    assert !client.cache.expired?(request), "cache said it was expired"
+
+    client.cache.clear(request)
+    assert !client.cache.exists?(request), "still cached after clear"
+  end
+
+  # cache expiration
+  def test_expiry
+    request = client.create_request(URL)
+    response = mcurl(HTTP_200) do
+      client.run(request)
+    end
+    assert_equal "hello\n", response.body
+
+    # make it look old
+    tm = Time.now - (client.expires_in + 9999)
+    path = request.body_path
+    File.utime(tm, tm, path)
+    assert client.cache.expired?(request), "#{path} was supposed to be expired"
+
+    # make sure we get the new body
+    response = mcurl(HTTP_200_ALTERNATE) do
+      client.run(request)
+    end
+    assert_equal "alternate\n", response.body
+    assert_equal 2, client.cache.misses
+  end
+
+  #
+  # requests and errors
+  #
+
+  def test_200
+    # cache miss
+    mcurl(HTTP_200) do
+      client.get(URL)
+    end
+
+    # cache hit
+    response = assert_if_system do
+      client.get(URL)
+    end
+    assert_equal 200, response.code
+    assert_equal URI.parse(URL), response.uri
+    assert_equal "hello\n", response.body
+    assert_equal 1, client.cache.hits
+    assert_equal 1, client.cache.misses
+  end
+
+  def test_404
+    # cache miss
+    begin
+      mcurl(HTTP_404) do
+        client.get(URL)
+      end
+    rescue Chuckle::Error => e
+    end
+
+    # cache hit
+    e = assert_raises Chuckle::Error do
+      assert_if_system do
+        client.get(URL)
+      end
+    end
+    assert_equal 404, e.response.code
+    assert_equal 1, client.cache.hits
+    assert_equal 1, client.cache.misses
+  end
+
+  def test_timeout
+    # cache miss
+    begin
+      mcurl(HTTP_404, Chuckle::Error::CURL_TIMEOUT) do
+        client.get(URL)
+      end
+    rescue Chuckle::Error => e
+    end
+
+    # cache hit
+    e = assert_raises Chuckle::Error do
+      assert_if_system do
+        client.get(URL)
+      end
+    end
+    assert e.timeout?, "exception didn't indicate timeout"
+    assert_equal 1, client.cache.hits
+    assert_equal 1, client.cache.misses
+  end
+
+  def test_post
+    # cache miss
+    mcurl(HTTP_200) do
+      client.post(URL, QUERY)
+    end
+
+    # cache hit
+    response = assert_if_system do
+      client.post(URL, QUERY)
+    end
+    assert_equal 200, response.code
+    assert_equal URI.parse(URL), response.uri
+    assert_equal "hello\n", response.body
+    assert_equal 1, client.cache.hits
+    assert_equal 1, client.cache.misses
+  end
+
+  def test_long_url
+    words = %w(the quick brown fox jumped over the lazy dog)
+    query = (1..100).map { words[rand(words.length)] }
+    query = query.each_slice(2).map { |i| i.join("=") }.join("&")
+
+    # cache miss
+    mcurl(HTTP_200) do
+      client.post(URL, query)
+    end
+
+    # cache hit
+    response = assert_if_system do
+      client.post(URL, query)
+    end
+
+    # make sure it turned into a checksum
+    assert response.request.body_path =~ /[a-f0-9]{32}$/, "body_path wasn't an md5 checksum"
+    assert_equal 200, response.code
+    assert_equal URI.parse(URL), response.uri
+    assert_equal "hello\n", response.body
+    assert_equal 1, client.cache.hits
+    assert_equal 1, client.cache.misses
+  end
+
+  def test_query
+    url = "#{URL}?abc=def"
+
+    # cache miss
+    mcurl(HTTP_200) do
+      client.get(url)
+    end
+
+    # cache hit
+    response = assert_if_system do
+      client.get(url)
+    end
+
+    # make sure it turned into a checksum
+    assert response.request.body_path =~ /abc=def$/, "body_path didn't contain query"
+    assert_equal 200, response.code
+    assert_equal URI.parse(url), response.uri
+    assert_equal "hello\n", response.body
+    assert_equal 1, client.cache.hits
+    assert_equal 1, client.cache.misses
+  end
+
+  def test_nocache_errors
+    # turn off error caching
+    client = Chuckle::Client.new(cache_dir: CACHE_DIR, expires_in: 10, cache_errors: false)
+
+    # cache misses
+    2.times do
+      begin
+        mcurl(HTTP_404) { client.get(URL) }
+      rescue Chuckle::Error => e
+      end
+    end
+    assert_equal 2, client.cache.misses
+  end
+end

data/test/test_network.rb

@@ -0,0 +1,56 @@
+require "helper"
+
+# these actually use the network, and get skipped unless ENV["NETWORK"].
+class TestNetwork < Minitest::Test
+  include Helper
+
+  def after_setup
+    skip if !ENV["NETWORK"]
+  end
+
+  def test_get
+    response = client.get("http://httpbin.org/get")
+    assert_equal 200, response.code
+  end
+
+  def test_timeout
+    e = assert_raises Chuckle::Error do
+      client(nretries: 0, timeout: 2).get("http://httpbin.org/delay/3")
+    end
+    assert e.timeout?, "exception didn't indicate timeout"
+  end
+
+  def test_post
+    response = client.post("http://httpbin.org/post", QUERY)
+    assert_equal JSON.parse(response.body)["form"], QUERY
+  end
+
+  def test_cookies
+    cookies = { "a" => "b" }
+
+    client(cookies: true, expires_in: 60) # set options
+
+    request = client.create_request("http://httpbin.org/get")
+    cookie_jar = Chuckle::CookieJar.new(request).path
+
+    # make sure there are no cookies after the GET
+    client.run(request)
+    assert !File.exists?(cookie_jar), "cookie jar shouldn't exist yet"
+
+    # make sure there ARE cookies after a Set-Cookie
+    client.get("http://httpbin.org/cookies/set?#{Chuckle::Util.hash_to_query(cookies)}")
+    assert File.exists?(cookie_jar), "cookie jar SHOULD exist now"
+
+    # make sure cookies come back from the server
+    response = client.get("http://httpbin.org/cookies")
+    assert_equal JSON.parse(response.body)["cookies"], cookies
+
+    # Finally, test cache expiry on cookie_jar. Note that this has to
+    # be an un-cached URL, otherwise the cookie_jar never gets
+    # checked!
+    tm = Time.now - (client.expires_in + 9999)
+    File.utime(tm, tm, cookie_jar)
+    client.get("http://httpbin.org/robots.txt")
+    assert !File.exists?(cookie_jar), "cookie jar should've expired"
+  end
+end

data/test/test_requests.rb

@@ -0,0 +1,55 @@
+require "helper"
+
+class TestRequests < Minitest::Test
+  include Helper
+
+  def test_200
+    response = mcurl(HTTP_200) { client.get(URL) }
+    assert_equal 200, response.code
+    assert_equal URI.parse(URL), response.uri
+    assert_equal "hello\n", response.body
+  end
+
+  def test_302
+    response = mcurl(HTTP_302) { client.get(URL) }
+    assert_equal 200, response.code
+    assert_equal URI.parse("http://one"), response.uri
+    assert_equal "hello\n", response.body
+  end
+
+  def test_302_2
+    response = mcurl(HTTP_302_2) { client.get(URL) }
+    assert_equal 200, response.code
+    assert_equal URI.parse("http://two"), response.uri
+    assert_equal "hello\n", response.body
+  end
+
+  def test_404
+    e = assert_raises Chuckle::Error do
+      mcurl(HTTP_404) do
+        client.get(URL)
+      end
+    end
+    assert_equal 404, e.response.code
+  end
+
+  def test_timeout
+    e = assert_raises Chuckle::Error do
+      mcurl(HTTP_404, Chuckle::Error::CURL_TIMEOUT) do
+        client.get(URL)
+      end
+    end
+    assert e.timeout?, "exception didn't indicate timeout"
+  end
+
+  def test_post
+    # just test hash_to_query first
+    assert_equal "a=34&b=12&x+y=56", Chuckle::Util.hash_to_query(QUERY)
+
+    response = mcurl(HTTP_200) { client.post(URL, QUERY) }
+    assert_equal response.request.body, Chuckle::Util.hash_to_query(QUERY)
+    assert_equal 200, response.code
+    assert_equal URI.parse(URL), response.uri
+    assert_equal "hello\n", response.body
+  end
+end

metadata

@@ -0,0 +1,153 @@
+--- !ruby/object:Gem::Specification
+name: chuckle
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+  prerelease: 
+platform: ruby
+authors:
+- Adam Doppelt
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-05-17 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: awesome_print
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rdoc
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A caching http client that wraps curl.
+email:
+- amd@gurge.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- chuckle.gemspec
+- lib/chuckle.rb
+- lib/chuckle/cache.rb
+- lib/chuckle/client.rb
+- lib/chuckle/cookie_jar.rb
+- lib/chuckle/curl.rb
+- lib/chuckle/error.rb
+- lib/chuckle/options.rb
+- lib/chuckle/request.rb
+- lib/chuckle/response.rb
+- lib/chuckle/util.rb
+- lib/chuckle/version.rb
+- test/helper.rb
+- test/test_cache.rb
+- test/test_network.rb
+- test/test_requests.rb
+homepage: http://github.com/gurgeous/chuckle
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: 1.9.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: 2600014295267267585
+requirements: []
+rubyforge_project: chuckle
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: Chuckle - caching http client that wraps curl.
+test_files:
+- test/helper.rb
+- test/test_cache.rb
+- test/test_network.rb
+- test/test_requests.rb