cacho 0.0.3 → 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 383e74d0ac66a216ed011020530efbc4570cce1e
+  data.tar.gz: 84c5210831578a5f131a66535fce4d53b709e203
+SHA512:
+  metadata.gz: 9a447d7cf5f0dfeefa4e011409f54cfcf5b8f4d266855903002e578fde6d50165895f443da1be3acb280f4b3e4fc220d957e861f5c2336665e0b66bd40cb7bba
+  data.tar.gz: 3b6a1da1d8299e79bc0aa2626548ca12ab847471afe914a58e2936f1fe9add41d4a8851327cf4a12c79e2bbedb44bc36566d472946570a3a18de407c469e1c37

data/.gitignore

@@ -0,0 +1 @@
+/pkg

data/README.md

@@ -0,0 +1,40 @@
+Cacho
+=====
+
+A careless caching client optimized for scraping.
+
+Description
+-----------
+
+Cacho is an HTTP client for scraping. It will do most of the things you want
+when scraping:
+
+* Follow redirects.
+* Set the `User-Agent` to a browser-like string.
+* Accept and process gzip encoding.
+* Detect when it's been rate limited and wait.
+* Retry on silly network errors.
+* Use persistent HTTP connections.
+
+Most importantly, Cacho will store responses on disk so that multiple runs of
+your script will not hit the endpoints you are scraping. This prevents being
+rate limited and also makes your script faster every time.
+
+Usage
+-----
+
+    require "cacho"
+
+    client = Cacho.new
+
+    res = client.request(:get, "https://news.ycombinator.com")
+
+Installation
+------------
+
+    $ gem install cacho
+
+License
+-------
+
+See the `UNLICENSE`.

data/UNLICENSE

@@ -0,0 +1,24 @@
+This is free and unencumbered software released into the public domain.
+
+Anyone is free to copy, modify, publish, use, compile, sell, or
+distribute this software, either in source code form or as a compiled
+binary, for any purpose, commercial or non-commercial, and by any
+means.
+
+In jurisdictions that recognize copyright laws, the author or authors
+of this software dedicate any and all copyright interest in the
+software to the public domain. We make this dedication for the benefit
+of the public at large and to the detriment of our heirs and
+successors. We intend this dedication to be an overt act of
+relinquishment in perpetuity of all present and future rights to this
+software under copyright law.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
+
+For more information, please refer to <http://unlicense.org/>

data/cacho.gemspec

@@ -1,14 +1,15 @@
+require_relative "lib/cacho"
+
 Gem::Specification.new do |s|
   s.name              = "cacho"
-  s.version           = "0.0.3"
-  s.summary           = "Cache aware, Redis based HTTP client."
-  s.description       = "HTTP client that understands cache responses and stores results in Redis."
-  s.authors           = ["Damian Janowski", "Michel Martens"]
-  s.email             = ["djanowski@dimaion.com", "michel@soveran.com"]
-  s.homepage          = "http://github.com/djanowski/cacho"
+  s.version           = Cacho::VERSION
+  s.summary           = "A careless caching client optimized for scraping."
+  s.description       = "A careless caching client optimized for scraping."
+  s.authors           = ["Damian Janowski", "Martín Sarsale"]
+  s.email             = ["djanowski@dimaion.com", "martin.sarsale@gmail.com"]
+  s.homepage          = "https://github.com/djanowski/cacho"
 
-  s.files = ["LICENSE", "Rakefile", "lib/cacho.rb", "cacho.gemspec", "test/cacho.rb"]
+  s.files = `git ls-files`.lines.to_a.map(&:chomp)
 
-  s.add_dependency "nest", "~> 1.0"
-  s.add_dependency "curb", "~> 0.7"
+  s.add_dependency "net-http-persistent"
 end

data/examples/simple.rb

@@ -0,0 +1,5 @@
+require_relative "../lib/cacho"
+
+client = Cacho.new
+
+puts client.request(:get, "https://news.ycombinator.com")

data/lib/cacho.rb

@@ -1,136 +1,203 @@
-# encoding: UTF-8
-
-require "json" unless defined?(JSON)
-require "curb"
-require "nest"
+require "net/http/persistent"
+require "uri"
+require "zlib"
+require "json"
+require "stringio"
+require "csv"
+require "fileutils"
 
 class Cacho
-  VERSION = "0.0.3"
+  VERSION = "0.1.0"
+
+  attr_accessor :hasher
 
-  def self.get(url, request_headers = {})
-    _request(:get, url, request_headers)
+  def initialize(*args)
+    @client = Client.new(*args)
+    @db = DB.new("~/.cacho/cache")
+    @hasher = -> *args { args }
   end
 
-  def self.head(url, request_headers = {})
-    _request(:head, url, request_headers)
+  def request(verb, *args)
+    if verb == :get
+      uri = @client.uri(*args)
+
+      @db.get(verb, uri) do
+        @client.request(verb, *args)
+      end
+    else
+      @client.request(verb, *args)
+    end
   end
+end
+
+class Cacho::Client
+  Error = Class.new(StandardError)
+  NotFound = Class.new(Error)
 
-  def self.options(url, request_headers = {})
-    _request(:options, url, request_headers)
+  def initialize(callbacks = {})
+    @http = Net::HTTP::Persistent.new
+    @callbacks = callbacks
+    @callbacks[:configure_http].(@http) if @callbacks[:configure_http]
   end
 
-  def self._request(verb, url, request_headers = {})
-    local = Local[verb, url]
+  def uri(url, options = {})
+    query = options.fetch(:query, {}).dup
 
-    unless local.fresh?
-      remote = Remote.request(verb, url, local.build_headers.merge(request_headers))
+    @callbacks[:process_query].(query) if @callbacks[:process_query]
 
-      local.set(remote) unless remote.first == 304
-    end
+    uri = URI(url)
 
-    local.response
+    uri.query = URI.encode_www_form(query) if query.size > 0
+
+    uri
   end
 
-  class Local
-    attr :etag
-    attr :last_modified
-    attr :expire
-    attr :response
+  def request(verb, url, options = {})
+    uri = self.uri(url, options)
 
-    def initialize(key)
-      @key = key
-      @etag, @last_modified, @expire, @response = @key.hmget(:etag, :last_modified, :expire, :response)
-      @response = JSON.parse(@response) if @response
-    end
+    loop do
+      request = Net::HTTP.const_get(verb.capitalize).new(uri.request_uri)
 
-    def self.[](verb, url)
-      new(Nest.new(verb)[url])
-    end
+      @callbacks[:before_request].(request) if @callbacks[:before_request]
 
-    def expire_in(ttl)
-      @expire = (Time.now + ttl).to_i
-    end
+      if options.include?(:headers)
+        options[:headers].each do |key, value|
+          request[key] = value
+        end
+      end
+
+      request["Accept-Encoding"] = "gzip"
+
+      if verb == :post
+        post_data = options.fetch(:data)
 
-    def build_headers
-      {}.tap do |headers|
-        headers["If-None-Match"] = etag if etag
-        headers["If-Modified-Since"] = last_modified if last_modified
+        case options[:content_type]
+        when :json
+          request["Content-Type"] = "application/json; charset=utf-8"
+          request.body = post_data.to_json
+        else
+          request.body = URI.encode_www_form(post_data)
+        end
       end
-    end
 
-    def set(response)
-      @response = response
+      if options[:content_encoding] == :deflate
+        request["Content-Encoding"] = "deflate"
 
-      return unless cacheable?
+        request.body = Zlib::Deflate.deflate(request.body)
+      end
 
-      _, headers, _ = response
+      $stderr.puts("-> #{verb.upcase} #{uri}")
 
-      if headers["Cache-Control"]
-        ttl = headers["Cache-Control"][/max\-age=(\d+)/, 1].to_i
-        expire_in(ttl)
+      if verb_idempotent?(verb)
+        res = protect { @http.request(uri, request) }
+      else
+        res = @http.request(uri, request)
       end
 
-      @etag = headers["ETag"]
-      @last_modified = headers["Last-Modified"]
+      body = res.body
 
-      store
-    end
+      if res["Content-Encoding"] == "gzip"
+        body = Zlib::GzipReader.new(StringIO.new(body)).read
+      end
 
-    def fresh?
-      expire && expire.to_i >= Time.now.to_i
+      if res["Content-Type"].start_with?("application/json")
+        parsed = JSON.parse(body)
+      else
+        parsed = body
+      end
+
+      if @callbacks[:rate_limit_detector]
+        if seconds = @callbacks[:rate_limit_detector].(res, body, parsed)
+          $stderr.puts("Rate limited for #{seconds} seconds.")
+          sleep(seconds)
+          next
+        end
+      end
+
+      case res.code
+      when "200"
+        return parsed
+      when "303" , "302" , "301"
+        if res["Location"].to_s.index(/^https?:\/\//)
+          redirect_url = URI.escape(res["Location"])
+        else
+          redirect_url = @base_url, res["Location"]
+        end
+        $stderr.write(" redirected to #{redirect_url}\n") 
+        uri = URI.join(redirect_url)
+      when "404"
+        return nil
+      else
+        raise "Got #{res.code}: #{body.inspect}"
+      end
     end
+  end
 
-  protected
+  def verb_idempotent?(verb)
+    verb == :get || verb == :head || verb == :options
+  end
 
-    def cacheable?
-      status, headers, _ = response
+  def protect(options = {})
+    throttle = options.fetch(:throttle, 1)
+    maximum_retries = options.fetch(:retries, nil)
+    retries = 0
 
-      status == 200 && (headers["Cache-Control"] || headers["ETag"] || headers["Last-Modified"])
-    end
+    begin
+      result = yield
 
-    def store
-      fields = {response: response.to_json}
+      retries = 0
 
-      fields[:etag] = etag if etag
-      fields[:last_modified] = last_modified if last_modified
-      fields[:expire] = expire if expire
+      return result
+    rescue SocketError, \
+           EOFError, \
+           Errno::ECONNREFUSED, \
+           Errno::ECONNRESET, \
+           Errno::EHOSTUNREACH, \
+           Errno::ENETUNREACH, \
+           Net::HTTP::Persistent::Error, \
+           Errno::ETIMEDOUT
 
-      @key.redis.multi do
-        @key.del
-        @key.hmset(*fields.to_a.flatten)
-      end
+      retries += 1
+
+      $stderr.puts("-> #{$!.class}: #{$!.message}")
+
+      sleep([retries ** 2 * throttle, 300].min)
+
+      retry if maximum_retries.nil? || retries < maximum_retries
     end
   end
+end
 
-  class Remote
-    def self.request(verb, url, request_headers)
-      status = nil
-      headers = {}
-      body = ""
+class Cacho::DB
+  attr :path
 
-      curl = Curl::Easy.new(url)
+  def initialize(path)
+    @path = File.expand_path(path)
 
-      curl.headers = request_headers
+    FileUtils.mkdir_p(@path)
+  end
 
-      curl.on_header do |header|
-        headers.store(*header.rstrip.split(": ", 2)) if header.include?(":")
-        header.bytesize
-      end
+  def get(verb, uri)
+    parts = [
+      "#{uri.host}-#{uri.port}",
+      verb.to_s,
+      Digest::MD5.hexdigest(uri.to_s)
+    ]
 
-      curl.on_body do |string|
-        body << string.force_encoding(Encoding::UTF_8)
-        string.bytesize
-      end
+    doc_path = File.join(@path, *parts)
 
-      curl.on_complete do |response|
-        status = response.response_code
-      end
+    FileUtils.mkdir_p(File.dirname(doc_path)) if doc_path.start_with?(@path)
+
+    if File.exist?(doc_path)
+      str = File.read(doc_path)
 
-      curl.head = verb == :head
+      return Marshal.load(str)
+    else
+      value = yield
 
-      curl.http(verb.to_s.upcase)
+      File.write(doc_path, Marshal.dump(value))
 
-      [status, headers, body]
+      return value
     end
   end
 end

metadata

@@ -1,98 +1,66 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: cacho
-version: !ruby/object:Gem::Version 
-  prerelease: false
-  segments: 
-  - 0
-  - 0
-  - 3
-  version: 0.0.3
+version: !ruby/object:Gem::Version
+  version: 0.1.0
 platform: ruby
-authors: 
+authors:
 - Damian Janowski
-- Michel Martens
+- Martín Sarsale
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2010-11-26 00:00:00 -03:00
-default_executable: 
-dependencies: 
-- !ruby/object:Gem::Dependency 
-  name: nest
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ~>
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 1
-        - 0
-        version: "1.0"
+date: 2014-10-30 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: net-http-persistent
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
   type: :runtime
-  version_requirements: *id001
-- !ruby/object:Gem::Dependency 
-  name: curb
   prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ~>
-      - !ruby/object:Gem::Version 
-        segments: 
-        - 0
-        - 7
-        version: "0.7"
-  type: :runtime
-  version_requirements: *id002
-description: HTTP client that understands cache responses and stores results in Redis.
-email: 
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: A careless caching client optimized for scraping.
+email:
 - djanowski@dimaion.com
-- michel@soveran.com
+- martin.sarsale@gmail.com
 executables: []
-
 extensions: []
-
 extra_rdoc_files: []
-
-files: 
-- LICENSE
+files:
+- ".gitignore"
+- README.md
 - Rakefile
-- lib/cacho.rb
+- UNLICENSE
 - cacho.gemspec
-- test/cacho.rb
-has_rdoc: true
-homepage: http://github.com/djanowski/cacho
+- examples/simple.rb
+- lib/cacho.rb
+homepage: https://github.com/djanowski/cacho
 licenses: []
-
+metadata: {}
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
-  none: false
-  requirements: 
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
   - - ">="
-    - !ruby/object:Gem::Version 
-      segments: 
-      - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
-  none: false
-  requirements: 
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
   - - ">="
-    - !ruby/object:Gem::Version 
-      segments: 
-      - 0
-      version: "0"
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: 
-rubygems_version: 1.3.7
+rubygems_version: 2.2.2
 signing_key: 
-specification_version: 3
-summary: Cache aware, Redis based HTTP client.
+specification_version: 4
+summary: A careless caching client optimized for scraping.
 test_files: []
-

data/LICENSE

@@ -1,19 +0,0 @@
-Copyright (c) 2010 Damian Janowski & Michel Martens
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.

data/test/cacho.rb

@@ -1,152 +0,0 @@
-# encoding: UTF-8
-
-require "cutest"
-require "socket"
-require "mock_server"
-
-require File.expand_path("../lib/cacho", File.dirname(__FILE__))
-
-include MockServer::Methods
-
-mock_server do
-  %w(GET OPTIONS).each do |method|
-    route method, "/cacheable" do
-      response.headers["Cache-Control"] = "public, max-age=2"
-      response.headers["Content-Type"] = "text/plain"
-      Time.now.httpdate
-    end
-  end
-
-  get "/non-cacheable" do
-    response.headers["Content-Type"] = "text/plain"
-    Time.now.httpdate
-  end
-
-  get "/etag" do
-    if request.env["HTTP_IF_MODIFIED_SINCE"]
-      halt 304
-    else
-      time = Time.now
-
-      response.headers["ETag"] = time.hash.to_s
-      response.headers["Last-Modified"] = time.httpdate
-      response.headers["Content-Type"] = "text/plain"
-
-      time.httpdate
-    end
-  end
-
-  get "/changing-etag" do
-    if request.env["HTTP_IF_MODIFIED_SINCE"]
-      time = Time.parse(request.env["HTTP_IF_MODIFIED_SINCE"]) + 1
-    else
-      time = Time.now
-
-      response.headers["ETag"] = time.hash.to_s
-      response.headers["Last-Modified"] = time.httpdate
-      response.headers["Content-Type"] = "text/plain"
-    end
-
-    time.httpdate
-  end
-
-  get "/utf" do
-    "Aló"
-  end
-
-  def route_missing
-    request.env.map do |name, value|
-      "#{name}: #{value}"
-    end.join("\n")
-  end
-end
-
-prepare do
-  Redis.current.flushdb
-end
-
-test "handles GET" do
-  _, _, body = Cacho.get("http://localhost:4000")
-
-  assert body["REQUEST_METHOD: GET"]
-end
-
-test "handles OPTIONS" do
-  _, _, body = Cacho.options("http://localhost:4000")
-
-  assert body["REQUEST_METHOD: OPTIONS"]
-end
-
-test "handles HEAD" do
-  status, headers, body = Cacho.head("http://localhost:4000")
-
-  assert status == 200
-  assert headers["Content-Type"]
-  assert body.empty?
-end
-
-test "caches cacheable responses" do
-  status, headers, body = Cacho.get("http://localhost:4000/cacheable")
-
-  assert_equal status, 200
-  assert_equal headers["Content-Type"], "text/plain"
-
-  t1 = body
-
-  sleep 1
-
-  status, headers, body = Cacho.get("http://localhost:4000/cacheable")
-
-  assert_equal t1, body
-
-  sleep 2
-
-  status, headers, body = Cacho.get("http://localhost:4000/cacheable")
-
-  assert body > t1
-end
-
-test "varies cache by HTTP method" do
-  status1, _, body1 = Cacho.get("http://localhost:4000/cacheable")
-  sleep 1
-  status2, _, body2 = Cacho.options("http://localhost:4000/cacheable")
-
-  assert status1 == 200
-  assert status2 == 200
-
-  assert body2 > body1
-end
-
-test "does not cache non-cacheable responses" do
-  _, _, t1 = Cacho.get("http://localhost:4000/non-cacheable")
-  sleep 1
-  _, _, t2 = Cacho.get("http://localhost:4000/non-cacheable")
-
-  assert t2 > t1
-end
-
-test "performs conditional GETs" do
-  _, _, t1 = Cacho.get("http://localhost:4000/etag")
-  sleep 1
-  _, _, t2 = Cacho.get("http://localhost:4000/etag")
-
-  assert_equal t1, t2
-
-  _, _, t1 = Cacho.get("http://localhost:4000/changing-etag")
-  _, _, t2 = Cacho.get("http://localhost:4000/changing-etag")
-
-  assert t2 > t1
-end
-
-test "allows to pass custom HTTP headers" do
-  _, _, body = Cacho.get("http://localhost:4000", "Accept" => "text/plain")
-
-  assert body["HTTP_ACCEPT: text/plain"]
-end
-
-test "accepts UTF-encoded bodies" do
-  _, _, body = Cacho.get("http://localhost:4000/utf")
-
-  assert body.include?("Aló")
-  assert body.encoding == Encoding::UTF_8
-end