scrape 0.2.4 → 0.3.0

data/Gemfile.lock

@@ -1,13 +1,18 @@
 PATH
   remote: .
   specs:
-    scrape (0.2.4)
+    scrape (0.3.0)
 
 GEM
   remote: http://rubygems.org/
   specs:
     addressable (2.2.8)
     crack (0.3.1)
+    faraday (0.8.1)
+      multipart-post (~> 1.1)
+    faraday_middleware (0.8.8)
+      faraday (>= 0.7.4, < 0.9)
+    multipart-post (1.1.5)
     nokogiri (1.5.5)
     webmock (1.8.7)
       addressable (>= 2.2.7)
@@ -18,6 +23,8 @@ PLATFORMS
 
 DEPENDENCIES
   addressable (~> 2.2.8)
+  faraday (~> 0.8.0)
+  faraday_middleware (~> 0.8.8)
   nokogiri (~> 1.5.5)
   scrape!
   webmock (~> 1.8.7)

data/README.md

@@ -13,6 +13,8 @@ end
 
 site "http://www.tumblr.com" # Can define multiple sites
 
+queue "http://www.tumblr.com/tagged" # Add specified urls to scrape
+
 match "/tagged" do |doc|
   # Do what ever we want with the document.
 end

data/lib/scrape.rb

@@ -1,6 +1,8 @@
 require "rubygems"
 require "logger"
-require "open-uri"
+require "addressable/uri"
+require "faraday"
+require "faraday_middleware"
 
 $: << File.dirname(__FILE__)
 
@@ -18,6 +20,7 @@ module Scrape
   autoload 'RobotsTxtRules', 'scrape/robots_txt_rules'
 
   class FileNotFound < Exception; end
+  class HTTPError < StandardError; end
 
   class << self
     attr_writer :user_agent
@@ -38,9 +41,26 @@ module Scrape
       Application.new path
     end
 
-    def open url, headers = {}, &block
-      headers = {"User-Agent" => user_agent}.merge(headers)
-      super(url, headers, &block).read
+    def open url, headers = nil, &block
+      url = Addressable::URI.parse url
+      headers ||= {}
+
+      conn = Faraday.new :url => url.to_s do |faraday|
+        faraday.response :follow_redirects, :cookies => :all, :limit => 3
+        faraday.adapter Faraday.default_adapter
+      end
+
+      conn.headers[:user_agent] = user_agent
+
+      res = conn.get url.request_uri do |req|
+        headers.each{|key, val| req[key] = val }
+      end
+
+      if res.success?
+        res.body
+      else
+        raise HTTPError, res.status
+      end
     end
   end
 end

data/lib/scrape/application.rb

@@ -3,38 +3,35 @@ class Scrape::Application
 
   def initialize scrapefile, options = {}, loader = Scrape::DefaultLoader
     @scrapefile = File.expand_path scrapefile
-    @options = options
+    @options = options.dup
     @loader = loader.class == Class ? loader.new(self) : loader
     @sites = {}
-    @queue = []
-    @history = []
+    reset
   end
 
   def run
     load_scrapefile
 
+    @queue = sites.values.map{|site| site.to_s } if @queue.empty?
+
     while url = @queue.shift
       @history << url
-      begin
-        if site = self[url]
-          if urls = site.parse(url)
-            enqueue *urls
-            Scrape.logger.info "Parsed #{url}, found #{urls.length} urls."
-          else
-            Scrape.logger.info "Parsed #{url}."
-          end
+      if site = self[url]
+        if urls = site.parse(url)
+          enqueue *urls
+          Scrape.logger.info "Parsed #{url}, found #{urls.length} urls."
         else
-          Scrape.logger.info "No rules defined for #{url}"
+          Scrape.logger.info "Parsed #{url}."
         end
-      rescue OpenURI::HTTPError => e
-        Scrape.logger.info "Error loading #{url}: #{e.message}"
+      else
+        Scrape.logger.info "No rules defined for #{url}"
       end
     end
   end
 
   def reset
     @history = []
-    @queue = sites.values.map{|site| site.to_s }
+    @queue = []
   end
 
   def queue
@@ -54,7 +51,7 @@ class Scrape::Application
   def add_site site, options = {}
     case site
     when String
-      site = Scrape::Site.new site, options
+      site = Scrape::Site.new site, options.dup
       @sites.update site.to_s => site
       site
     end
@@ -63,7 +60,6 @@ class Scrape::Application
   def load_scrapefile
     return if @scrapefile_loaded
     loader.load(scrapefile)
-    reset
     @scrapefile_loaded = true
   end
 end

data/lib/scrape/core_ext/array.rb

@@ -1,5 +1,5 @@
 class Array
   def extract_options!
     last.instance_of?(Hash) ? pop : {}
-  end unless instance_methods.include?(:extract_options!)
+  end unless Array.respond_to?(:extract_options!)
 end

data/lib/scrape/core_ext/string.rb

@@ -2,5 +2,5 @@ class String
   def starts_with str
     str = str.to_str
     self[0, str.length] == str
-  end unless instance_methods.include?(:starts_with)
+  end unless String.respond_to?(:starts_with)
 end

data/lib/scrape/dsl.rb

@@ -15,4 +15,9 @@ class Scrape::DSL
     matches = @sites.map{|site| site.add_match matcher, &proc }
     matches.size == 1 ? matches.first : matches
   end
+
+  def enqueue *urls
+    @application.enqueue *urls
+  end
+  alias_method :queue, :enqueue
 end

data/lib/scrape/robots_txt.rb

@@ -29,6 +29,7 @@ class Scrape::RobotsTxt
   end
 
   def self.parse content
+    return if content.nil?
     rules, user_agent = Hash.new, nil
 
     content.split("\n").each do |line|
@@ -49,7 +50,8 @@ class Scrape::RobotsTxt
   def self.load url, default = true
     url = Addressable::URI.join(url, "/robots.txt") if default
     parse Scrape.open(url)
-  rescue OpenURI::HTTPError
+  rescue Scrape::HTTPError
+    Scrape.logger.warn "Failed to obtain robots.txt: #{url}"
     nil
   end
   public :load

data/lib/scrape/site.rb

@@ -20,7 +20,7 @@ class Scrape::Site
 
   def open url
     headers = Hash.new
-    headers['Set-Cookie'] = options[:cookie].to_s if options.has_key? :cookie
+    headers[:cookie] = cookie if options[:cookie]
     Scrape.open url, headers
   end
 
@@ -31,6 +31,9 @@ class Scrape::Site
     @matches.each{|match| match.invoke doc, url if match =~ url }
 
     doc.css("a[href]").map{|node| normalize node['href'], url }.select{|url| accept? url }
+  rescue Scrape::HTTPError => e
+    Scrape.logger.info "Error loading #{url}: #{e.message}"
+    nil
   end
 
   def accept? url
@@ -55,4 +58,18 @@ private
   def disallowed? url
     !options[:ignore_robots_txt] && robots_txt =~ Addressable::URI.parse(url).path
   end
+
+  def cookie
+    cookie = options[:cookie]
+    case cookie
+    when Hash
+      cookie.map{|name, val| "#{encode(name)}=#{encode(val)}" }.join("; ")
+    when String
+      cookie
+    end
+  end
+
+  def encode str
+    str.to_s.gsub(" ", "%20").gsub(",", "%2C").gsub(";", "%3B")
+  end
 end

data/lib/scrape/version.rb

@@ -1,3 +1,3 @@
 module Scrape
-  VERSION = '0.2.4' unless defined? ::Scrape::VERSION
+  VERSION = '0.3.0' unless defined? ::Scrape::VERSION
 end

data/scrape.gemspec

@@ -20,4 +20,6 @@ Gem::Specification.new do |s|
 
   s.add_development_dependency "nokogiri", "~> 1.5.5"
   s.add_development_dependency "addressable", "~> 2.2.8"
+  s.add_development_dependency "faraday", "~> 0.8.0"
+  s.add_development_dependency "faraday_middleware", "~> 0.8.8"
 end

data/test/test_helper.rb

@@ -8,6 +8,9 @@ Bundler.setup(:default, :test)
 
 require "scrape"
 
+# surpress log messages while we're testing
+Scrape.logger = Class.new{ def method_missing name, *args; end }.new
+
 class Scrape::TestCase < MiniTest::Unit::TestCase
   class << self
     def test name, &block

data/test/unit/application_test.rb

@@ -18,37 +18,33 @@ class ApplicationTest < Scrape::TestCase
   end
 
   test "#[] should return the site that matches the given url" do
-    site1 = Scrape::Site.new "http://example.com"
-    site2 = Scrape::Site.new "http://example.org"
     app = Scrape::Application.new(".")
-    app.sites.update site1.to_s => site1, site2.to_s => site2
+    site1 = app.add_site "http://example.com"
+    app.add_site "http://example.org"
     assert_equal site1, app["http://example.com"]
   end
 
   test "#[] should return the site that is relative to the given url" do
-    site1 = Scrape::Site.new "http://example.com"
-    site2 = Scrape::Site.new "http://example.org"
     app = Scrape::Application.new(".")
-    app.sites.update site1.to_s => site1, site2.to_s => site2
+    site1 = app.add_site "http://example.com"
+    app.add_site "http://example.org"
     assert_equal site1, app["http://example.com/test"]
   end
 
   test "#[] should return nil when no site matches the given url" do
-    site1 = Scrape::Site.new "http://example.com"
-    site2 = Scrape::Site.new "http://example.org"
     app = Scrape::Application.new(".")
-    app.sites.update site1.to_s => site1, site2.to_s => site2
+    app.add_site "http://example.com"
+    app.add_site "http://example.org"
     assert_nil app["http://example.net"]
   end
 
-  test "#reset should enqueue the sites that have been defined" do
-    site1 = Scrape::Site.new "http://example.com"
-    site2 = Scrape::Site.new "http://example.org"
-    app = Scrape::Application.new(".")
-    app.sites.update site1.to_s => site1, site2.to_s => site2
-    app.reset
-    assert_equal ["http://example.com", "http://example.org"], app.queue
-  end
+  # test "#reset should enqueue the sites that have been defined" do
+  #   app = Scrape::Application.new(".")
+  #   app.add_site "http://example.com"
+  #   app.add_site "http://example.org"
+  #   app.reset
+  #   assert_equal ["http://example.com", "http://example.org"], app.queue
+  # end
 
   test "#run should load the specified file" do
     filepath = File.join(SUPPORT_FILES, 'test1.scrape')

data/test/unit/dsl_test.rb

@@ -40,4 +40,11 @@ class DSLTest < Scrape::TestCase
       dsl.match("test"){|*args|}
     end
   end
+
+  test "#enqueue should add the specified urls to the queue" do
+    app = Scrape::Application.new(".")
+    dsl = Scrape::DSL.new app
+    dsl.enqueue "http://example.com"
+    assert_equal ["http://example.com"], app.queue
+  end
 end

data/test/unit/scrape_test.rb

@@ -1,25 +1,38 @@
 require "test_helper"
 
 class ScrapeTest < Scrape::TestCase
-  test "#user_agent should return default when not set" do
+  test ".user_agent should return default when not set" do
     assert_equal Scrape.user_agent, "Scrape/#{Scrape::VERSION}"
   end
 
-  test "#load_scrapefile should return a new application" do
+  test ".load_scrapefile should return a new application" do
     app = Scrape.load_scrapefile '.'
     assert_kind_of Scrape::Application, app
   end
 
-  test "#open should send a request to the specified url and return the contents" do
-    stub_request(:get, "http://example.com/").to_return(:status => 200, :body => "booyah")
-    assert_equal "booyah", Scrape.open("http://example.com")
-  end
-
-  test "#open should set the user agent in the request header" do
+  test ".open should set the user agent in the request header" do
     stub_request(:get, "http://example.com/").
       with(:headers => {"User-Agent" => "Scrape/#{Scrape::VERSION}"}).
       to_return(:status => 200, :body => "")
     Scrape.open("http://example.com")
     assert true
   end
+
+  test ".open should redirect when response is indicates redirection" do
+    stub_request(:get, "http://example.com/foo").
+      to_return(:status => 301, :headers => {:location => "http://example.com/bar"})
+    stub_request(:get, "http://example.com/bar").
+      to_return(:status => 200, :body => "booyah")
+    Scrape.open("http://example.com/foo")
+    assert true
+  end
+
+  test ".open should raise error when not successful" do
+    stub_request(:get, "http://example.com/").
+      to_return(:status => 404, :body => "")
+
+    assert_raises Scrape::HTTPError do
+      Scrape.open("http://example.com")
+    end
+  end
 end

data/test/unit/site_test.rb

@@ -9,13 +9,22 @@ class SiteTest < Scrape::TestCase
 
   test "#open should include cookie header when cookie option is set" do
     stub_request(:get, "http://www.example.com/").
-      with(:headers => {'Set-Cookie'=>'omnom'}).
+      with(:headers => {'Cookie' => 'omnom'}).
       to_return(:status => 200, :body => "")
 
     site = Scrape::Site.new "http://www.example.com", :cookie => "omnom"
     site.open "http://www.example.com"
   end
 
+  test "#open should include cookie header when cookie option is a hash" do
+    stub_request(:get, "http://www.example.com/").
+      with(:headers => {'Cookie' => 'foo=bar'}).
+      to_return(:status => 200, :body => "")
+
+    site = Scrape::Site.new "http://www.example.com", :cookie => {:foo => "bar"}
+    site.open "http://www.example.com"
+  end
+
   test "#parse should return absolute urls that match the site's url" do
     stub_request(:get, "http://www.example.com/test").
       with(:headers => {"User-Agent" => Scrape.user_agent}).

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: scrape
 version: !ruby/object:Gem::Version
-  version: 0.2.4
+  version: 0.3.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-07-16 00:00:00.000000000 Z
+date: 2012-07-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -43,6 +43,38 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: 2.2.8
+- !ruby/object:Gem::Dependency
+  name: faraday
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.8.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.8.0
+- !ruby/object:Gem::Dependency
+  name: faraday_middleware
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.8.8
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.8.8
 description: An easy to use utility to scrape websites using a DSL similar to rake.
 email:
 - evilmarty@gmail.com