upton 0.2.0 → 0.2.2

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    Y2M0NjVlMWNlN2ZkNTEwMzIzYmQ2N2MxYjBjZjExZmQ1ZTYyZjAzNw==
+  data.tar.gz: !binary |-
+    NTg4NTc3NDQ3OWQ1OGRiY2Y0ZmFkYjI5OWU2NmRkNjVlNDNlYzRkZQ==
+!binary "U0hBNTEy":
+  metadata.gz: !binary |-
+    NzY5YWNlMTIwNWM5YTNiM2YwMDkwNDNhMTlhNjM2ZTQyMWQ1YjRhMTUwNThm
+    YThkYzU4ZjI2ZWU0YzA1MjRjMDgxMmEwOGU1MGFlYjMyZDZjYTNjOGU3NDAy
+    OTQ0Yzc5NTNiNWU0MjE0YTZmYWZkZTQxZjQ1NjliZGEyOGYyZWI=
+  data.tar.gz: !binary |-
+    Zjg4ODk0ZGZjMDVkZGE5MGEzMTk1Mjg0MzQ1ODllZjUzZjc2MWFjOTU3YzAz
+    YzE5ZDAyZGZmNDdmZDU3ZWViZTEzNjNhZGNmMGZmMDgwY2YxYTgwZjkxNzFj
+    NTMwODU4ODg2Nzg2NGYzOGU2NmJhNDBmNmI1OWQwZDVkN2UzZmM=

data/lib/upton.rb

@@ -11,8 +11,11 @@
 #     site's search page or a newspaper's homepage.
 # 2. Instance pages, which represent the goal of your scraping, e.g.
 #     job listings or news articles.
+#
 
 require 'nokogiri'
+require 'uri'
+require 'restclient'
 require './lib/utils'
 
 module Upton
@@ -21,7 +24,7 @@ module Upton
   # in more complicated cases; e.g. +MyScraper < Upton::Scraper+
   class Scraper
 
-    attr_accessor :verbose, :debug, :nice_sleep_time, :stash_folder, :url_array
+    attr_accessor :verbose, :debug, :sleep_time_between_requests, :stash_folder, :url_array
 
     # == Basic use-case methods.
 
@@ -36,16 +39,16 @@ module Upton
       self.scrape_from_list(self.url_array, blk)
     end
 
-
-    # == Configuration Options
-
-    # +index_url+: The URL of the page containing the list of instances.
+    # +index_url_or_array+: A list of string URLs, OR
+    #              the URL of the page containing the list of instances.
     # +selector+: The XPath or CSS that specifies the anchor elements within 
-    # the page.
+    #              the page, if a url is specified for the previous argument.
     # +selector_method+: +:xpath+ or +:css+. By default, +:xpath+.
     #
     # These options are a shortcut. If you plant to override +get_index+, you
     # do not need to set them.
+    # If you don't specify a selector, the first argument will be treated as a
+    # list of URLs.
     def initialize(index_url_or_array, selector="", selector_method=:xpath)
 
       #if first arg is a valid URL, do already-written stuff;
@@ -55,7 +58,7 @@ module Upton
       #TODO: rewrite this, because it's a little silly. (i.e. should be a more sensical division of how these arguments work)
       if selector.empty?
         @url_array = index_url_or_array
-      elsif index_url_or_array =~ URI::ABS_URI
+      elsif index_url_or_array =~ ::URI::ABS_URI
         @index_url = index_url_or_array
         @index_selector = selector
         @index_selector_method = selector_method
@@ -79,7 +82,7 @@ module Upton
 
       # In order to not hammer servers, Upton waits for, by default, 30  
       # seconds between requests to the remote server.
-      @nice_sleep_time = 30 #seconds
+      @sleep_time_between_requests = 30 #seconds
 
       # Folder name for stashes, if you want them to be stored somewhere else,
       # e.g. under /tmp.
@@ -90,13 +93,15 @@ module Upton
     end
 
 
+    # == Configuration Options
 
     # If instance pages are paginated, <b>you must override</b> 
     # this method to return the next URL, given the current URL and its index.
     #
     # If instance pages aren't paginated, there's no need to override this.
     #
-    # Return URLs that are empty strings are ignored (and recursion stops.)
+    # Recursion stops if the fetching URL returns an empty string or an error.
+    #
     # e.g. next_instance_page_url("http://whatever.com/article/upton-sinclairs-the-jungle?page=1", 2)
     # ought to return "http://whatever.com/article/upton-sinclairs-the-jungle?page=2"
     def next_instance_page_url(url, index)
@@ -108,7 +113,8 @@ module Upton
     #
     # If index pages aren't paginated, there's no need to override this.
     #
-    # Return URLs that are empty strings are ignored (and recursion stops.)
+    # Recursion stops if the fetching URL returns an empty string or an error.
+    #
     # e.g. +next_index_page_url("http://whatever.com/articles?page=1", 2)+
     # ought to return "http://whatever.com/articles?page=2"
     def next_index_page_url(url, index)
@@ -121,7 +127,7 @@ module Upton
         self.url_array = self.get_index
       end
       CSV.open filename, 'wb' do |csv|
-        self.scrape_from_list(self.url_array, blk).each{|document| document.each{|line| csv << line }}
+        self.scrape_from_list(self.url_array, blk).each{|document| csv << document }
       end
     end
 
@@ -139,7 +145,7 @@ module Upton
       else
         begin
           puts "getting " + url if @verbose
-          sleep @nice_sleep_time
+          sleep @sleep_time_between_requests
           resp = RestClient.get(url, {:accept=> "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"})
 
           #this is silly, but rest-client needs to get on their game.
@@ -152,7 +158,6 @@ module Upton
             elsif content_type.split('/').first == 'text'
               'iso-8859-1'
             end
-            puts charset
             resp.force_encoding(charset) if charset
           end
 
@@ -201,8 +206,12 @@ module Upton
       resp
     end
 
-    # Returns the concatenated output of each member of a paginated instance,
-    # e.g. a news article with 2 pages.
+    # Returns the article at `url`.
+    # 
+    # If the page is stashed, returns that, otherwise, fetches it from the web.
+    #
+    # If an instance is paginated, returns the concatenated output of each 
+    # page, e.g. if a news article has two pages.
     def get_instance(url, index=0)
       resp = self.get_page(url, @debug)
       if !resp.empty? 

metadata

@@ -1,20 +1,18 @@
 --- !ruby/object:Gem::Specification
 name: upton
 version: !ruby/object:Gem::Version
-  version: 0.2.0
-  prerelease: 
+  version: 0.2.2
 platform: ruby
 authors:
 - Jeremy B. Merrill
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-06-25 00:00:00.000000000 Z
+date: 2013-07-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rack
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -22,7 +20,6 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -30,7 +27,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: thin
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -38,7 +34,6 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -46,7 +41,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -54,7 +48,6 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -62,7 +55,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: yard
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -70,7 +62,6 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -78,7 +69,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -86,7 +76,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -94,7 +83,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -102,7 +90,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -125,27 +112,26 @@ files:
 homepage: http://github.org/propublica/upton
 licenses:
 - MIT
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: 1.8.7
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.23
+rubygems_version: 2.0.5
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: A simple web-scraping framework
 test_files:
 - test/data/discussion.html