rubyretriever 1.2.4 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a5749cf55198f97bab6c77297bf6409a2518bca0
-  data.tar.gz: df792b6d3b1d03a8b70faadf651e20779f4fd1e8
+  metadata.gz: 05f8e6c0169af87c8284c8b6e98d5f25488b0980
+  data.tar.gz: a45a361b215b5ae7832e762b08bbdb989d0847a1
 SHA512:
-  metadata.gz: 76e12598873e8779e196f84ec040a5c9fef01e410c884fa88b18f41335848a69dea778459bbb9629f941dbe12c8ab2c57032d24007a1f4fd57c5caec3a76abea
-  data.tar.gz: 13b91c96ce17eb8ce802250505943d13efbd292cd95b120336ddd0ddd98278c513310c68eb9606a92b4018ef6ec9369369ee5869eff0abb589457c7b3bde41c3
+  metadata.gz: 8cee32f96e0ea0fe003a109016c6b17f9ddde9a73d72dbfd0a95c63e413b87b41c2ccf5bbc86b9886f78f59f053fbcd27aad0cbbbededbe8b002f0f7d986c528
+  data.tar.gz: fd0762069a69f7383a59b4058b46bde46793defec437eb1525e69f98f60e4429c1dc676d2d8ae3fa828c8d82d2018a9e9685bb15537424cb3e15fe0d5c472ade

data/bin/rr

@@ -32,7 +32,7 @@ optparse = OptionParser.new do |opts|
   opts.on('-p', '--progress', 'Output progress bar') do
     options['progress'] = true
   end
-  options['maxpages'] = false
+  options['maxpages'] = 100
   opts.on('-l',
           '--limit PAGE_LIMIT_#',
           'set a max on the total number of crawled pages') do |maxp|

data/lib/retriever/core_ext.rb

@@ -0,0 +1,13 @@
+require 'htmlentities'
+#
+module SourceString
+  refine String do
+    def decode_html
+      HTMLEntities.new.decode(self)
+    end
+
+    def encode_utf8_and_replace
+      encode('UTF-8', invalid: :replace, undef: :replace)
+    end
+  end
+end

data/lib/retriever/fetch.rb

@@ -15,6 +15,7 @@ module Retriever
     # There is no direct output
     # this is a parent class that the other fetch classes build off of.
     def initialize(url, options)
+      @iterator = false
       @result = []
       @connection_tally = {
         success: 0,
@@ -27,6 +28,9 @@ module Retriever
       @t = Retriever::Target.new(url, @file_re)
       @output = "rr-#{@t.host.split('.')[1]}" if @fileharvest && !@output
       @already_crawled = setup_bloom_filter
+    end
+
+    def start
       @page_one = crawl_page_one
       @link_stack = create_link_stack
       @temp_link_stack = []
@@ -80,6 +84,39 @@ module Retriever
       puts
     end
 
+ # returns true is resp is ok to continue
+    def good_response?(resp, url)
+      return false unless resp
+      hdr = resp.response_header
+      if hdr.redirection?
+        loc = hdr.location
+        lg("#{url} Redirected to #{loc}")
+        if t.host_re =~ loc
+          @temp_link_stack.push(loc) unless @already_crawled.include?(loc)
+          lg('--Added to stack for later')
+          return false
+        end
+        lg("Redirection outside of target host. No - go. #{loc}")
+        return false
+      end
+      # lets not continue if unsuccessful connection
+      unless hdr.successful?
+        lg("UNSUCCESSFUL CONNECTION -- #{url}")
+        @connection_tally[:error] += 1
+        @connection_tally[:error_server] += 1 if hdr.server_error?
+        @connection_tally[:error_client] += 1 if hdr.client_error?
+        return false
+      end
+      # let's not continue if not text/html
+      unless hdr['CONTENT_TYPE'] =~ %r{(text/html|application/xhtml+xml)}
+        @already_crawled.insert(url)
+        lg("Page Not text/html -- #{url}")
+        return false
+      end
+      @connection_tally[:success] += 1
+      true
+    end
+
     private
 
     def setup_options(options)
@@ -119,7 +156,7 @@ module Retriever
     end
 
     def crawl_page_one
-      page_one = Retriever::Page.new(@t.source, @t)
+      page_one = Retriever::Page.new(@t.target, @t.source, @t)
       lg("URL Crawled: #{@t.target}")
       page_one
     end
@@ -141,13 +178,13 @@ module Retriever
 
     # iterates over the existing @link_stack
     # running until we reach the @max_pages value.
-    def async_crawl_and_collect
+    def async_crawl_and_collect(&block)
       while @already_crawled.size < @max_pages
         if @link_stack.empty?
           end_crawl_notice
           break
         end
-        new_links_arr = process_link_stack
+        new_links_arr = process_link_stack(&block)
         @temp_link_stack = []
         next if new_links_arr.nil? || new_links_arr.empty?
         @link_stack.concat(new_links_arr)
@@ -157,47 +194,14 @@ module Retriever
       @result.uniq!
     end
 
-    # returns true is resp is ok to continue
-    def good_response?(resp, url)
-      return false unless resp
-      hdr = resp.response_header
-      if hdr.redirection?
-        loc = hdr.location
-        lg("#{url} Redirected to #{loc}")
-        if t.host_re =~ loc
-          @temp_link_stack.push(loc) unless @already_crawled.include?(loc)
-          lg('--Added to stack for later')
-          return false
-        end
-        lg("Redirection outside of target host. No - go. #{loc}")
-        return false
-      end
-      # lets not continue if unsuccessful connection
-      unless hdr.successful?
-        lg("UNSUCCESSFUL CONNECTION -- #{url}")
-        @connection_tally[:error] += 1
-        @connection_tally[:error_server] += 1 if hdr.server_error?
-        @connection_tally[:error_client] += 1 if hdr.client_error?
-        return false
-      end
-      # let's not continue if not text/html
-      unless hdr['CONTENT_TYPE'].include?('text/html')
-        @already_crawled.insert(url)
-        lg("Page Not text/html -- #{url}")
-        return false
-      end
-      @connection_tally[:success] += 1
-      true
-    end
-
-    def push_seo_to_data(url, new_page)
+    def push_seo_to_result(url, new_page)
       seos = [url]
       seos.concat(new_page.parse_seo)
       @result.push(seos)
       lg('--page SEO scraped')
     end
 
-    def push_files_to_data(new_page)
+    def push_files_to_result(new_page)
       filez = new_page.parse_files(new_page.parse_internal)
       @result.concat(filez) unless filez.empty?
       lg("--#{filez.size} files found")
@@ -209,7 +213,7 @@ module Retriever
       if @progress && (@already_crawled.size < @max_pages)
         @progressbar.increment
       end
-      Retriever::Page.new(response, @t)
+      Retriever::Page.new(url, response, @t)
     end
 
     def new_visitable_links(current_page)
@@ -217,10 +221,16 @@ module Retriever
       current_page.parse_internal_visitable
     end
 
+    def push_custom_to_result(url, current_page, &block)
+      data = block.call current_page
+      @result.push(data) unless data.empty?
+      lg("-- PageIterator called on: #{url}")
+    end
+
     # send a new wave of GET requests, using current @link_stack
     # at end of the loop it empties link_stack
     # puts new links into temporary stack
-    def process_link_stack
+    def process_link_stack(&block)
       EM.synchrony do
         concurrency = 10
         EM::Synchrony::FiberIterator.new(@link_stack, concurrency).each do |url|
@@ -230,20 +240,19 @@ module Retriever
           next unless good_response?(resp, url)
           current_page = page_from_response(url, resp.response)
           # non-link dependent modes
-          push_seo_to_data(url, current_page) if @seo
+          push_seo_to_result(url, current_page) if @seo
+          push_custom_to_result(url, current_page, &block) if @iterator
           next unless current_page.links.size > 0
           @temp_link_stack.push(new_visitable_links(current_page))
           # link dependent modes
           next unless @fileharvest
-          push_files_to_data(current_page)
+          push_files_to_result(current_page)
         end
         EventMachine.stop
       end
       # empty the stack. most clean way
       @link_stack = []
       # temp contains redirects + new visitable links
-      # we will re-initialize it as empty right after this function
-      # in the parent method 'async crawl and collect'
       @temp_link_stack.flatten.uniq!
     end
   end

data/lib/retriever/fetchfiles.rb

@@ -5,6 +5,7 @@ module Retriever
   class FetchFiles < Fetch
     def initialize(url, options)
       super
+      start
       temp_file_collection = @page_one.parse_files(@page_one.parse_internal)
       @result.concat(temp_file_collection) if temp_file_collection.size > 0
       lg("#{@result.size} new files found")

data/lib/retriever/fetchseo.rb

@@ -6,6 +6,7 @@ module Retriever
     #   on all unique pages found on the site
     def initialize(url, options)
       super
+      start
       @result.push(@page_one.parse_seo)
 
       async_crawl_and_collect

data/lib/retriever/fetchsitemap.rb

@@ -5,6 +5,7 @@ module Retriever
     # returns an array of all unique pages found on the site
     def initialize(url, options)
       super
+      start
       @result.push(@t.target)
       @result.concat(@link_stack)
 

data/lib/retriever/page.rb

@@ -1,5 +1,6 @@
 require 'addressable/uri'
-
+#
+using SourceString
 module Retriever
   #
   class Page
@@ -30,11 +31,12 @@ module Retriever
                                 wmv|flv|mp3|wav|doc|txt|ico|xml)
                                 /ix).freeze
 
-    attr_reader :links, :source, :t
+    attr_reader :links, :source, :t, :url
 
-    def initialize(source, t)
+    def initialize(url, source, t)
+      @url = url
       @t = t
-      @source = source.encode('UTF-8', invalid: :replace, undef: :replace)
+      @source = source.encode_utf8_and_replace
       @links = nil
     end
 
@@ -59,24 +61,24 @@ module Retriever
       parse_internal.select { |x| !(NONPAGE_EXT_RE =~ x) }
     end
 
-    def parse_files(arr)
+    def parse_files(arr = parse_internal)
       arr.select { |x| @t.file_re =~ x }
     end
 
     def title
-      TITLE_RE =~ @source ? @source.match(TITLE_RE)[1] : ''
+      TITLE_RE =~ @source ? @source.match(TITLE_RE)[1].decode_html : ''
     end
 
     def desc
-      DESC_RE =~ @source ? @source.match(DESC_RE)[1] : ''
+      DESC_RE =~ @source ? @source.match(DESC_RE)[1].decode_html  : ''
     end
 
     def h1
-      H1_RE =~ @source ? @source.match(H1_RE)[1] : ''
+      H1_RE =~ @source ? @source.match(H1_RE)[1].decode_html  : ''
     end
 
     def h2
-      H2_RE =~ @source ? @source.match(H2_RE)[1] : ''
+      H2_RE =~ @source ? @source.match(H2_RE)[1].decode_html  : ''
     end
 
     def parse_seo

data/lib/retriever/page_iterator.rb

@@ -0,0 +1,21 @@
+module Retriever
+  #
+  class PageIterator < Fetch
+    # recieves target url and RR options, and a block
+    # runs the block on all pages during crawl, pushing
+    #   the returned value of the block onto a result stack
+    #   the complete data returned from the crawl is accessible thru self.result
+    def initialize(url, options, &block)
+      super
+      start
+      fail 'block required for PageIterator' unless block_given?
+      @iterator = true
+      @result.push(block.call @page_one)
+      lg("-- PageIterator crawled- #{url}")
+      async_crawl_and_collect(&block)
+      # done, make sure progress bar says we are done
+      @progressbar.finish if @progress
+      @result.sort_by! { |x| x.length } if @result.size > 1
+    end
+  end
+end

data/lib/retriever/version.rb

@@ -1,4 +1,4 @@
 #
 module Retriever
-  VERSION = '1.2.4'
+  VERSION = '1.3.0'
 end

data/lib/retriever.rb

@@ -1,7 +1,9 @@
+require 'retriever/core_ext'
 require 'retriever/fetch'
 require 'retriever/fetchfiles'
 require 'retriever/fetchsitemap'
 require 'retriever/fetchseo'
+require 'retriever/page_iterator'
 require 'retriever/cli'
 require 'retriever/link'
 require 'retriever/target'

data/readme.md

@@ -25,7 +25,8 @@ Features
 
 Use cases  
 ---------
-RubyRetriever can do multiple things for you. With a single command at the terminal, RR can:  
+RubyRetriever can do multiple things for you. As an Executable
+With a single command at the terminal, RR can:  
 1. Crawl your website and output a *valid XML sitemap* based on what it found.  
 2. Crawl a target website and *download all files of a given filetype*.  
 3. Crawl a target website, *collect important SEO information* such as page titles, meta descriptions and h1 tags, and write it to CSV.  
@@ -36,41 +37,44 @@ Getting started
 -----------
 Install the gem
 ```sh
-gem install rubyretriever
+$ gem install rubyretriever
 ```  
+  
 
+Using the Executable  
+--------------------
  **Example: Sitemap mode**  
 ```sh
-rr --sitemap CSV --progress --limit 100 http://www.cnet.com
+$ rr --sitemap CSV --progress --limit 10 http://www.cnet.com
 ```  
 OR -- SAME COMMAND  
 ```sh
-rr -s csv -p -l 100 http://www.cnet.com
+$ rr -s csv -p -l 10 http://www.cnet.com
 ```  
   
-This would map http://www.cnet.com until it crawled a max of 100 pages, then write the results to a CSV named cnet. Optionally, you could also use the format XML and RR would output the same URL list into a valid XML sitemap that could be submitted to Google.  
+This would map http://www.cnet.com until it crawled a max of 10 pages, then write the results to a CSV named cnet. Optionally, you could also use the format XML and RR would output the same URL list into a valid XML sitemap that could be submitted to Google.  
   
  **Example: File Harvesting mode**  
 ```sh
-rr --files pdf --progress --limit 1000 --out hubspot http://www.hubspot.com
+$ rr --files txt --verbose --limit 1 http://textfiles.com/programming/
 ```  
 OR -- SAME COMMAND  
 ```sh
-rr -f pdf -p -l 100 http://www.hubspot.com
+$ rr -f txt -v -l 1 http://textfiles.com/programming/
 ```  
   
-This would crawl http://www.hubspot.com looking for filetype:PDF until it hit a max of 100 pages, then write out a list of filepaths to a CSV named hubspot (based on the website host name). Optionally, you could have the script autodownload all the files by adding the -a/--auto flag.
+This would crawl http://textfiles.com/programming/ looking for txt files for only a single page, then write out a list of filepaths to txt files to the terminal. Optionally, you could have the script autodownload all the files by adding the -a/--auto flag.
 
 **Example: SEO mode**  
 ```sh
-rr --seo --progress --limit 100 --out cnet-seo http://www.cnet.com
+$ rr --seo --progress --limit 10 --out cnet-seo http://www.cnet.com
 ```  
 OR -- SAME COMMAND  
 ```sh
-rr -e -p -l 10 -o cnet-seo http://www.cnet.com
+$ rr -e -p -l 10 -o cnet-seo http://www.cnet.com
 ```  
   
-This would go to http://www.cnet.com and crawl a max of 100 pages, during which it would collect the SEO fields on those pages - this currently means [url, page title, meta description, h1 text, h2 text]. It would then write the fields to a csv named cnet-seo.
+This would go to http://www.cnet.com and crawl a max of 10 pages, during which it would collect the SEO fields on those pages - this currently means [url, page title, meta description, h1 text, h2 text]. It would then write the fields to a csv named cnet-seo.
   
 
 command-line arguments
@@ -89,11 +93,48 @@ and OPTIONS is the applicable:
     -l, --limit PAGE_LIMIT_#            *set a max on the total number of crawled pages*  
     -h, --help                          *Display this screen*  
   
+
+Using as a Library (starting as of version 1.3.0 -- yet to be released)  
+------------------
+
+If you want to collect something, other than that which the executable allows, on a 'per page' basis then you want to use the PageIterator class. Then you can run whatever block you want against each individual page's source code located during the crawl.   
+
+Sample Script using **PageIterator**  
+```ruby
+require 'retriever'
+opts = {
+  'maxpages' => 1
+}
+t = Retriever::PageIterator.new('http://www.basecamp.com', opts) do |page|
+  [page.url, page.title]
+end
+puts t.result.to_s
+```
+
+```sh
+>> [["http://www.basecamp.com", "Basecamp is everyone’s favorite project management app."]]  
+```  
+Available methods on the page iterator:  
+* **#url** - returns full URL of current page  
+* **#source** - returns raw page source code  
+* **#title** - returns html decoded verson of curent page title  
+* **#desc** - returns html decoded verson of curent page meta description  
+* **#h1**  - returns html decoded verson of current page's h1 tag  
+* **#h2**  - returns html decoded verson of current page's h2 tag
+* **#links** - returns array of all links on the page  
+* **#parse_internal** - returns array of current page's internal (same host) links  
+* **#parse_internal_visitable** - returns #parse_internal plus added filtering of only links that are visitable  
+* **#parse_seo** - returns array of current page's html decoded title, desc, h1 and h2  
+* **#parse_files** - returns array of downloaded files of type supplied as RR options (fileharvest options)  
+
+
 Current Requirements
 ------------ 
 em-synchrony  
 ruby-progressbar  
 bloomfilter-rb  
+addressable  
+htmlentities  
 
 License
 -------  

data/spec/link_spec.rb

@@ -3,7 +3,7 @@ require 'retriever'
 describe 'Link' do
 
   t = Retriever::Target.new('http://www.cnet.com/reviews/')
-  let(:links) { Retriever::Page.new(@source, t).links }
+  let(:links) { Retriever::Page.new('http://www.cnet.com/reviews/', @source, t).links }
 
   it 'collects links in anchor tags' do
     @source = (<<SOURCE).strip

data/spec/page_spec.rb

@@ -4,9 +4,18 @@ require 'retriever/fetch'
 t = Retriever::Target.new('http://www.cnet.com/reviews/', /\.exe\z/)
 
 describe 'Page' do
+  describe '#url' do
+    let(:page) { Retriever::Page.new('http://www.cnet.com/', @source, t) }
+    it 'returns current page URL' do
+      @source = (<<SOURCE).strip
+<a href='http://www.cnet.com/'>download</a>
+SOURCE
+      expect(page.url).to eq('http://www.cnet.com/')
+    end
+  end
 
   describe '#links' do
-    let(:links) { Retriever::Page.new(@source, t).links }
+    let(:page) { Retriever::Page.new('http://www.cnet.com/', @source, t) }
     it 'collects all unique href links on the page' do
       @source = (<<SOURCE).strip
 <a href='www.cnet.com/download.exe'>download</a>
@@ -17,12 +26,12 @@ describe 'Page' do
  <a href='http://www.yahoo.com/test/'>yahoo</a>
 SOURCE
 
-      expect(links.size).to eq(4)
+      expect(page.links.size).to eq(4)
     end
   end
 
   describe '#parse_internal' do
-    let(:page) { Retriever::Page.new(@source, t) }
+    let(:page) { Retriever::Page.new('http://www.cnet.com/', @source, t) }
     let(:links) { page.parse_internal }
     it 'filters links by host' do
       @source = (<<SOURCE).strip
@@ -35,7 +44,7 @@ SOURCE
   end
 
   describe '#parse_internal_visitable' do
-    let(:page) { Retriever::Page.new(@source, t) }
+    let(:page) { Retriever::Page.new('http://www.cnet.com/', @source, t) }
     let(:links) { page.parse_internal_visitable }
     it "filters out 'unvisitable' URLS like JS, Stylesheets, Images" do
       @source = (<<SOURCE).strip
@@ -46,7 +55,7 @@ SOURCE
   end
 
   describe '#parse_files' do
-    let(:page) { Retriever::Page.new(@source, t) }
+    let(:page) { Retriever::Page.new('http://www.cnet.com/', @source, t) }
     let(:files) { page.parse_files(page.parse_internal) }
     it 'filters links by filetype' do
       @source = (<<SOURCE).strip
@@ -59,7 +68,7 @@ SOURCE
   end
 
   describe '#title' do
-    let(:page) { Retriever::Page.new(@source, t) }
+    let(:page) { Retriever::Page.new('http://www.cnet.com/', @source, t) }
     it 'returns page title' do
       @source = (<<SOURCE).strip
 <title>test</title>
@@ -68,7 +77,7 @@ SOURCE
     end
   end
   describe '#desc' do
-    let(:page) { Retriever::Page.new(@source, t) }
+    let(:page) { Retriever::Page.new('http://www.cnet.com/', @source, t) }
     it 'returns meta description' do
       @source = (<<SOURCE).strip
 <meta name='description' content="test2 ">
@@ -77,7 +86,7 @@ SOURCE
     end
   end
   describe '#h1' do
-    let(:page) { Retriever::Page.new(@source, t) }
+    let(:page) { Retriever::Page.new('http://www.cnet.com/', @source, t) }
     it 'returns h1 text' do
       @source = (<<SOURCE).strip
 <h1>test 3</h1>
@@ -86,7 +95,7 @@ SOURCE
     end
   end
   describe '#h2' do
-    let(:page) { Retriever::Page.new(@source, t) }
+    let(:page) { Retriever::Page.new('http://www.cnet.com/', @source, t) }
     it 'returns h2 text' do
       @source = (<<SOURCE).strip
 <h2> test 4 </h2>

data/spec/retriever_spec.rb

@@ -1,4 +1,69 @@
 require 'retriever'
 
 describe 'Fetch' do
+  describe '#good_response?' do
+    let(:r) do
+      Retriever::Fetch.new('http://www.yahoo.com', {})
+    end
+
+    let(:resp) do
+      {}
+    end
+
+    let(:nil_response) do
+      r.good_response?(nil,'http://www.yahoo.com')
+    end
+
+    let(:unsuccessful_resp) do
+      resp.stub(:response_header).and_return(resp)
+      resp.stub(:redirection?).and_return(false)
+      resp.stub(:successful?).and_return(false)
+      resp.stub(:server_error?).and_return(false)
+      resp.stub(:client_error?).and_return(false)
+      r.good_response?(resp,'http://www.yahoo.com')
+    end
+
+    let(:redir_resp) do
+      resp.stub(:response_header).and_return(resp)
+      resp.stub(:redirection?).and_return(true)
+      resp.stub(:location).and_return('http://www.google.com')
+      r.good_response?(resp,'http://www.yahoo.com')
+    end
+
+    let(:bad_content_type_resp) do
+      resp.stub(:response_header).and_return(resp)
+      resp.stub(:redirection?).and_return(false)
+      resp.stub(:successful?).and_return(true)
+      resp['CONTENT_TYPE'] = 'image/jpeg'
+      r.good_response?(resp,'http://www.yahoo.com')
+    end
+
+    let(:success_resp) do
+      resp.stub(:response_header).and_return(resp)
+      resp.stub(:redirection?).and_return(false)
+      resp.stub(:successful?).and_return(true)
+      resp['CONTENT_TYPE'] = 'text/html'
+      r.good_response?(resp,'http://www.yahoo.com')
+    end
+
+    it 'returns false if the response is empty' do
+      expect(nil_response).to eq(false)
+    end
+
+    it 'returns false on unsuccessful connection' do
+      expect(unsuccessful_resp).to eq(false)
+    end
+
+    it 'returns false on redirecting host' do
+      expect(redir_resp).to eq(false)
+    end
+
+    it 'returns false on non-visitable content type' do
+      expect(bad_content_type_resp).to eq(false)
+    end
+
+    it 'returns true otherwise' do
+      expect(success_resp).to eq(true)
+    end
+  end
 end

data/spec/target_spec.rb

@@ -1,9 +1,10 @@
 require 'retriever'
 require 'open-uri'
 
-t = Retriever::Target.new('http://www.cnet.com/reviews/', /\.exe\z/)
-
 describe 'Target' do
+  let(:t) do
+    Retriever::Target.new('http://www.cnet.com/reviews/', /\.exe\z/)
+  end
 
   it 'creates target var' do
     expect(t.target).to eq('http://www.cnet.com/reviews/')
@@ -30,6 +31,9 @@ describe 'Target' do
   end
 
   describe '#source' do
+    let(:redirecting_url) do
+      Retriever::Target.new('http://software-by-joe.appspot.com').source
+    end
 
     it 'opens URL and returns source as String' do
       expect(Retriever::Target.new('http://techcrunch.com/').source.class)
@@ -37,8 +41,7 @@ describe 'Target' do
     end
 
     it 'fails if target redirects to new host' do
-      expect { Retriever::Target.new('http://tinyurl.com/nkfkypa').source }
-      .to raise_error
+      expect { redirecting_url }.to raise_error
     end
   end
 end

metadata

@@ -1,125 +1,139 @@
 --- !ruby/object:Gem::Specification
 name: rubyretriever
 version: !ruby/object:Gem::Version
-  version: 1.2.4
+  version: 1.3.0
 platform: ruby
 authors:
 - Joe Norton
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-16 00:00:00.000000000 Z
+date: 2014-06-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: em-synchrony
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: em-http-request
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: ruby-progressbar
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: bloomfilter-rb
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: addressable
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: htmlentities
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.6'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.6'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '10.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '10.3'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '2.14'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '2.14'
 description: Asynchronous web crawler, scraper and file harvester
@@ -134,6 +148,7 @@ files:
 - bin/rr
 - lib/retriever.rb
 - lib/retriever/cli.rb
+- lib/retriever/core_ext.rb
 - lib/retriever/fetch.rb
 - lib/retriever/fetchfiles.rb
 - lib/retriever/fetchseo.rb
@@ -141,6 +156,7 @@ files:
 - lib/retriever/link.rb
 - lib/retriever/openuri_redirect_patch.rb
 - lib/retriever/page.rb
+- lib/retriever/page_iterator.rb
 - lib/retriever/target.rb
 - lib/retriever/version.rb
 - readme.md
@@ -159,17 +175,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.8.6
+      version: 2.0.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: 1.3.6
 requirements: []
 rubyforge_project: rubyretriever
-rubygems_version: 2.2.2
+rubygems_version: 2.3.0
 signing_key: 
 specification_version: 4
 summary: Ruby Web Crawler & File Harvester