rubyretriever 1.4.1 → 1.4.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9b337916fddfc246a2b3cd73bdd40cfe13eccb9a
-  data.tar.gz: 908d0a752ab89adaa9c8530ae7803329c14591ce
+  metadata.gz: dca26f55e68ab2c5095d9e633592735dc67323e9
+  data.tar.gz: 4e96d543d7ea4001ef4531db3beb5bf3e8accf4d
 SHA512:
-  metadata.gz: 7bf4c80012e0232b6bea179b5e41ab36c8066e7b0bb6da39b0eaedd0a6b67a6b826754f6f4e94d9dd21287ebc05579ebc759b7887ed29218c82eccb68b31fd79
-  data.tar.gz: 2bc3158bf98beb2c2b701b7811b774c1ff32236899683b6486bf2d1680ddbb87e8ebbb7aaa27eb8aa968a20f63158640c9d45c1d38c992484a9920069b38b976
+  metadata.gz: f08c1d9c7c2395cfc297f1c0abb80760d08d35d24394d381b703d982709757ffc9072e55a60e15b123e45ae6731a388cb2afa914f2a136ab3fce454af11b7b8c
+  data.tar.gz: 0f75a30cb8a5f969d7bc196c6f83e0f25ddd0f76ca2a931aefb6206180172dbb6c4dbb5d4b926d69e75555dd0eec3a8b1f8a528cf18799cca31af9a753e89fc5

data/lib/retriever/fetchsitemap.rb

@@ -16,8 +16,6 @@ module Retriever
       @result.uniq!
     end
 
-    private
-
     # produces valid XML sitemap based on page collection fetched.
     # Writes to current directory.
     def gen_xml
@@ -33,6 +31,8 @@ module Retriever
       print_file_info(filename)
     end
 
+    private
+
     def print_file_info(filename)
       puts HR
       puts "File Created: sitemap-#{filename}.xml"

data/lib/retriever/page.rb

@@ -5,6 +5,7 @@ using SourceString
 module Retriever
   #
   class Page
+    HASH_RE   = Regexp.new(/^#/i).freeze
     HTTP_RE   = Regexp.new(/^http/i).freeze
     H1_RE     = Regexp.new(/<h1>(.*)<\/h1>/i).freeze
     H2_RE     = Regexp.new(/<h2>(.*)<\/h2>/i).freeze
@@ -50,6 +51,7 @@ module Retriever
         # filter some malformed URLS that come in
         # meant to be a loose filter to catch all reasonable HREF attributes.
         link = match[0]
+        next if HASH_RE =~ link
         Link.new(@t.scheme, @t.host, link, @url).path
       end.compact.uniq
     end

data/lib/retriever/version.rb

@@ -1,4 +1,4 @@
 #
 module Retriever
-  VERSION = '1.4.1'
+  VERSION = '1.4.2'
 end

data/readme.md

@@ -8,6 +8,8 @@ RubyRetriever is a Web Crawler, Scraper & File Harvester. Available as a command
 
 RubyRetriever (RR) uses asynchronous HTTP requests via [Eventmachine](https://github.com/eventmachine/eventmachine) & [Synchrony](https://github.com/igrigorik/em-synchrony) to crawl webpages *very quickly*. RR also uses a Ruby implementation of the [bloomfilter](https://github.com/igrigorik/bloomfilter-rb) in order to keep track of pages it has already crawled in a memory efficient manner.  
 
+**v1.4.2 Update (3/24/2016)** - Fixes problem with named anchors (divs) being counted as links.
+
 **v1.4.1 Update (3/24/2016)** - Update gemfile & external dependency versioning
 
 **v1.4.0 Update (3/24/2016)** - Several bug fixes.

data/spec/page_spec.rb

@@ -27,9 +27,10 @@ describe 'Page' do
   end
 
   describe '#links' do
-    let(:page) { Retriever::Page.new('http://www.cnet.com/', common_source, t) }
-    it 'collects all unique href links on the page' do
-      expect(page.links.size).to eq(4)
+    let(:source) { "<a href='/profile/'>profile</a><a href='#top'>top</a> <link rel='stylesheet' id='gforms_reset_css-css'  href='http://www.cnet.com/wp-content/plugins/gravityforms/css/formreset.css?ver=1.7.12' type='text/css' media='all' />" }
+    let(:page) { Retriever::Page.new('http://www.cnet.com/', source, t) }
+    it 'collects all unique href links on the page, skips div anchors' do
+      expect(page.links.size).to eq(2)
     end
   end
 
@@ -42,11 +43,11 @@ describe 'Page' do
   end
 
   describe '#parse_internal_visitable' do
-    let(:source) { "<link rel='stylesheet' id='gforms_reset_css-css'  href='http://www.cnet.com/wp-content/plugins/gravityforms/css/formreset.css?ver=1.7.12' type='text/css' media='all' />" }
+    let(:source) { "<a href='/profile/'>profile</a> <link rel='stylesheet' id='gforms_reset_css-css'  href='http://www.cnet.com/wp-content/plugins/gravityforms/css/formreset.css?ver=1.7.12' type='text/css' media='all' />" }
     let(:page) { Retriever::Page.new('http://www.cnet.com/', source, t) }
     let(:links) { page.parse_internal_visitable }
     it "filters out 'unvisitable' URLS like JS, Stylesheets, Images" do
-      expect(links.size).to eq(0)
+      expect(links.size).to eq(1)
     end
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rubyretriever
 version: !ruby/object:Gem::Version
-  version: 1.4.1
+  version: 1.4.2
 platform: ruby
 authors:
 - Joe Norton