grell 1.3.0 → 1.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 85e9d00d79051e54ba1c5361e7f1edfd8abf8af6
-  data.tar.gz: 25b1f2db2ef87e61294158843fe6e396ed15b046
+  metadata.gz: eb66e60267eec85ffc191a0aa22313e6b2c7ea81
+  data.tar.gz: b00d7a5d2a56ff0a24157a7c0a4b34c22adbb7de
 SHA512:
-  metadata.gz: b00755884c96ddf04e6954d5761903e28cbd46207f45ba5a97f1f1b727b70aad2361386a35e3c96ecb8b65089f1412ac4d7a1bdc8a18d70b87009198883205c3
-  data.tar.gz: 927714b4cd7e7b520d75755e993eff343cebd06a8c36993bcd81abb7453e1ace497e05766f0b8d1ae4bed2e7c00447a1ded267c20028d6d118f32d661c36685e
+  metadata.gz: 38f9699d3431a4189457a7da6a8768bd87c3f8c0203fb85bbacdd4e360a32a8736088128ee7f1a56d40156ab407a8b35b9beffd0decb286674ad77acb920c75c
+  data.tar.gz: 0f5ded5c4c18cb85fd23f6a256ccb26e23804f01e61ec1cb7ab6a22242c32b0d272f4f9871fe10e2c347b3d5abd63578fa64fb9d52addc23936b77ca2a200bd6

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+* Version 1.3.1
+  Added whitelisting and blacklisting
+  Better info in gemspec
+
 * Version 1.3
   The Crawler object allows you to provide an external logger object.
   Clearer semantics when an error happens, special headers are returned so the user can inspect the error

data/README.md

@@ -51,6 +51,54 @@ end
 Grell keeps a list of pages previously crawled and do not visit the same page twice.
 This list is indexed by the complete url, including query parameters.
 
+### Selecting links to follow
+
+Grell by default will follow all the links it finds going to the site
+your are crawling. It will never follow links linking outside your site.
+If you want to further limit the amount of links crawled, you can use
+whitelisting, blacklisting or manual filtering.
+
+#### Whitelisting
+
+```ruby
+require 'grell'
+
+crawler = Grell::Crawler.new
+crawler.whitelist([/games\/.*/, '/fun'])
+crawler.start_crawling('http://www.google.com')
+```
+
+Grell here will only follow links to games and '/fun' and ignore all
+other links. You can provide a regexp, strings (if any part of the
+string match is whitelisted) or an array with regexps and/or strings.
+
+#### Blacklisting
+
+```ruby
+require 'grell'
+
+crawler = Grell::Crawler.new
+crawler.blacklist(/games\/.*/)
+crawler.start_crawling('http://www.google.com')
+```
+
+Similar to whitelisting. But now Grell will follow every other link in
+this site which does not go to /games/...
+
+If you call both whitelist and blacklist then both will apply, a link
+has to fullfill both conditions to survive. If you do not call any, then
+all links on this site will be crawled. Think of these methods as
+filters.
+
+#### Manual link filtering
+
+If you have a more complex use-case, you can modify the list of links
+manually.
+Grell yields the page to you before it adds the links to the list of
+links to visit. So you can modify in your block of code "page.links" to
+add and delete links to instruct Grell to add them to the list of links
+to visit next.
+
 ### Pages' id
 
 Each page has an unique id, accessed by the property 'id'. Also each page stores the id of the page from which we found this page, accessed by the property 'parent_id'.
@@ -64,6 +112,11 @@ When there is an error in the page or an internal error in the crawler (Javascri
 - errorClass: The class of the error which broke this page.
 - errorMessage: A descriptive message with the information Grell could gather about the error.
 
+### Logging
+You can pass your logger to Grell. For example in a Rails app:
+```Ruby
+crawler = Grell::Crawler.new(logger: Rails.logger)
+```
 
 ## Tests
 

data/grell.gemspec

@@ -6,24 +6,28 @@ require 'grell/version'
 Gem::Specification.new do |spec|
   spec.name          = "grell"
   spec.version       = Grell::VERSION
+  spec.platform      = Gem::Platform::RUBY
   spec.authors       = ["Jordi Polo Carres"]
   spec.email         = ["jcarres@mdsol.com"]
   spec.summary       = %q{Ruby web crawler}
   spec.description   = %q{Ruby web crawler using PhantomJS}
   spec.homepage      = "https://github.com/mdsol/grell"
+  spec.license       = 'MIT'
 
   spec.files         = `git ls-files -z`.split("\x0")
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ["lib"]
 
+  spec.required_ruby_version = '>= 1.9.3'
+
   spec.add_dependency 'capybara', '~> 2.2'
   spec.add_dependency 'poltergeist', '~> 1.5'
 
   spec.add_development_dependency "bundler", "~> 1.6"
   spec.add_development_dependency "byebug", "~> 4.0"
-  spec.add_development_dependency "kender"
-  spec.add_development_dependency "rake"
+  spec.add_development_dependency "kender", '~> 0.2'
+  spec.add_development_dependency "rake", '~> 10.0'
   spec.add_development_dependency "webmock", '~> 1.18'
   spec.add_development_dependency 'rspec', '~> 3.0'
   spec.add_development_dependency 'puffing-billy', '~> 0.5'

data/lib/grell/crawler.rb

@@ -17,6 +17,14 @@ module Grell
       @collection = PageCollection.new
     end
 
+    def whitelist(list)
+      @whitelist_regexp = Regexp.union(list)
+    end
+
+    def blacklist(list)
+      @blacklist_regexp = Regexp.union(list)
+    end
+
 
     def start_crawling(url, &block)
       Grell.logger.info "GRELL Started crawling"
@@ -32,6 +40,8 @@ module Grell
       Grell.logger.info "Visiting #{site.url}, visited_links: #{@collection.visited_pages.size}, discovered #{@collection.discovered_pages.size}"
       site.navigate
 
+      filter!(site.links)
+
       block.call(site) if block
 
       site.links.each do |url|
@@ -39,6 +49,12 @@ module Grell
       end
     end
 
+    private
+    def filter!(links)
+      links.select!{ |link| link =~ @whitelist_regexp } if @whitelist_regexp
+      links.delete_if{ |link| link =~ @blacklist_regexp } if @blacklist_regexp
+    end
+
   end
 
 end

data/lib/grell/page_collection.rb

@@ -1,4 +1,8 @@
 module Grell
+  # Keeps a record of all the pages crawled.
+  # When a new url is found it is added to this collection, which makes sure it is unique.
+  # This page is part of the discovered pages. Eventually that page will be navigated to, then
+  # the page will be part of the visited pages.
   class PageCollection
     attr_reader :collection
 

data/lib/grell/reader.rb

@@ -1,4 +1,7 @@
 module Grell
+  # A tooling class, it waits a maximum of max_waiting for an action to finish. If the action is not
+  # finished by then, it will continue anyway.
+  # The wait may be long but we want to finish it as soon as the action has finished
   class Reader
     def self.wait_for(action, max_waiting, sleeping_time)
       time_start = Time.now

data/lib/grell/version.rb

@@ -1,3 +1,3 @@
 module Grell
-  VERSION = "1.3.0"
+  VERSION = "1.3.1"
 end

data/spec/lib/crawler_spec.rb

@@ -5,7 +5,7 @@ RSpec.describe Grell::Crawler do
   let(:page) {Grell::Page.new(url, page_id, parent_page_id)}
   let(:host) {"http://www.example.com"}
   let(:url) {"http://www.example.com/test"}
-  let(:crawler) { Grell::Crawler.new(external_driver: true)}
+  let(:crawler) { Grell::Crawler.new(logger: Logger.new(nil), external_driver: true)}
   let(:body) {'body'}
 
   before do
@@ -64,24 +64,144 @@ RSpec.describe Grell::Crawler do
     end
   end
 
+  shared_examples_for 'visits all available pages' do
+    it 'visits all the pages' do
+      crawler.start_crawling(url)
+      expect(crawler.collection.visited_pages.size).to eq(visited_pages_count)
+    end
+    it 'has no more pages to discover' do
+      crawler.start_crawling(url)
+      expect(crawler.collection.discovered_pages.size).to eq(0)
+    end
+
+    it 'contains the whitelisted page and the base page only' do
+      crawler.start_crawling(url)
+      expect(crawler.collection.visited_pages.map(&:url)).
+        to eq(visited_pages)
+    end
+  end
+
   context 'the url has no links' do
     let(:body) do
       "<html><head></head><body>
       Hello world!
       </body></html>"
     end
+    let(:visited_pages_count) {1}
+    let(:visited_pages) {['http://www.example.com/test']}
+
+    it_behaves_like 'visits all available pages'
+  end
+
+  context 'the url has several links' do
+    let(:body) do
+      "<html><head></head><body>
+      <a href=\"/trusmis.html\">trusmis</a>
+      <a href=\"/help.html\">help</a>
+      Hello world!
+      </body></html>"
+    end
     before do
-      crawler.start_crawling(url)
+      proxy.stub('http://www.example.com/trusmis.html').and_return(body: 'body', code: 200)
+      proxy.stub('http://www.example.com/help.html').and_return(body: 'body', code: 200)
     end
-    it 'visits all the pages' do
-      expect(crawler.collection.visited_pages.size).to eq(1)
+    let(:visited_pages_count) {3}
+    let(:visited_pages) do
+      ['http://www.example.com/test','http://www.example.com/trusmis.html', 'http://www.example.com/help.html']
     end
-    it 'has no more pages to discover' do
-      expect(crawler.collection.discovered_pages.size).to eq(0)
+
+    it_behaves_like 'visits all available pages'
+  end
+
+  describe '#whitelist' do
+    let(:body) do
+      "<html><head></head><body>
+      <a href=\"/trusmis.html\">trusmis</a>
+      <a href=\"/help.html\">help</a>
+      Hello world!
+      </body></html>"
+    end
+
+    before do
+      proxy.stub('http://www.example.com/trusmis.html').and_return(body: 'body', code: 200)
+      proxy.stub('http://www.example.com/help.html').and_return(body: 'body', code: 200)
+    end
+
+    context 'using a single string' do
+      before do
+        crawler.whitelist('/trusmis.html')
+      end
+      let(:visited_pages_count) {2} #my own page + trusmis
+      let(:visited_pages) do
+        ['http://www.example.com/test','http://www.example.com/trusmis.html']
+      end
+
+      it_behaves_like 'visits all available pages'
+    end
+
+    context 'using an array of strings' do
+      before do
+        crawler.whitelist(['/trusmis.html', '/nothere', 'another.html'])
+      end
+      let(:visited_pages_count) {2}
+      let(:visited_pages) do
+        ['http://www.example.com/test','http://www.example.com/trusmis.html']
+      end
+
+      it_behaves_like 'visits all available pages'
+    end
+
+    context 'using a regexp' do
+      before do
+        crawler.whitelist(/\/trusmis\.html/)
+      end
+      let(:visited_pages_count) {2}
+      let(:visited_pages) do
+        ['http://www.example.com/test','http://www.example.com/trusmis.html']
+      end
+
+      it_behaves_like 'visits all available pages'
+    end
+
+    context 'using an array of regexps' do
+      before do
+        crawler.whitelist([/\/trusmis\.html/])
+      end
+      let(:visited_pages_count) {2}
+      let(:visited_pages) do
+        ['http://www.example.com/test','http://www.example.com/trusmis.html']
+      end
+
+      it_behaves_like 'visits all available pages'
+    end
+
+    context 'using an empty array' do
+      before do
+        crawler.whitelist([])
+      end
+      let(:visited_pages_count) {1} #my own page only
+      let(:visited_pages) do
+        ['http://www.example.com/test']
+      end
+
+      it_behaves_like 'visits all available pages'
+    end
+
+    context 'adding all links to the whitelist' do
+      before do
+        crawler.whitelist(['/trusmis', '/help'])
+      end
+      let(:visited_pages_count) {3} #all links
+      let(:visited_pages) do
+        ['http://www.example.com/test','http://www.example.com/trusmis.html', 'http://www.example.com/help.html']
+      end
+
+      it_behaves_like 'visits all available pages'
     end
   end
 
-  context 'the url has several links' do
+
+  describe '#blacklist' do
     let(:body) do
       "<html><head></head><body>
       <a href=\"/trusmis.html\">trusmis</a>
@@ -89,18 +209,124 @@ RSpec.describe Grell::Crawler do
       Hello world!
       </body></html>"
     end
+
     before do
       proxy.stub('http://www.example.com/trusmis.html').and_return(body: 'body', code: 200)
       proxy.stub('http://www.example.com/help.html').and_return(body: 'body', code: 200)
     end
 
-    it 'visits all the pages' do
-      crawler.start_crawling(url)
-      expect(crawler.collection.visited_pages.size).to eq(3)
+    context 'using a single string' do
+      before do
+        crawler.blacklist('/trusmis.html')
+      end
+      let(:visited_pages_count) {2}
+      let(:visited_pages) do
+        ['http://www.example.com/test','http://www.example.com/help.html']
+      end
+
+      it_behaves_like 'visits all available pages'
     end
-    it 'has no more pages to discover' do
-      crawler.start_crawling(url)
-      expect(crawler.collection.discovered_pages.size).to eq(0)
+
+    context 'using an array of strings' do
+      before do
+        crawler.blacklist(['/trusmis.html', '/nothere', 'another.html'])
+      end
+      let(:visited_pages_count) {2}
+      let(:visited_pages) do
+        ['http://www.example.com/test','http://www.example.com/help.html']
+      end
+
+      it_behaves_like 'visits all available pages'
+    end
+
+    context 'using a regexp' do
+      before do
+        crawler.blacklist(/\/trusmis\.html/)
+      end
+      let(:visited_pages_count) {2}
+      let(:visited_pages) do
+        ['http://www.example.com/test','http://www.example.com/help.html']
+      end
+
+      it_behaves_like 'visits all available pages'
+    end
+
+    context 'using an array of regexps' do
+      before do
+        crawler.blacklist([/\/trusmis\.html/])
+      end
+      let(:visited_pages_count) {2}
+      let(:visited_pages) do
+        ['http://www.example.com/test','http://www.example.com/help.html']
+      end
+
+      it_behaves_like 'visits all available pages'
+    end
+
+    context 'using an empty array' do
+      before do
+        crawler.blacklist([])
+      end
+      let(:visited_pages_count) {3} #all links
+      let(:visited_pages) do
+        ['http://www.example.com/test','http://www.example.com/trusmis.html', 'http://www.example.com/help.html']
+      end
+
+      it_behaves_like 'visits all available pages'
+    end
+
+    context 'adding all links to the whitelist' do
+      before do
+        crawler.blacklist(['/trusmis', '/help'])
+      end
+      let(:visited_pages_count) {1}
+      let(:visited_pages) do
+        ['http://www.example.com/test']
+      end
+
+      it_behaves_like 'visits all available pages'
+    end
+  end
+
+
+  describe 'Whitelisting and blacklisting' do
+    let(:body) do
+      "<html><head></head><body>
+      <a href=\"/trusmis.html\">trusmis</a>
+      <a href=\"/help.html\">help</a>
+      Hello world!
+      </body></html>"
+    end
+
+    before do
+      proxy.stub('http://www.example.com/trusmis.html').and_return(body: 'body', code: 200)
+      proxy.stub('http://www.example.com/help.html').and_return(body: 'body', code: 200)
+    end
+
+    context 'we blacklist the only whitelisted page' do
+      before do
+        crawler.whitelist('/trusmis.html')
+        crawler.blacklist('/trusmis.html')
+      end
+      let(:visited_pages_count) {1}
+      let(:visited_pages) do
+        ['http://www.example.com/test']
+      end
+
+      it_behaves_like 'visits all available pages'
+    end
+
+    context 'we blacklist none of the whitelisted pages' do
+      before do
+        crawler.whitelist('/trusmis.html')
+        crawler.blacklist('/raistlin.html')
+      end
+      let(:visited_pages_count) {2}
+      let(:visited_pages) do
+        ['http://www.example.com/test', 'http://www.example.com/trusmis.html']
+      end
+
+      it_behaves_like 'visits all available pages'
     end
   end
 

data/spec/lib/page_spec.rb

@@ -9,6 +9,7 @@ RSpec.describe Grell::Page do
   let(:now) {Time.now}
   before do
     allow(Time).to receive(:now).and_return(now)
+    Grell.logger = Logger.new(nil) #avoids noise in rspec output
   end
 
   it "gives access to the url" do

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: grell
 version: !ruby/object:Gem::Version
-  version: 1.3.0
+  version: 1.3.1
 platform: ruby
 authors:
 - Jordi Polo Carres
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-05-07 00:00:00.000000000 Z
+date: 2015-05-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: capybara
@@ -70,30 +70,30 @@ dependencies:
   name: kender
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.2'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '0.2'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '10.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '10.0'
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
@@ -165,7 +165,8 @@ files:
 - spec/lib/reader_spec.rb
 - spec/spec_helper.rb
 homepage: https://github.com/mdsol/grell
-licenses: []
+licenses:
+- MIT
 metadata: {}
 post_install_message: 
 rdoc_options: []
@@ -175,7 +176,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.9.3
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="