rawler 0.1.6 → 0.1.7

data/README.md

@@ -15,6 +15,8 @@ Rawler will only parse pages with content type 'text/html', but it will check fo
                  --log, -l:   Log results to file rawler_log.txt
          --logfile, -o <s>:   Specify logfile, implies --log (default: rawler_log.txt)
                  --css, -c:   Check CSS links
+            --skip, -s <s>:   Skip URLS that match a regexp
+           --iskip, -i <s>:   Skip URLS that match a case insensitive regexp
              --version, -v:   Print version and exit
                 --help, -h:   Show this message
 

data/VERSION

@@ -1 +1 @@
-0.1.6
+0.1.7

data/bin/rawler

@@ -20,6 +20,8 @@ EOS
   opt :log, "Log results to file #{Rawler::Base::DEFAULT_LOGFILE}", :type => :boolean, :default => false
   opt :logfile, "Specify logfile, implies --log", :type => :string, :default => Rawler::Base::DEFAULT_LOGFILE
   opt :css, "Check CSS links", :type => :boolean, :default => false
+  opt :skip, "Skip URLS that match a pattern", :type => :string
+  opt :iskip, "Skip URLS that match a case insensitive pattern", :type => :string
 end
 
 

data/lib/rawler.rb

@@ -12,6 +12,7 @@ module Rawler
   mattr_accessor :username, :password
   mattr_accessor :log, :logfile
   mattr_accessor :css
+  mattr_accessor :skip_url_pattern
 
   autoload :Base, "rawler/base"
   autoload :Crawler, "rawler/crawler"
@@ -26,4 +27,8 @@ module Rawler
 
     @@url = url
   end
+
+  def self.set_skip_pattern(pattern, icase)
+    self.skip_url_pattern = pattern.nil? ? nil : Regexp.new(pattern, icase ? Regexp::IGNORECASE : nil )
+  end
 end

data/lib/rawler/base.rb

@@ -16,6 +16,9 @@ module Rawler
       Rawler.wait     = options[:wait]
       Rawler.css      = options[:css]
 
+      Rawler.set_skip_pattern(options[:skip], false) unless options[:skip].nil?
+      Rawler.set_skip_pattern(options[:iskip], true) unless options[:iskip].nil?
+
       # Using a custom logfile implies logging.
       Rawler.logfile  = options[:logfile] || DEFAULT_LOGFILE
       Rawler.log      = options[:log] || Rawler.logfile != DEFAULT_LOGFILE

data/lib/rawler/crawler.rb

@@ -64,6 +64,10 @@ module Rawler
       Rawler.output.error(message)
     end
 
+    def info(message)
+      Rawler.output.info(message)
+    end
+
     def different_domain?(url_1, url_2)
       URI.parse(url_1).host != URI.parse(url_2).host
     end
@@ -85,7 +89,9 @@ module Rawler
       url.strip!
 
       scheme = URI.parse(url).scheme
-      if ['http', 'https'].include?(scheme)
+      if url =~ Rawler.skip_url_pattern
+        false
+      elsif ['http', 'https'].include?(scheme)
         true
       else
         write("Invalid url - #{url}") unless url =~ SKIP_FORMATS
@@ -93,8 +99,8 @@ module Rawler
       end
 
     rescue URI::InvalidURIError
+      write("Invalid url - #{url}")
       false
-       write("Invalid url - #{url}")
     end
   end
 end

data/rawler.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "rawler"
-  s.version = "0.1.6"
+  s.version = "0.1.7"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Oscar Del Ben"]
-  s.date = "2012-07-17"
+  s.date = "2012-12-02"
   s.description = "Rawler is a tool that crawls the links of your website"
   s.email = "info@oscardelben.com"
   s.executables = ["rawler"]
@@ -25,6 +25,7 @@ Gem::Specification.new do |s|
     "Rakefile",
     "VERSION",
     "bin/rawler",
+    "custom_logfile",
     "lib/rawler.rb",
     "lib/rawler/base.rb",
     "lib/rawler/core_extensions.rb",

data/spec/lib/rawler/crawler_spec.rb

@@ -183,6 +183,46 @@ describe Rawler::Crawler do
       end
     end
 
+    context "skip matches" do
+      let(:url)     { 'http://example.com/path' }
+      let(:crawler) { Rawler::Crawler.new(url) }
+      let(:content) { "<a href=\"http://example.com/search/page:1/\">foo</a><a href=\"http://example.com/search/page:2/\">foo</a>" }
+    
+      before(:each) do
+        Rawler.set_skip_pattern('\/search\/(.*\/)?page:[2-9]', false)
+        register(url, content)
+      end
+    
+      it "should return one links" do
+        crawler.links.length.should eql(1)
+      end
+
+      it "should not report that it's skipping" do
+        crawler.should_not_receive(:write)
+        crawler.links
+      end
+    end
+
+    context "case-insensitive skip matches" do
+      let(:url)     { 'http://example.com/path' }
+      let(:crawler) { Rawler::Crawler.new(url) }
+      let(:content) { "<a href=\"http://example.com/search/page:1/\">foo</a><a href=\"http://example.com/search/page:2/\">foo</a>" }
+    
+      before(:each) do
+        Rawler.set_skip_pattern('\/seArcH\/(.*\/)?PAGE:[2-9]', true)
+        register(url, content)
+      end
+    
+      it "should return one links" do
+        crawler.links.length.should eql(1)
+      end
+
+      it "should not report that it's skipping" do
+        crawler.should_not_receive(:write)
+        crawler.links
+      end
+    end
+
   end
 
   context "content type" do

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rawler
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.1.7
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-07-17 00:00:00.000000000 Z
+date: 2012-12-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -139,6 +139,7 @@ files:
 - Rakefile
 - VERSION
 - bin/rawler
+- custom_logfile
 - lib/rawler.rb
 - lib/rawler/base.rb
 - lib/rawler/core_extensions.rb
@@ -170,7 +171,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -2845352824216855200
+      hash: -477710177479430630
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements: