spieker 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6fadff203628d5b933856d22c19b3b828c61a31b
-  data.tar.gz: 1bb4b613e70657c5824e1685999f85449280ad68
+  metadata.gz: c787dd82625950fc0ce4eca39e86d29c021b5414
+  data.tar.gz: 9825847bfbe9cb7a09961a26f191b88ff8b0ec0a
 SHA512:
-  metadata.gz: 6cf3c371aa3e12dd6753f0b44ee054689af1e596c518c576edfc25655945c3df37ad868e47e38ac5044d1e0313670f0b0f7a7bff7f9a797ef9f45da200b1f8f4
-  data.tar.gz: 163dd14767485e49dc15a299bdc8ad5927a0a0e6fc8b73870a3ca4d2f5fcd7ca68b992a882e996b20b76375af16956c2faa2519d135e79952b40050592b7c06e
+  metadata.gz: 4f50c13bdac9438682de85294c0cad57848d0e7205be4d76d0d4809894b6338cb989ff833ec4e7304bfa9bb092a6ee52d7cfdac9aeb120b258c4dcc11622a59c
+  data.tar.gz: 9dc328ea3b99435490d8620cc65f42604415592c826c8ff56ce897a107ca4f0d63881991ab411a7bacab58940acc893f082362380813e52e593700d85f5bfbb8

data/CHANGELOG

@@ -0,0 +1,2 @@
+0.0.3
+	* Ignore pdf and javascript links

data/Rakefile

@@ -1 +1,10 @@
 require "bundler/gem_tasks"
+
+require 'rake/testtask'
+
+Rake::TestTask.new do |t|
+  t.libs << "test"
+  t.test_files = FileList['test/**/*_test.rb']
+end
+
+task default: :test

data/circle.yml

@@ -0,0 +1,3 @@
+machine:
+  ruby:
+    version: 2.0.0

data/lib/spieker.rb

@@ -1,3 +1,4 @@
 require "spieker/version"
 require "spieker/crawler"
 require "spieker/link_scraper"
+require "spieker/link_validator"

data/lib/spieker/link_scraper.rb

@@ -3,7 +3,6 @@ require 'capybara/poltergeist'
 
 module Spieker
   class LinkScraper
-    LOCAL_LINK_REGEX = /^(?!(http(s)?\:|\/\/)|data\:).*/
 
     include Capybara::DSL
     attr_writer :links
@@ -45,19 +44,10 @@ module Spieker
 
     def cleaned_up_links(links)
       links.select { |link|
-        is_local?(link) && !is_email?(link)
+        LinkValidator.new(link, @url.to_s).valid?
       }.map(&method(:filter_hash)).compact.uniq
     end
 
-    def is_local?(link)
-      link =~ LOCAL_LINK_REGEX ||
-        begin
-          URI.parse(link).hostname == @url.hostname
-      rescue
-        false
-      end
-    end
-
     def filter_hash(link)
       if match = link.match(/(.*)#(.*)$/)
         match[1]
@@ -65,10 +55,6 @@ module Spieker
         link
       end
     end
-
-    def is_email? link
-      link =~ /mailto/
-    end
   end
 end
 

data/lib/spieker/link_validator.rb

@@ -0,0 +1,51 @@
+class Spieker::LinkValidator
+  LOCAL_LINK_REGEX = /^(?!(http(s)?\:|\/\/)).*/
+  
+  def initialize(link, origin)
+    @link = link
+    @origin = origin
+  end
+
+  def valid?
+    assertions.all? { |assert| send(assert) }
+  end
+
+  private
+
+  def assertions
+    private_methods.select { |m| m.to_s.start_with?('assert_') }
+  end
+
+  def assert_is_local
+    link =~ LOCAL_LINK_REGEX ||
+      begin
+        URI.parse(link).hostname == origin.hostname
+      rescue
+        false
+      end
+  end
+
+  def assert_is_not_email
+    link !~ /mailto/
+  end
+
+  def assert_is_not_javascript
+    link !~ /javascript/
+  end
+
+  def assert_is_not_pdf
+    link !~ /\bpdf\b/
+  end
+
+  def assert_is_not_data
+    link !~ /^data/
+  end
+
+  def link
+    @link
+  end
+
+  def origin
+    URI.parse @origin.start_with?('http') ? @origin : 'http://' + @origin
+  end
+end

data/lib/spieker/version.rb

@@ -1,3 +1,3 @@
 module Spieker
-  VERSION = "0.0.2"
+  VERSION = "0.0.3"
 end

data/spieker.gemspec

@@ -20,6 +20,7 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "bundler", "~> 1.3"
   spec.add_development_dependency "rake"
+  spec.add_development_dependency "pry"
 
   spec.add_dependency "capybara"
   spec.add_dependency "poltergeist"

data/test/link_scraper_test.rb

@@ -1,5 +1,4 @@
-require 'test/unit'
-require 'spieker.rb'
+require 'test_helper'
 
 class TestLinkScraper < Test::Unit::TestCase
   def setup
@@ -22,7 +21,9 @@ class TestLinkScraper < Test::Unit::TestCase
       '/local#justahash',
       '#justahash',
       'http://www.remote.com',
-      'mailto:timonv@gmail.com'
+      'mailto:timonv@gmail.com',
+      'javascript',
+      'pdf'
     ]
     expected_links = [
       'http://www.google.com/local',

data/test/link_validator_test.rb

@@ -0,0 +1,33 @@
+require 'test_helper'
+
+class TestLinkValidator < Test::Unit::TestCase
+  def is_valid?(link)
+    @validator = Spieker::LinkValidator.new(link, 'http://www.google.com').valid?
+  end
+
+  def test_normal_local_link
+    [
+    'http://www.google.com/local',
+      '/local',
+      ''
+    ].each do |link|
+      assert is_valid?(link), "Expected #{link} to be valid"
+    end
+  end
+
+  def test_email
+    assert !is_valid?('mailto:timonv@gmail.com')
+  end
+
+  def test_javascript
+    assert !is_valid?('javascript')
+  end
+
+  def test_pdf
+    assert !is_valid?('pdf')
+  end
+
+  def test_data
+    assert !is_valid?('data')
+  end
+end

data/test/spieker_test.rb

@@ -1,6 +1,4 @@
-require 'test/unit'
-
-require 'spieker.rb'
+require 'test_helper'
 
 class TestSpieker < Test::Unit::TestCase
   def setup
@@ -10,8 +8,4 @@ class TestSpieker < Test::Unit::TestCase
   def test_current_path
     assert_equal '/path', @crawler.current_path
   end
-
-  def test_all_links
-    assert_equal [], @crawler.current_links
-  end
 end

data/test/test_helper.rb

@@ -0,0 +1,3 @@
+require 'test/unit'
+require 'spieker'
+require 'pry'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: spieker
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - Timon Vonk
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-07-25 00:00:00.000000000 Z
+date: 2013-08-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -38,6 +38,20 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: capybara
   requirement: !ruby/object:Gem::Requirement
@@ -89,18 +103,23 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- CHANGELOG
 - Gemfile
 - LICENSE.txt
 - README.md
 - Rakefile
 - bin/spieker
+- circle.yml
 - lib/spieker.rb
 - lib/spieker/crawler.rb
 - lib/spieker/link_scraper.rb
+- lib/spieker/link_validator.rb
 - lib/spieker/version.rb
 - spieker.gemspec
 - test/link_scraper_test.rb
+- test/link_validator_test.rb
 - test/spieker_test.rb
+- test/test_helper.rb
 homepage: ''
 licenses:
 - MIT
@@ -127,4 +146,7 @@ specification_version: 4
 summary: Easilly crawl a website
 test_files:
 - test/link_scraper_test.rb
+- test/link_validator_test.rb
 - test/spieker_test.rb
+- test/test_helper.rb
+has_rdoc: