spieker 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 5b4522f86a2664e3028ed3193ba87f782dbdcb95
+  data.tar.gz: d110801ec5b674af967fe84a398176ce467b3a2b
+SHA512:
+  metadata.gz: 87176bd62d604ac693772fb7dc761b0807c8776142470049e9ea729a8f1230cbee03010a40661d9e887f53f6ce8e2a3c21810c12e47e3aeb96298dc71334f324
+  data.tar.gz: c4f41f4346cba9e79ec7a6511f1dd3d7f6c7991668392d911a40a8b5f3570f6432bdb5c793f7123315c94b324a6511e61d0f7fd94a08b720fc71162a8a7fc333

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in crawler.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Timon Vonk
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/spieker

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require 'crawler'
+
+Spieker::Crawler.new(ARGV[0], verbose: true).crawl!

data/lib/spieker/crawler.rb

@@ -0,0 +1,64 @@
+module Spieker
+  class Crawler
+    def initialize(url, verbose: false)
+      @url = url
+      @tracked_links = []
+      @verbose = verbose
+    end
+
+    def crawl!
+      report "Starting to crawl on #{@url}"
+
+      scraper = LinkScraper.new(@url)
+      track_link(@url)
+      links =  scraper.result
+      recursively_crawl(links)
+
+      print_results if @verbose
+    end
+
+    def current_path
+      URI.parse(@url).path
+    end
+
+    private
+    def recursively_crawl(links)
+      new_links_threaded = {}
+      links.each do |link|
+        report "Crawling page #{link}"
+
+        scraper = LinkScraper.new(link)
+        new_links_threaded[link] = scraper.result
+        track_link(link)
+
+        report "Finished page #{link}, #{new_links_threaded[link].length} links found"
+      end
+      new_links = select_untracked_links(new_links_threaded.values.flatten.uniq)
+      report "Recursively crawling #{new_links.length} links ..."
+      report "NEW LINKS FOUND: \n#{new_links.join("\n")}"
+      report "TRACKED LINKS: \n#{@tracked_links.join("\n")}"
+
+      recursively_crawl(new_links) if new_links.any?
+    end
+
+    def select_untracked_links(links)
+      links.select { |l| !@tracked_links.include?(l) }
+    end
+
+    def track_link(link)
+      @tracked_links = @tracked_links.push(link)
+    end
+
+    def report(text)
+      if @verbose
+        puts text
+      end
+    end
+
+    def print_results
+      report "\n\n:::RESULTS:::\n\n"
+      report "Pages found #{@tracked_links.uniq.compact.length}\n\n"
+      report "All links found:\n\n #{@tracked_links.compact.join("\n")}"
+    end
+  end
+end

data/lib/spieker/link_scraper.rb

@@ -0,0 +1,77 @@
+require 'capybara'
+require 'capybara/poltergeist'
+
+module Spieker
+  class LinkScraper
+    LOCAL_LINK_REGEX = /^(?!(http(s)?\:|\/\/)|data\:).*/
+
+    include Capybara::DSL
+    attr_writer :links
+
+    def initialize(url)
+      @url = URI.parse(url)
+      Capybara.app_host = app_host
+
+      Capybara.register_driver :poltergeist do |app|
+          Capybara::Poltergeist::Driver.new(app, phantomjs_logger: NullStream.new )
+      end
+
+      Capybara.current_driver = :selenium
+    end
+
+    def result
+      cleaned_up_links(found_links)
+    end
+
+    def app_host
+      "#{@url.scheme}://#{@url.hostname}"
+    end
+
+    private
+
+    def found_links
+      @links ||= drive_page_for_links 
+    end
+
+    def drive_page_for_links
+      begin
+        visit @url.path
+        page.all('a').map { |el| el[:href]}
+      rescue
+        puts "Error parsing #{@url.to_s}"
+        []
+      end
+    end
+
+    def cleaned_up_links(links)
+      links.select { |link|
+        is_local?(link) && !is_email?(link)
+      }.map(&method(:filter_hash)).compact.uniq
+    end
+
+    def is_local?(link)
+      link =~ LOCAL_LINK_REGEX ||
+        begin
+          URI.parse(link).hostname == @url.hostname
+      rescue
+        false
+      end
+    end
+
+    def filter_hash(link)
+      if match = link.match(/(.*)#(.*)$/)
+        match[1]
+      else
+        link
+      end
+    end
+
+    def is_email? link
+      link =~ /mailto/
+    end
+  end
+end
+
+class NullStream
+  def puts; end
+end

data/lib/spieker/version.rb

@@ -0,0 +1,3 @@
+module Spieker
+  VERSION = "0.0.1"
+end

data/lib/spieker.rb

@@ -0,0 +1,3 @@
+require "spieker/version"
+require "spieker/instance"
+require "spieker/link_scraper"

data/spieker.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'spieker/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "spieker"
+  spec.version       = Spieker::VERSION
+  spec.authors       = ["Timon Vonk"]
+  spec.email         = ["timonv@gmail.com"]
+  spec.description   = %q{Easilly crawl a website}
+  spec.summary       = %q{Easilly crawl a website}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+
+  spec.add_dependency "capybara"
+  spec.add_dependency "poltergeist"
+  spec.add_dependency "selenium-webdriver"
+end

data/test/link_scraper_test.rb

@@ -0,0 +1,36 @@
+require 'test/unit'
+require 'spieker.rb'
+
+class TestLinkScraper < Test::Unit::TestCase
+  def setup
+    @scraper = Spieker::LinkScraper.new('http://www.google.com')
+  end
+
+  def test_apphost
+    assert_equal 'http://www.google.com', @scraper.app_host
+  end
+
+  def test_result
+    return
+    assert @scraper.result.any?
+  end
+
+  def test_filter_local
+    found_links = [
+      'http://www.google.com/local',
+      '/local',
+      '/local#justahash',
+      '#justahash',
+      'http://www.remote.com',
+      'mailto:timonv@gmail.com'
+    ]
+    expected_links = [
+      'http://www.google.com/local',
+      '/local',
+      ''
+    ]
+
+    @scraper.links = found_links
+    assert_equal expected_links, @scraper.result
+  end
+end

data/test/spieker_test.rb

@@ -0,0 +1,17 @@
+require 'test/unit'
+
+require 'spieker.rb'
+
+class TestSpieker < Test::Unit::TestCase
+  def setup
+    @crawler = Spieker::Crawler.new('http://www.google.com/path')
+  end
+
+  def test_current_path
+    assert_equal '/path', @crawler.current_path
+  end
+
+  def test_all_links
+    assert_equal [], @crawler.current_links
+  end
+end

metadata

@@ -0,0 +1,130 @@
+--- !ruby/object:Gem::Specification
+name: spieker
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Timon Vonk
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-07-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: capybara
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: poltergeist
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: selenium-webdriver
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Easilly crawl a website
+email:
+- timonv@gmail.com
+executables:
+- spieker
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/spieker
+- lib/spieker.rb
+- lib/spieker/crawler.rb
+- lib/spieker/link_scraper.rb
+- lib/spieker/version.rb
+- spieker.gemspec
+- test/link_scraper_test.rb
+- test/spieker_test.rb
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: Easilly crawl a website
+test_files:
+- test/link_scraper_test.rb
+- test/spieker_test.rb