site_mapper 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: c4123dd260ed51448c09ecca619fd3bf9733cb3b
+  data.tar.gz: f2fbb4cfc682dec7da9188048e89856326d68b9e
+SHA512:
+  metadata.gz: 87cd845d828c07778a62bf428cad6b66e0cc9a74fa5b43a7d0676a020d6c282459b7f488d33a02f14a00125b59105cfb9b53e1db6587c5ea9d7dfc75a07e5034
+  data.tar.gz: c62359abc64089d6a8f6e8672657bdf2b56b6dfca747dd831790f9b0dd6ba8acc6b1094aa1e61effd9ffa86691367a9055b229809095809afcc89c323e1ab1c2

data/bin/site_mapper

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+require 'site_mapper'
+
+url = ARGV[0]
+SiteMapper.map(url)

data/lib/site_mapper/crawl_url.rb

@@ -0,0 +1,74 @@
+module SiteMapper
+  class CrawlUrl
+    attr_reader :resolved_base_url, :base_hostname
+
+    def initialize(base_url)
+      @resolved_base_url = Request.resolve_url(base_url)
+      @base_hostname     = URI.parse(@resolved_base_url).hostname
+      @resolved_base_url.prepend('http://') unless @resolved_base_url.start_with?('http')
+    end
+
+    def absolute_url_from(raw_url, get_url)
+      return nil unless eligible_url?(raw_url)
+      parsed_url = URI.parse(raw_url) rescue URI.parse('')  
+      if parsed_url.relative?
+        url_from_relative(raw_url, get_url)
+      elsif same_domain?(raw_url, @resolved_base_url)
+        raw_url
+      else
+        nil
+      end
+    end
+
+    private
+
+    def url_from_relative(url, current_page_url)
+      if url.start_with?('/')
+        "#{without_path_suffix(resolved_base_url)}#{url}"
+      elsif url.start_with?('../')
+        "#{url_from_dotted_url(url, current_page_url)}"
+      else
+        "#{with_path_suffix(resolved_base_url)}#{url}"
+      end
+    end
+
+    def url_from_dotted_url(url, current_page_url)
+      absolute_url = with_path_suffix(current_page_url.dup)
+      found_dots   = without_path_suffix(url).scan('../').length
+      removed_dots = 0
+      max_levels   = 4
+      while found_dots >= removed_dots && max_levels > removed_dots
+        index = absolute_url.rindex('/') or break
+        absolute_url = absolute_url[0..(index - 1)]
+        removed_dots += 1
+      end
+      "#{with_path_suffix(absolute_url)}#{url.gsub('../', '')}"
+    end
+
+    def with_path_suffix(passed_url)
+      url = passed_url.dup
+      url.end_with?('/') ? url : url << '/'
+    end
+
+    def without_path_suffix(passed_url)
+      url = passed_url.dup
+      url.end_with?('/') ? url[0...(url.length - 1)] : url
+    end
+
+    def eligible_url?(href)
+      return false if href.nil? || href.empty?
+      dont_start   = %w(javascript: callto: mailto: tel: skype: facetime: wtai: #)
+      dont_include = %w(/email-protection#)
+      dont_end     = %w(.zip .rar .pdf .exe .dmg .pkg .dpkg .bat)
+
+      dont_start.each   { |pattern| return false if href.start_with?(pattern) }      
+      dont_include.each { |pattern| return false if href.include?(pattern) }
+      dont_end.each     { |pattern| return false if href.end_with?(pattern) }
+      true
+    end
+
+    def same_domain?(first, second)
+      first.include?(second)
+    end
+  end
+end

data/lib/site_mapper/crawler.rb

@@ -0,0 +1,56 @@
+require 'set'
+require 'nokogiri'
+
+module SiteMapper
+  class Crawler
+    CRAWLER_INFO_LINK = 'https://rubygems.org/gems/wayback_archiver'
+    HEADERS_HASH      = {
+      'User-Agent' => "SiteMapper/#{SiteMapper::VERSION} (+#{CRAWLER_INFO_LINK})"
+    }
+
+    def initialize(url, resolve = false)
+      base_url     = Request.resolve_url(url)
+      @options     = { resolve: resolve }
+      @crawl_url   = CrawlUrl.new(base_url)
+      @fetch_queue = Set.new
+      @processed   = Set.new
+      @fetch_queue << @crawl_url.resolved_base_url
+    end
+
+    def self.collect_urls(base_url)
+      new(base_url).collect_urls
+    end
+
+    def collect_urls
+      until @fetch_queue.empty?
+        url = @fetch_queue.first
+        @fetch_queue.delete(@fetch_queue.first)
+        page_links(url)
+      end
+      puts "Crawling finished, #{@processed.length} links found"
+      @processed.to_a
+    rescue Interrupt, IRB::Abort
+      puts 'Crawl interrupted.'
+      @fetch_queue.to_a
+    end
+
+    private
+
+    def page_links(get_url)
+      puts "Queue length: #{@fetch_queue.length}, Parsing: #{get_url}"
+      link_elements = Request.get_page(get_url).css('a') rescue []
+      @processed << get_url
+      link_elements.each do |page_link|
+        absolute_url = @crawl_url.absolute_url_from(page_link.attr('href'), get_url)
+        if absolute_url
+          resolved_url = resolve(absolute_url)
+          @fetch_queue << resolved_url if !@processed.include?(resolved_url)
+        end
+      end
+    end
+
+    def resolve(url)
+      @options[:resolve] ? Request.resolve_url(url) : url
+    end
+  end
+end

data/lib/site_mapper/request.rb

@@ -0,0 +1,29 @@
+require 'url_resolver' # TODO: Allow users to use any resolver
+
+module SiteMapper
+  class Request
+    INFO_LINK  = 'https://rubygems.org/gems/site_mapper'
+    USER_AGENT = "SiteMapper/#{VERSION} (+#{INFO_LINK})"
+
+    class << self
+      def get_page(url, document_type = :html)
+        Nokogiri::HTML(Request.get_response(url).body)
+      end
+
+      def get_response(url, resolve = false)
+        resolved_url = resolve ? resolve_url(url) : url
+        uri          = URI.parse(resolved_url)
+        http         = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = true if resolved_url.include?('https://')
+
+        request = Net::HTTP::Get.new(uri.request_uri)
+        request['User-Agent'] = USER_AGENT
+        http.request(request)
+      end
+
+      def resolve_url(url)
+        UrlResolver.resolve(url)
+      end
+    end
+  end
+end

data/lib/site_mapper/version.rb

@@ -0,0 +1,3 @@
+module SiteMapper
+  VERSION = '0.0.1'
+end

data/lib/site_mapper.rb

@@ -0,0 +1,12 @@
+require 'uri'
+require 'net/http'
+
+require 'site_mapper/request'
+require 'site_mapper/crawler'
+require 'site_mapper/crawl_url'
+
+module SiteMapper
+  def self.map(source)
+    Crawler.collect_urls(source)
+  end
+end

metadata

@@ -0,0 +1,150 @@
+--- !ruby/object:Gem::Specification
+name: site_mapper
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Jacob Burenstam
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-10-22 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: url_resolver
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.1'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '10.3'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.1'
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.8'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.8'
+- !ruby/object:Gem::Dependency
+  name: coveralls
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.7'
+description: Find all links on domain.
+email:
+- burenstam@gmail.com
+executables:
+- site_mapper
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/site_mapper
+- lib/site_mapper.rb
+- lib/site_mapper/crawl_url.rb
+- lib/site_mapper/crawler.rb
+- lib/site_mapper/request.rb
+- lib/site_mapper/version.rb
+homepage: https://github.com/buren/site_mapper
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: 1.9.3
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Find all links on domain
+test_files: []
+has_rdoc: