redback 0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ad5bb571eee56b563c4d85fc2fd00342d25c8a16
+  data.tar.gz: 451b6ab74b8039579180da7cd1f423ae5c54909d
+SHA512:
+  metadata.gz: 0df94ad9c9cd34bd4e735a0fe25f9750d71762538f7a4c1a1e27e5e491b2e1cd8f6a5827cedc660231a5e254a07b86afad626b91cb26abe740af19875a0bd0b9
+  data.tar.gz: c3b5f59eb773157c74666e694dda602e2a7fc6ec1664f0957397ee654461bec54b4d7a13058a06b2ab439c67a27f2b493b88a245d86290075dfa6f438dc03107

data/bin/redback

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require_relative '../lib/redback.rb'
+
+Redback.new ARGV[0] { |url| puts url }

data/lib/redback.rb

@@ -0,0 +1,171 @@
+require 'rubygems'
+require 'hpricot'
+require 'net/http'
+require 'parallel'
+
+class Redback
+
+  def initialize(url, &each_site)
+    if url =~ /^(([a-zA-Z]|[a-zA-Z][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z]|[A-Za-z][A-Za-z0-9\-]*[A-Za-z0-9])$/
+      url = 'http://' + url
+    end
+
+    @uri = URI.parse(url)
+
+    @pages_hit = 0
+
+    @visited = []
+    @to_visit = []
+
+    @each_site = each_site
+
+    @options = {
+      :ignore_hash => true,
+      :ignore_query_string => false,
+      :search_in_comments => false,
+      :threads => 4,
+      :num_pages => 1000
+    }
+
+    crawl_page(url)
+    spider
+  end
+
+  def queue_link(url)
+    @to_visit << url
+  end
+
+  def crawl_page(url, limit = 10)
+    # Don't crawl a page twice
+    return if @visited.include? url
+
+    # Let's not hit this again
+    @visited << url
+
+    begin
+      uri = URI.parse(URI.encode(url.to_s.strip))
+    rescue
+      return
+    end
+
+    headers = {
+      "User-Agent"     => "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_3) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.43 Safari/537.31",
+      "Accept-Charset" => "ISO-8859-1,utf-8;q=0.7,*;q=0.3",
+      "Accept"         => "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
+    }
+
+    begin
+      req = Net::HTTP::Get.new(uri.path, headers)
+      response = Net::HTTP.start(uri.host, uri.port) { |http| http.request(req) }
+
+      case response
+      when Net::HTTPRedirection
+        return crawl_page(response['location'], limit - 1)
+      when Net::HTTPSuccess
+        doc = Hpricot(response.body)
+      end
+    rescue
+      return
+    end
+
+    @pages_hit += 1
+
+    @each_site.call url
+
+    find_links(doc, url) do |link|
+      next if @visited.include? link
+      next if @to_visit.include? link
+
+      @to_visit << link
+    end
+  end
+
+  def find_links(doc, url)
+    return unless doc.respond_to? 'search'
+
+    begin
+      uri = URI.parse(URI.encode(url.to_s.strip))
+    rescue
+      return
+    end
+
+    hrefs = []
+
+    # Looks like a valid document! Let's parse it for links
+    doc.search("//a[@href]").each do |e|
+      hrefs << e.get_attribute("href")
+    end
+
+    if @options[:search_in_comments]
+      # Let's also look for commented-out URIs
+      doc.search("//comment()").each do |e|
+        e.to_html.scan(/https?:\/\/[^\s\"]*/) { |url| hrefs << url; }
+      end
+    end
+
+    hrefs.each do |href|
+        # Skip mailto links
+        next if href =~ /^mailto:/
+
+        # If we're dealing with a host-relative URL (e.g. <img src="/foo/bar.jpg">), absolutify it.
+        if href.to_s =~ /^\//
+          href = uri.scheme + "://" + uri.host + href.to_s
+        end
+
+        # If we're dealing with a path-relative URL, make it relative to the current directory.
+        unless href.to_s =~ /[a-z]+:\/\//
+          # Take everything up to the final / in the path to be the current directory.
+          if uri.path =~ /\//
+            /^(.*)\//.match(uri.path)
+            path = $1
+          # If we're on the homepage, then we don't need a path.
+          else
+            path = ""
+          end
+
+          href = uri.scheme + "://" + uri.host + path + "/" + href.to_s
+        end
+
+        # At this point, we should have an absolute URL regardless of
+        # its original format.
+
+        # Strip hash links
+        if ( @options[:ignore_hash] )
+          href.gsub!(/(#.*?)$/, '')
+        end
+
+        # Strip query strings
+        if ( @options[:ignore_query_string] )
+          href.gsub!(/(\?.*?)$/, '')
+        end
+
+        begin
+          href_uri = URI.parse(href)
+        rescue
+          # No harm in this — if we can't parse it as a URI, it probably isn't one (`javascript:` links, etc.) and we can safely ignore it.
+          next
+        end
+
+        next if href_uri.host != uri.host
+        next unless href_uri.scheme =~ /^https?$/
+
+        yield href
+    end
+  end
+
+  def spider(&block)
+    Parallel.in_threads(@options[:threads]) { |thread_number|
+        # We've crawled too many pages
+        next if @pages_hit > @options[:num_pages] && @options[:num_pages] >= 0
+
+        while @to_visit.length > 0 do
+          begin
+            url = @to_visit.pop
+          end while ( @visited.include? url )
+
+          crawl_page(url, block)
+        end
+      }
+  end
+end
+

metadata

@@ -0,0 +1,46 @@
+--- !ruby/object:Gem::Specification
+name: redback
+version: !ruby/object:Gem::Version
+  version: '0.1'
+platform: ruby
+authors:
+- Rob Miller
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-04-12 00:00:00.000000000 Z
+dependencies: []
+description: Fetches a URL you give it and recursively searches for all URLs it can
+  find, building up a list of unique URLs on the same hostname.
+email: rob@bigfish.co.uk
+executables:
+- redback
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/redback
+- lib/redback.rb
+homepage: https://github.com/robmiller/redback
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: Spiders a website, pulling out a list of unique URLs.
+test_files: []