krawler 0.1.1 → 0.1.2

data/lib/krawler/version.rb

@@ -1,3 +1,3 @@
 module Krawler
-  VERSION = "0.1.1"
+  VERSION = "0.1.2"
 end

data/lib/krawler.rb

@@ -2,6 +2,7 @@ require 'krawler/version'
 require 'mechanize'
 require 'timeout'
 require 'uri'
+require 'thread'
 require 'pry'
 
 module Krawler
@@ -9,67 +10,103 @@ module Krawler
   class Base
 
     def initialize(url, options)
-      url = URI(url)
-      @host           = "#{url.scheme}://#{url.host}"
-      @base_path      = url.path
-      @agent          = Mechanize.new
-      @links_to_crawl = [url]
+      @url            = URI(url)
+      @host           = "#{@url.scheme}://#{@url.host}"
+      @base_path      = @url.path
+      @links_to_crawl = [@url.to_s]
       @crawled_links  = []
       @bad_links      = []
       @suspect_links  = []
       @exclude        = options[:exclude]
       @restrict       = options[:restrict]
+      @randomize      = true
+      @max_threads    = 4
+      @mutex          = Mutex.new
+      @agent          = Mechanize.new
     end
   
     def base
       puts "Crawling..."
 
-      while !@links_to_crawl.empty? do
-        crawl_page(@links_to_crawl.pop)
-      end
+      crawl_page(@url, @agent)
+      initialize_threads(@agent)
     
       puts "#{@crawled_links.size} total Good Links"
     
       puts "Bad Links:"
-      @bad_links.each {|link| puts link }
+      @bad_links.each { |link| puts link }
     
       puts "Suspect Links:"
-      @suspect_links.each {|link| puts link}
+      @suspect_links.each { |link| puts link }
+    end
+
+    def initialize_threads
+      threads = []
+      @max_threads.times do |i|
+        threads << Thread.new(i) do
+
+          agent = @agent.dup
+
+          while !@links_to_crawl.empty? do
+            link = @mutex.synchronize {
+              if @randomize
+                @links_to_crawl.slice!(rand(@links_to_crawl.size))
+              else
+                @links_to_crawl.pop
+              end
+            }
+
+            crawl_page(link, agent)
+          end
+        end
+      end
+      
+      threads.each { |t| t.join }
     end
   
-    def crawl_page(link)
+    def crawl_page(link, agent)
       @crawled_links << link
-      puts link
+
       begin
         start = Time.now
-        page = @agent.get(link)
+        page = agent.get(link)
       rescue Mechanize::ResponseCodeError => e
-        puts e
+        @mutex.synchronize { puts e }
         @bad_links << link
         return
       rescue Timeout::Error => e
         @suspect_links << link
         return
       ensure
-        puts "    [#{Time.now - start}s] #{@links_to_crawl.size} links..."
+        @mutex.synchronize {
+          puts link
+          puts "    [#{Time.now - start}s] #{@links_to_crawl.size} links..."
+        }
       end
   
-      return if !page.respond_to?(:links)
-      page.links.each do |new_link|
-        begin
-          new_url = URI(new_link.href)
-          new_link = new_url.to_s
-        rescue ArgumentError # junk link
-          next
-        end
-
-        if (new_link =~ /^#{Regexp.escape(@host)}/) || (new_link =~ /^\//) # don't crawl external domains
-
-          next if @crawled_links.include?(new_link)       # don't crawl what we've alread crawled
-          next if @exclude && new_link =~ /#{@exclude}/   # don't crawl excluded matched paths
-          next if @restrict && (new_url.path !~ /^#{Regexp.escape(@base_path)}/) # don't crawl outside of our restricted base path
-          
-          @links_to_crawl << new_link
+      @mutex.synchronize do
+        return if !page.respond_to?(:links)
+        page.links.each do |new_link|
+          next if new_link.href.nil?
+  
+          # quick scrub known issues
+          new_link = new_link.href.gsub(/ /, '%20')
+  
+          begin
+            new_url = URI(new_link)
+            new_link = new_url.to_s
+          rescue ArgumentError # junk link
+            next
+          end
+  
+          if (new_link =~ /^#{Regexp.escape(@host)}/) || (new_link =~ /^\//) # don't crawl external domains
+  
+            next if @crawled_links.include?(new_link) || @links_to_crawl.include?(new_link)       # don't crawl what we've alread crawled
+            next if @exclude  && new_link =~ /#{@exclude}/   # don't crawl excluded matched paths
+            next if @restrict && (new_url.path !~ /^#{Regexp.escape(@base_path)}/) # don't crawl outside of our restricted base path
+            
+            @links_to_crawl << new_link
+          end
         end
       end
     end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: krawler
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-05-15 00:00:00.000000000 Z
+date: 2012-05-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize
-  requirement: &70155565557800 !ruby/object:Gem::Requirement
+  requirement: &70168780830200 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,7 +21,7 @@ dependencies:
         version: 2.5.0
   type: :runtime
   prerelease: false
-  version_requirements: *70155565557800
+  version_requirements: *70168780830200
 description: Simple little website crawler.
 email:
 - mike@urlgonomics.com