jeremylightsmith-actionsite 0.3 → 0.4

data/lib/action_site/async_link_checker.rb

@@ -0,0 +1,28 @@
+module ActionSite
+  class AsyncLinkChecker < LinkChecker
+    def check(url)
+      url, html = fetch(url)
+      do_check(host_for(url), 1, url)
+    end
+    
+    def do_check(host, level, *urls)
+      return if urls.empty?
+      raise "too many levels down" if level > 20
+      urls = urls.map {|url| clean_url(url) }.
+                  reject {|url| ignore_url?(url, host)}
+      urls -= @checked_urls
+      @checked_urls += urls
+      puts "checking #{level} level(s) down :\n  #{urls.join("\n  ")}\n"
+      
+      child_urls = []
+      urls.map do |url| 
+        Thread.new do
+          url, html = fetch(url)
+          child_urls << links_from(html, url) if local?(host, url)
+        end
+      end.each {|t| t.join}
+      
+      do_check(host, level + 1, *child_urls.flatten)
+    end
+  end
+end

data/lib/action_site/link_checker.rb

@@ -11,37 +11,40 @@ module ActionSite
     end
   
     def check(url, host = nil, indent = "")
-      url = url.sub(/\#.*/, '')
-      return if url.blank?
-      return if @options[:local] && host && !local?(host, url)
+      url = clean_url(url)
+      return if ignore_url?(url, host)
       return if @checked_urls.include?(url)
       @checked_urls << url
       
       puts "#{indent}checking #{url}"
 
-      begin
-        url, html = fetch(url)
-      rescue
-        puts "#{url} not found"
-        raise
-      end
+      url, html = fetch(url)
       
       host ||= host_for(url)
-      doc = Hpricot(html)
-
       if local?(host, url)
-        (doc / "a").each do |link|
-          if child_link = expand_link(url, link["href"])
-            begin
-              check child_link, host, indent + "   "
-            rescue
-              puts "from #{url} as #{link}"
-              raise
-            end
+        links_from(html, url).each do |child_url|
+          begin
+            check child_url, host, indent + "   "
+          rescue
+            puts "from #{url} as #{link}"
+            raise
           end
         end
       end
     end
+    
+    def clean_url(url)
+      url.sub(/\#.*/, '')
+    end
+    
+    def ignore_url?(url, host = nil)
+      url.blank? || (@options[:local] && host && !local?(host, url))
+    end
+    
+    def links_from(html, url)
+      doc = Hpricot(html)
+      (doc / "a").map {|link| expand_link(url, link["href"])}.compact
+    end
   
     def expand_link(from, to)
       case to
@@ -89,6 +92,9 @@ module ActionSite
       else
         response.error!
       end
+    rescue
+      puts "#{url} not found"
+      raise
     end
     
     def host_for(url)

data/lib/action_site.rb

@@ -16,7 +16,8 @@ require 'action_site/extensions/string'
 
 require 'action_site/site'
 require 'action_site/link_checker'
+require 'action_site/async_link_checker'
 
 module ActionSite
-  VERSION = 0.3
+  VERSION = 0.4
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: jeremylightsmith-actionsite
 version: !ruby/object:Gem::Version 
-  version: "0.3"
+  version: "0.4"
 platform: ruby
 authors: 
 - Jeremy Lightsmith
@@ -14,6 +14,7 @@ default_executable:
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: activesupport
+  type: :runtime
   version_requirement: 
   version_requirements: !ruby/object:Gem::Requirement 
     requirements: 
@@ -23,6 +24,7 @@ dependencies:
     version: 
 - !ruby/object:Gem::Dependency 
   name: RedCloth
+  type: :runtime
   version_requirement: 
   version_requirements: !ruby/object:Gem::Requirement 
     requirements: 
@@ -32,6 +34,7 @@ dependencies:
     version: 
 - !ruby/object:Gem::Dependency 
   name: markaby
+  type: :runtime
   version_requirement: 
   version_requirements: !ruby/object:Gem::Requirement 
     requirements: 
@@ -51,6 +54,7 @@ files:
 - Rakefile
 - readme.txt
 - lib/action_site
+- lib/action_site/async_link_checker.rb
 - lib/action_site/context.rb
 - lib/action_site/extensions
 - lib/action_site/extensions/string.rb