cobweb 0.0.55 → 0.0.57

data/README.textile

@@ -1,5 +1,6 @@
 
-h1. Cobweb v0.0.55
+h1. Cobweb v0.0.57
+!https://secure.travis-ci.org/stewartmckee/cobweb.png?branch=master!
 
 h2. Intro 
   

data/lib/cobweb.rb

@@ -56,7 +56,9 @@ class Cobweb
     
     if @options[:internal_urls].nil? || @options[:internal_urls].empty?
       uri = Addressable::URI.parse(base_url)
-      @options[:internal_urls] = [[uri.scheme, "://", uri.host, "/*"].join]
+      @options[:internal_urls] = []
+      @options[:internal_urls] << [uri.scheme, "://", uri.host, "/*"].join
+      @options[:internal_urls] << [uri.scheme, "://", uri.host, ":", uri.inferred_port, "/*"].join
     end
     
     request.merge!(@options)
@@ -72,6 +74,7 @@ class Cobweb
     @options[:internal_urls].map{|url| @redis.sadd("internal_urls", url)}
     
     Resque.enqueue(CrawlJob, request)
+    request
   end
   
   # Returns array of cookies from content

data/lib/cobweb_crawler.rb

@@ -20,7 +20,7 @@ class CobwebCrawler
       @options[:crawl_id] = @crawl_id
     end
     
-    @redis = NamespacedRedis.new(@options[:redis_options], "cobweb-#{@crawl_id}")
+    @redis = NamespacedRedis.new(@options[:redis_options], "cobweb-#{Cobweb.version}-#{@crawl_id}")
     @options[:internal_urls] = [] if @options[:internal_urls].nil?
     @options[:internal_urls].map{|url| @redis.sadd("internal_urls", url)}
     @debug = @options[:debug]
@@ -42,9 +42,6 @@ class CobwebCrawler
     
     @crawl_options = crawl_options
     
-    puts "http://localhost:4567/statistics/#{@crawl_id}"
-    puts ""
-    
     @redis.sadd("queued", base_url) unless @redis.sismember("crawled", base_url) || @redis.sismember("queued", base_url)
     crawl_counter = @redis.scard("crawled").to_i
     queue_counter = @redis.scard("queued").to_i

data/lib/cobweb_version.rb

@@ -3,7 +3,7 @@ class CobwebVersion
   
   # Returns a string of the current version
   def self.version
-    "0.0.55"
+    "0.0.57"
   end
 
 end

data/lib/content_link_parser.rb

@@ -46,6 +46,7 @@ class ContentLinkParser
     data = link_data
     links = data.keys.map{|key| data[key]}.flatten.uniq
     links = links.map{|link| UriHelper.join_no_fragment(@url, link).to_s }
+    links = links.reject{|link| link =~ /\/([^\/]+?)\/\1\// }
     links = links.reject{|link| link =~ /([^\/]+?)\/([^\/]+?)\/.*?\1\/\2/ }    
     links = links.select{|link| options[:valid_schemes].include? link.split(':')[0].to_sym}
     links

data/lib/crawl_job.rb

@@ -36,6 +36,7 @@ class CrawlJob
         content = Cobweb.new(content_request).get(content_request[:url], content_request)
         
         ## update statistics
+        @stats.update_status("Crawling #{content_request[:url]}...")
         @stats.update_statistics(content)
         
         # set the base url if this is the first page
@@ -72,7 +73,7 @@ class CrawlJob
           if @redis.scard("queued") == 0
             finished(content_request)
           end
-        elsif @queue_counter == 0 || @crawl_counter >= content_request[:crawl_limit].to_i
+        elsif @queue_counter == 0 || @crawl_counter > content_request[:crawl_limit].to_i
           finished(content_request)
         end
       end
@@ -109,12 +110,12 @@ class CrawlJob
   
   # Returns true if the crawl count is within limits
   def self.within_crawl_limits?(crawl_limit)
-    crawl_limit.nil? or @crawl_counter < crawl_limit.to_i
+    crawl_limit.nil? or @crawl_counter <= crawl_limit.to_i
   end
   
   # Returns true if the queue count is calculated to be still within limits when complete
   def self.within_queue_limits?(crawl_limit)
-    within_crawl_limits?(crawl_limit) and (crawl_limit.nil? or (@queue_counter + @crawl_counter) < crawl_limit.to_i)
+    within_crawl_limits?(crawl_limit) && (crawl_limit.nil? || (@queue_counter + @crawl_counter) < crawl_limit.to_i)
   end
   
   # Sets the base url in redis.  If the first page is a redirect, it sets the base_url to the destination

data/lib/server.rb

@@ -17,7 +17,7 @@ class Server < Sinatra::Base
     
     @crawls = []
     @full_redis.smembers("cobweb_crawls").each do |crawl_id|
-      redis = NamespacedRedis.new({}, "cobweb-#{crawl_id}")
+      redis = NamespacedRedis.new({}, "cobweb-#{Cobweb.version}-#{crawl_id}")
       stats = HashUtil.deep_symbolize_keys({
         :crawl_details => redis.hgetall("crawl_details"), 
         :statistics => redis.hgetall("statistics"),
@@ -31,7 +31,7 @@ class Server < Sinatra::Base
   
   # Sinatra Crawl Detail
   get '/statistics/:crawl_id' do
-    redis = NamespacedRedis.new({}, "cobweb-#{params[:crawl_id]}")
+    redis = NamespacedRedis.new({}, "cobweb-#{Cobweb.version}-#{params[:crawl_id]}")
     
     @statistics = HashUtil.deep_symbolize_keys(redis.hgetall("statistics"))
     if @statistics[:status_counts].nil?

data/lib/stats.rb

@@ -4,8 +4,9 @@ class Stats
   
   # Sets up redis usage for statistics
   def initialize(options)
+    options[:redis_options] = {} unless options.has_key? :redis_options
     @full_redis = Redis.new(options[:redis_options])
-    @redis = NamespacedRedis.new(options[:redis_options], "cobweb-#{options[:crawl_id]}")
+    @redis = NamespacedRedis.new(options[:redis_options], "cobweb-#{Cobweb.version}-#{options[:crawl_id]}")
   end
   
   # Sets up the crawl in statistics
@@ -26,6 +27,10 @@ class Stats
     @redis.del "crawl_details"
   end
   
+  def get_crawled
+    @redis.smembers "crawled"
+  end
+  
   # Returns statistics hash.  update_statistics takes the content hash, extracts statistics from it and updates redis with the data.  
   def update_statistics(content, crawl_counter=@redis.scard("crawled").to_i, queue_counter=@redis.scard("queued").to_i)
     

data/spec/cobweb/cobweb_job_spec.rb

@@ -1,41 +1,179 @@
 require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
 
-describe Cobweb do
+describe Cobweb, :local_only => true do
+
+  before(:all) do
+    #store all existing resque process ids
+    @existing_processes = `ps aux | grep resque | grep -v grep | grep -v resque-web | awk '{print $2}'`.split("\n")
+  
+    # START WORKERS ONLY FOR CRAWL QUEUE SO WE CAN COUNT ENQUEUED PROCESS AND FINISH QUEUES
+    puts "Starting Workers... Please Wait..."
+    `mkdir log`
+    io = IO.popen("nohup rake resque:workers PIDFILE=./tmp/pids/resque.pid COUNT=5 QUEUE=cobweb_crawl_job > log/output.log &")
+    puts "Workers Started."
+  
+    # START THIN SERVER TO HOST THE SAMPLE SITE FOR CRAWLING
+    @thin = nil
+    Thread.new do
+      @thin = Thin::Server.start("0.0.0.0", 3532, SampleServer.app)
+    end
+  
+    # WAIT FOR START TO COMPLETE
+    sleep 1
+  
+  end
 
   before(:each) do
-    @base_url = "http://www.baseurl.com/"
-    @cobweb = Cobweb.new :quiet => true, :cache => nil
+    @base_url = "http://localhost:3532/"
+    @base_page_count = 77
+  
+    clear_queues
   end
 
-  describe "detect finish of crawl" do
-    
-    describe "with no crawl limit" do
-      before(:each) do
-        @request = {
-          :crawl_id => Digest::SHA1.hexdigest("#{Time.now.to_i}.#{Time.now.usec}"),
-          :url => @base_url,
-          :crawl_limit => nil
-        }
-      end
-      
-      it "should not limit crawl"
-      it "should detect the end or crawl"
+  describe "with no crawl limit" do
+    before(:each) do
+      @request = {
+        :crawl_id => Digest::SHA1.hexdigest("#{Time.now.to_i}.#{Time.now.usec}"),
+        :crawl_limit => nil,
+        :quiet => false,
+        :debug => false,
+        :cache => nil
+      }
+      @cobweb = Cobweb.new @request
     end
-    
-    describe "with a crawl limit" do
+  
+    it "should crawl entire site" do
+      crawl = @cobweb.start(@base_url)
+      @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
+      wait_for_crawl_finished crawl[:crawl_id]
+      Resque.size("cobweb_process_job").should == @base_page_count
+    end
+    it "detect crawl finished" do
+      crawl = @cobweb.start(@base_url)
+      @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
+      wait_for_crawl_finished crawl[:crawl_id]
+      Resque.size("cobweb_finished_job").should == 1
+    end
+  end
+  
+  describe "with a crawl limit" do
+    before(:each) do
       @request = {
         :crawl_id => Digest::SHA1.hexdigest("#{Time.now.to_i}.#{Time.now.usec}"),
-        :url => @base_url,
-        :crawl_limit => 3
+        :quiet => true,
+        :cache => nil
       }
+      @cobweb = Cobweb.new @request
+    end
+  
+    describe "limit to 1" do
+      before(:each) do
+        @request[:crawl_limit] = 1
+      end
 
-      it "should limit crawl"
-      it "should detect the end or crawl based on limit"
-
+      it "should not crawl the entire site" do
+        crawl = @cobweb.start(@base_url)
+        @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
+        wait_for_crawl_finished crawl[:crawl_id]
+        Resque.size("cobweb_process_job").should_not == @base_page_count
+      end      
+      it "should only crawl 1 page" do
+        crawl = @cobweb.start(@base_url)
+        @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
+        wait_for_crawl_finished crawl[:crawl_id]
+        Resque.size("cobweb_process_job").should == 1
+      end      
+      it "should notify of crawl finished" do
+        crawl = @cobweb.start(@base_url)
+        @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
+        wait_for_crawl_finished crawl[:crawl_id]
+        Resque.size("cobweb_finished_job").should == 1
+      end      
+    
     end
+  
+    describe "limit to 3" do
+      before(:each) do
+        @request[:crawl_limit] = 3
+      end
+      it "should not crawl the entire site" do
+        crawl = @cobweb.start(@base_url)
+        @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
+        wait_for_crawl_finished crawl[:crawl_id]
+        Resque.size("cobweb_process_job").should_not == @base_page_count
+      end      
+      it "should notify of crawl finished" do
+        crawl = @cobweb.start(@base_url)
+        @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
+        wait_for_crawl_finished crawl[:crawl_id]
+        Resque.size("cobweb_finished_job").should == 1
+      end      
+      it "should only crawl 3 pages" do
+        crawl = @cobweb.start(@base_url)
+        @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
+        wait_for_crawl_finished crawl[:crawl_id]
+        Resque.size("cobweb_process_job").should == 3
+      end      
     
+    end
+  
+    describe "limit to 100" do
+      before(:each) do
+        @request[:crawl_limit] = 100
+      end
     
+      it "should crawl the entire site" do
+        crawl = @cobweb.start(@base_url)
+        @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
+        wait_for_crawl_finished crawl[:crawl_id]
+        Resque.size("cobweb_process_job").should == @base_page_count
+      end      
+      it "should notify of crawl finished" do
+        crawl = @cobweb.start(@base_url)
+        @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
+        wait_for_crawl_finished crawl[:crawl_id]
+        Resque.size("cobweb_finished_job").should == 1
+      end      
+      it "should not crawl 100 pages" do
+        crawl = @cobweb.start(@base_url)
+        @stat = Stats.new({:crawl_id => crawl[:crawl_id]})
+        wait_for_crawl_finished crawl[:crawl_id]
+        Resque.size("cobweb_process_job").should_not == 100
+      end      
+    end    
+  end
+
+  after(:all) do
+    @all_processes = `ps aux | grep resque | grep -v grep | grep -v resque-web | awk '{print $2}'`.split("\n")
+    command = "kill #{(@all_processes - @existing_processes).join(" ")}"
+    IO.popen(command)
+    #@thin.stop!    
+  end
+
+end
+
+def wait_for_crawl_finished(crawl_id, timeout=20)
+  counter = 0 
+  while(running?(crawl_id) && counter < timeout) do
+    sleep 1
+    counter+=1
+  end
+  if counter > timeout
+    raise "End of crawl not detected"
+  end 
+end
+
+def running?(crawl_id)
+  @stat.get_status != "Crawl Stopped"
+end
+
+def clear_queues
+  Resque.queues.each do |queue|
+    Resque.remove_queue(queue)
   end
+  
+  Resque.size("cobweb_process_job").should == 0
+  Resque.size("cobweb_finished_job").should == 0      
+end
 
 
-end

data/spec/samples/sample_server.rb

@@ -0,0 +1,21 @@
+class SampleServer
+
+  # This is the root of our app
+  @root = File.expand_path(File.dirname(__FILE__)) + "/sample_site"
+
+  def self.app
+    Proc.new { |env|
+      # Extract the requested path from the request
+      path = Rack::Utils.unescape(env['PATH_INFO'])
+      index_file = @root + "#{path}/index.html"
+
+      if File.exists?(index_file)
+        # Return the index
+        [200, {'Content-Type' => 'text/html'}, File.read(index_file)]
+      else
+        # Pass the request to the directory app
+        Rack::Directory.new(@root).call(env)
+      end
+    }
+  end
+end

data/spec/samples/sample_site/boxgrid.html

@@ -0,0 +1,258 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
+
+<head>
+	
+	<meta http-equiv="Content-Type" content="text/html;charset=utf-8" /> 
+	<title>CleanDream</title>
+	
+	<style type="text/css">
+		@import url("css/style.css");
+		@import url('css/style_text.css');
+		@import url('css/form-buttons.css');
+		@import url('css/link-buttons.css');
+		@import url('css/menu.css');
+		@import url('css/statics.css');
+		@import url('css/messages.css');
+		@import url('css/datatable.css');
+		@import url('css/accordion.css');
+		@import url('css/tabs.css');
+		@import url('css/forms.css');
+		@import url('css/datepicker.css');
+		@import url('css/modalbox.css');
+		@import url('css/jquery.fancybox-1.3.4.css');
+		@import url('css/jquery.treeview.css');
+		@import url('css/wysiwyg.css');
+		@import url('css/wysiwyg.modal.css');
+		@import url('css/wysiwyg-editor.css');
+	</style>
+	
+	<script type="text/javascript" src="js/jquery-1.7.1.min.js"></script>
+	
+	<!--[if lte IE 8]>
+		<script type="text/javascript" src="js/excanvas.min.js"></script>
+	<![endif]-->
+	
+</head>
+
+<body>
+
+<div class="container">   
+  
+	<div class="logo-labels">
+		<h1><a href="#">CleanDream</a></h1>
+		<ul>
+			<li><a href="#"><span>Settings</span></a></li>
+			<li class="usermessage"><a href="#"><span>1 new message</span></a></li>
+			<li class="logout"><a href="#"><span>Logout</span></a></li>
+		</ul>
+	</div>
+	
+	<div class="menu-search">
+		<ul> 
+			<li><a href="dashboard.html">Dashboard</a></li> 
+			<li> 
+				<a href="#">Dropdown</a> 
+				<ul> 
+					<li><a href="#">Submenu item 001</a></li>
+					<li><a href="#">Submenu item 002</a></li>
+					<li><a href="#">Submenu item 003</a></li>
+					<li class="current">
+						<a href="#">Submenu item 004</a>
+						<ul> 
+							<li><a href="#">Subsubmenu item 001</a></li>
+							<li><a href="#">Subsubmenu item 002</a></li>
+							<li class="current"><a href="#">Subsubmenu item 003</a></li>
+							<li><a href="#">Subsubmenu item 003</a></li>
+							<li><a href="#">Subsubmenu item 005</a></li>
+						</ul> 
+					</li> 
+					<li> <a href="#">Submenu item 005</a> </li>
+				</ul> 
+			</li> 
+			<li><a href="typography.html">Typography</a></li>
+			<li class="current"><a href="boxgrid.html">Boxes Grid</a></li>
+			<li><a href="forms.html">Forms</a></li>
+			<li><a href="gallery.html">Gallery</a></li>
+			<li><a href="tables.html">Tables</a></li>
+			<li><a href="more.html">More</a></li>
+		</ul>
+		<div class="search">
+			<form action="" method="post">
+				<input type="text" value="Seachterm" />
+				<button type="submit"></button>
+			</form> 
+		</div>
+	</div>
+	
+	<div class="breadcrumbs">
+		<ul>
+			<li class="home"><a href="#"></a></li>
+			<li class="break">&#187;</li>
+			<li><a href="#">Menu item</a></li>
+			<li class="break">&#187;</li>
+			<li><a href="#">Menu item</a></li>
+		</ul>
+	</div>
+	
+	<div class="section">
+		<div class="full">
+			<div class="box">
+				<div class="title">
+					<h2>Full box</h2>
+					<span class="hide"></span>
+				</div>
+				<div class="content">Place your content here.</div>
+			</div>
+		</div>
+	</div>
+	
+	<div class="section">
+		<div class="small">
+			<div class="box">
+				<div class="title">
+					<h2>Small box</h2>
+					<span class="hide"></span>
+				</div>
+				<div class="content">Place your content here.</div>
+			</div>
+		</div>
+		
+		<div class="big">
+			<div class="box">
+				<div class="title">
+					<h2>Big box</h2>
+					<span class="hide"></span>
+				</div>
+				<div class="content">Place your content here.</div>
+			</div>
+		</div>
+	</div>
+	
+	<div class="section">
+		<div class="medium">
+			<div class="box">
+				<div class="title">
+					<h2>Medium box</h2>
+					<span class="hide"></span>
+				</div>
+				<div class="content">Place your content here.</div>
+			</div>
+		</div>
+		
+		<div class="medium">
+			<div class="box">
+				<div class="title">
+					<h2>Medium box</h2>
+					<span class="hide"></span>
+				</div>
+				<div class="content">Place your content here.</div>
+			</div>
+		</div>
+	</div>
+	
+	<div class="section">
+		<div class="medium">
+			<div class="box">
+				<div class="title">
+					<h2>Medium box</h2>
+					<span class="hide"></span>
+				</div>
+				<div class="content">Place your content here.</div>
+			</div>
+		</div>
+		
+		<div class="small">
+			<div class="box">
+				<div class="title">
+					<h2>Small box</h2>
+					<span class="hide"></span>
+				</div>
+				<div class="content">Place your content here.</div>
+			</div>
+		</div>
+		
+		<div class="small">
+			<div class="box">
+				<div class="title">
+					<h2>Small box</h2>
+					<span class="hide"></span>
+				</div>
+				<div class="content">Place your content here.</div>
+			</div>
+		</div>
+	</div>
+	
+	<div class="section">
+		<div class="small">
+			<div class="box">
+				<div class="title">
+					<h2>Small box</h2>
+					<span class="hide"></span>
+				</div>
+				<div class="content">Place your content here.</div>
+			</div>
+		</div>
+		
+		<div class="small">
+			<div class="box">
+				<div class="title">
+					<h2>Small box</h2>
+					<span class="hide"></span>
+				</div>
+				<div class="content">Place your content here.</div>
+			</div>
+		</div>
+		
+		<div class="small">
+			<div class="box">
+				<div class="title">
+					<h2>Small box</h2>
+					<span class="hide"></span>
+				</div>
+				<div class="content">Place your content here.</div>
+			</div>
+		</div>
+		
+		<div class="small">
+			<div class="box">
+				<div class="title">
+					<h2>Small box</h2>
+					<span class="hide"></span>
+				</div>
+				<div class="content">Place your content here.</div>
+			</div>
+		</div>
+	</div>
+
+	<div class="footer">
+		<div class="split">&#169; Copyright <a href="#">website.com</a></div> 
+		<div class="split right">Powered by <a href="http://themeforest.net/item/cleandream/490140" target="_blank">CleanDream</a></div> 
+	</div>
+</div>
+
+<script type="text/javascript" src="js/superfish.js"></script>
+<script type="text/javascript" src="js/supersubs.js"></script>
+<script type="text/javascript" src="js/hoverIntent.js"></script>
+<script type="text/javascript" src="js/jquery.flot.js"></script>
+<script type="text/javascript" src="js/jquery.graphtable-0.2.js"></script>
+<script type="text/javascript" src="js/jquery.flot.resize.min.js"></script>
+<script type="text/javascript" src="js/jquery-ui.js"></script>
+<script type="text/javascript" src="js/jquery-ui-select.js"></script>
+<script type="text/javascript" src="js/customInput.jquery.js"></script>
+<script type="text/javascript" src="js/jquery.dataTables.js"></script>
+<script type="text/javascript" src="js/jquery.fancybox-1.3.4.js"></script>
+<script type="text/javascript" src="js/jquery.filestyle.mini.js"></script>
+<script type="text/javascript" src="js/jquery-ui-timepicker-addon.js"></script>
+<script type="text/javascript" src="js/jquery.treeview.js"></script>
+<script type="text/javascript" src="js/jquery.tipsy.js"></script>
+<script type="text/javascript" src="js/jquery.wysiwyg.js"></script>
+<script type="text/javascript" src="js/plugins/wysiwyg.rmFormat.js"></script>
+<script type="text/javascript" src="js/controls/wysiwyg.image.js"></script>
+<script type="text/javascript" src="js/controls/wysiwyg.link.js"></script>
+<script type="text/javascript" src="js/controls/wysiwyg.table.js"></script>
+<script type="text/javascript" src="js/inline.js"></script>
+
+</body>
+
+</html>