cobweb 0.0.62 → 0.0.63

data/README.textile

@@ -1,5 +1,5 @@
 
-h1. Cobweb v0.0.62
+h1. Cobweb v0.0.63
 !https://secure.travis-ci.org/stewartmckee/cobweb.png?branch=master!
 
 h2. Intro 

data/lib/cobweb_links.rb

@@ -28,27 +28,11 @@ class CobwebLinks
   
   # Returns true if the link is matched to an internal_url and not matched to an external_url
   def internal?(link)
-    if @options[:debug]
-      puts "--------------------------------"
-      puts "Link: #{link}"
-      puts "Internal matches"
-      ap @internal_patterns.select{|pattern| link.match(pattern)}
-      puts "External matches"
-      ap @external_patterns.select{|pattern| link.match(pattern)}
-    end
     !@internal_patterns.select{|pattern| link.match(pattern)}.empty? && @external_patterns.select{|pattern| link.match(pattern)}.empty?
   end
   
   # Returns true if the link is matched to an external_url or not matched to an internal_url
   def external?(link)
-    if @options[:debug]
-      puts "--------------------------------"
-      puts "Link: #{link}"
-      puts "Internal matches"
-      ap @internal_patterns.select{|pattern| link.match(pattern)}
-      puts "External matches"
-      ap @external_patterns.select{|pattern| link.match(pattern)}
-    end
     @internal_patterns.select{|pattern| link.match(pattern)}.empty? || !@external_patterns.select{|pattern| link.match(pattern)}.empty?
   end
   

data/lib/cobweb_version.rb

@@ -3,7 +3,7 @@ class CobwebVersion
   
   # Returns a string of the current version
   def self.version
-    "0.0.62"
+    "0.0.63"
   end
 
 end

data/lib/crawl_job.rb

@@ -28,14 +28,14 @@ class CrawlJob
     
     # check we haven't crawled this url before
     unless @redis.sismember "crawled", content_request[:url]
-      content = Cobweb.new(content_request).get(content_request[:url], content_request)
-      if content_request[:url] == @redis.get("original_base_url")
-        puts content
-         @redis.set("crawled_base_url", content[:base_url])
-      end
-      if is_permitted_type(content)
-        # if there is no limit or we're still under it lets get the url
-        if within_crawl_limits?(content_request[:crawl_limit])
+      # if there is no limit or we're still under it lets get the url
+      if within_crawl_limits?(content_request[:crawl_limit])
+        content = Cobweb.new(content_request).get(content_request[:url], content_request)
+        if content_request[:url] == @redis.get("original_base_url")
+          puts content
+           @redis.set("crawled_base_url", content[:base_url])
+        end
+        if is_permitted_type(content)
           begin
             # move the url from the queued list to the crawled list - for both the original url, and the content url (to handle redirects)
             @redis.srem "queued", content_request[:url]
@@ -99,10 +99,10 @@ class CrawlJob
             puts "Crawled: #{@crawl_counter} Limit: #{content_request[:crawl_limit]} Queued: #{@queue_counter} In Progress: #{@crawl_started_counter-@crawl_counter}" if @debug
           end
         else
-          puts "ignoring #{content_request[:url]} as outside of crawl limits." if content_request[:debug]
+          puts "ignoring #{content_request[:url]} as mime_type is #{content[:mime_type]}" if content_request[:debug]
         end
       else
-        puts "ignoring #{content_request[:url]} as mime_type is #{content[:mime_type]}" if content_request[:debug]
+        puts "ignoring #{content_request[:url]} as outside of crawl limits." if content_request[:debug]
       end
       
     else
@@ -122,15 +122,15 @@ class CrawlJob
     
   end
 
-  # Sets the crawl status to 'Crawl Stopped' and enqueues the crawl finished job
+  # Sets the crawl status to 'Crawl Finished' and enqueues the crawl finished job
   def self.finished(content_request)
     # finished
-    if @redis.hget("statistics", "current_status")!= "Crawl Stopped"
+    if @redis.hget("statistics", "current_status")!= "Crawl Finished"
       ap "CRAWL FINISHED  #{content_request[:url]}, #{counters}, #{@redis.get("original_base_url")}, #{@redis.get("crawled_base_url")}" if content_request[:debug]
       @stats.end_crawl(content_request)
       Resque.enqueue(const_get(content_request[:crawl_finished_queue]), @stats.get_statistics.merge({:redis_options => content_request[:redis_options], :crawl_id => content_request[:crawl_id], :source_id => content_request[:source_id], :crawled_base_url => @redis.get("crawled_base_url")}))
     else
-      ap "CRAWL REFINISHED  #{content_request[:url]}, #{counters}" if content_request[:debug]
+      # nothing to report here, we're skipping the remaining urls as we're outside of the crawl limit
     end
   end
   

data/lib/stats.rb

@@ -23,7 +23,7 @@ class Stats
   # Removes the crawl from the running crawls and updates status
   def end_crawl(options)
     @full_redis.srem "cobweb_crawls", options[:crawl_id]
-    @redis.hset "statistics", "current_status", "Crawl Stopped"
+    @redis.hset "statistics", "current_status", "Crawl Finished"
     @redis.del "crawl_details"
   end
   

data/spec/cobweb/cobweb_job_spec.rb

@@ -200,7 +200,7 @@ def wait_for_crawl_finished(crawl_id, timeout=20)
 end
 
 def running?(crawl_id)
-  @stat.get_status != "Crawl Stopped"
+  @stat.get_status != "Crawl Finished"
 end
 
 def clear_queues

data/views/layout.haml

@@ -51,17 +51,17 @@
         %ul
           %li.home
             %a{:href => "#"}
-          %li.break »
+          %li.break >
           %li
             %a{:href => "#"} Menu item
-          %li.break »
+          %li.break >
           %li
             %a{:href => "#"} Menu item
       = yield
 
       .footer
         .split
-          © Copyright
+          Copyright
           %a{:href => "http://www.activeinformationdesign.com"} activeinformationdesign.com
         .split.right
           Powered by

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cobweb
 version: !ruby/object:Gem::Version
-  version: 0.0.62
+  version: 0.0.63
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-07-27 00:00:00.000000000 Z
+date: 2012-08-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: resque
-  requirement: &70305062213520 !ruby/object:Gem::Requirement
+  requirement: &70236896384860 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70305062213520
+  version_requirements: *70236896384860
 - !ruby/object:Gem::Dependency
   name: redis
-  requirement: &70305062213040 !ruby/object:Gem::Requirement
+  requirement: &70236896384380 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70305062213040
+  version_requirements: *70236896384380
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &70305062212160 !ruby/object:Gem::Requirement
+  requirement: &70236896383640 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70305062212160
+  version_requirements: *70236896383640
 - !ruby/object:Gem::Dependency
   name: addressable
-  requirement: &70305062211220 !ruby/object:Gem::Requirement
+  requirement: &70236896382560 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70305062211220
+  version_requirements: *70236896382560
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70305062210040 !ruby/object:Gem::Requirement
+  requirement: &70236896381420 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70305062210040
+  version_requirements: *70236896381420
 - !ruby/object:Gem::Dependency
   name: awesome_print
-  requirement: &70305062208860 !ruby/object:Gem::Requirement
+  requirement: &70236896380260 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +76,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70305062208860
+  version_requirements: *70236896380260
 - !ruby/object:Gem::Dependency
   name: sinatra
-  requirement: &70305062224320 !ruby/object:Gem::Requirement
+  requirement: &70236896395760 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +87,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70305062224320
+  version_requirements: *70236896395760
 - !ruby/object:Gem::Dependency
   name: thin
-  requirement: &70305062223780 !ruby/object:Gem::Requirement
+  requirement: &70236896395220 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,10 +98,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70305062223780
+  version_requirements: *70236896395220
 - !ruby/object:Gem::Dependency
   name: haml
-  requirement: &70305062223120 !ruby/object:Gem::Requirement
+  requirement: &70236896394540 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -109,10 +109,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70305062223120
+  version_requirements: *70236896394540
 - !ruby/object:Gem::Dependency
   name: namespaced_redis
-  requirement: &70305062222400 !ruby/object:Gem::Requirement
+  requirement: &70236896393840 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -120,7 +120,7 @@ dependencies:
         version: 1.0.2
   type: :runtime
   prerelease: false
-  version_requirements: *70305062222400
+  version_requirements: *70236896393840
 description: Cobweb is a web crawler that can use resque to cluster crawls to quickly
   crawl extremely large sites which is much more perofmant than multi-threaded crawlers.  It
   is also a standalone crawler that has a sophisticated statistics monitoring interface