cobweb 0.0.63 → 0.0.64

data/README.textile

@@ -1,11 +1,13 @@
 
-h1. Cobweb v0.0.63
+h1. Cobweb v0.0.64
 !https://secure.travis-ci.org/stewartmckee/cobweb.png?branch=master!
 
 h2. Intro 
   
   CobWeb has three methods of running.  Firstly it is a http client that allows get and head requests returning a hash of data relating to the requested resource.  The second main function is to utilize this combined with the power of Resque to cluster the crawls allowing you crawl quickly.  Lastly you can run the crawler with a block that uses each of the pages found in the crawl.
   
+  I've created a sample app to help with setting up cobweb at http://github.com/stewartmckee/cobweb_sample
+  
 h3. Resque
 
   When running on resque, passing in a Class and queue name it will enqueue all resources to this queue for processing, passing in the hash it has generated.  You then implement the perform method to process the resource for your own application.

data/lib/cobweb.rb

@@ -34,8 +34,8 @@ class Cobweb
     default_use_encoding_safe_process_job_to  false
     default_follow_redirects_to               true
     default_redirect_limit_to                 10
-    default_processing_queue_to               CobwebProcessJob
-    default_crawl_finished_queue_to           CobwebFinishedJob
+    default_processing_queue_to               "CobwebProcessJob"
+    default_crawl_finished_queue_to           "CobwebFinishedJob"
     default_quiet_to                          true
     default_debug_to                          false
     default_cache_to                          300

data/lib/cobweb_crawler.rb

@@ -64,7 +64,6 @@ class CobwebCrawler
               @redis.incr "crawl-counter" 
               
               internal_links = ContentLinkParser.new(url, content[:body]).all_links(:valid_schemes => [:http, :https])
-              ap internal_links
 
               # select the link if its internal (eliminate external before expensive lookups in queued and crawled)
               cobweb_links = CobwebLinks.new(@options)

data/lib/cobweb_version.rb

@@ -3,7 +3,7 @@ class CobwebVersion
   
   # Returns a string of the current version
   def self.version
-    "0.0.63"
+    "0.0.64"
   end
 
 end

data/lib/crawl_job.rb

@@ -32,7 +32,6 @@ class CrawlJob
       if within_crawl_limits?(content_request[:crawl_limit])
         content = Cobweb.new(content_request).get(content_request[:url], content_request)
         if content_request[:url] == @redis.get("original_base_url")
-          puts content
            @redis.set("crawled_base_url", content[:base_url])
         end
         if is_permitted_type(content)
@@ -60,7 +59,7 @@ class CrawlJob
 
             @cobweb_links = CobwebLinks.new(content_request)
             if within_queue_limits?(content_request[:crawl_limit])
-              internal_links = ContentLinkParser.new(content_request[:url], content[:body]).all_links(:valid_schemes => [:http, :https])
+              internal_links = ContentLinkParser.new(content_request[:url], content[:body], content_request).all_links(:valid_schemes => [:http, :https])
 
               # select the link if its internal
               internal_links.select! { |link| @cobweb_links.internal?(link) }
@@ -128,7 +127,12 @@ class CrawlJob
     if @redis.hget("statistics", "current_status")!= "Crawl Finished"
       ap "CRAWL FINISHED  #{content_request[:url]}, #{counters}, #{@redis.get("original_base_url")}, #{@redis.get("crawled_base_url")}" if content_request[:debug]
       @stats.end_crawl(content_request)
-      Resque.enqueue(const_get(content_request[:crawl_finished_queue]), @stats.get_statistics.merge({:redis_options => content_request[:redis_options], :crawl_id => content_request[:crawl_id], :source_id => content_request[:source_id], :crawled_base_url => @redis.get("crawled_base_url")}))
+      
+      additional_stats = {:crawl_id => content_request[:crawl_id], :crawled_base_url => @redis.get("crawled_base_url")}
+      additional_stats[:redis_options] = content_request[:redis_options] unless content_request[:redis_options] == {}
+      additional_stats[:source_id] = content_request[:source_id] unless content_request[:source_id].nil?
+      
+      Resque.enqueue(const_get(content_request[:crawl_finished_queue]), @stats.get_statistics.merge(additional_stats))
     else
       # nothing to report here, we're skipping the remaining urls as we're outside of the crawl limit
     end

data/lib/stats.rb

@@ -14,7 +14,7 @@ class Stats
     unless @full_redis.sismember "cobweb_crawls", options[:crawl_id]
       @full_redis.sadd "cobweb_crawls", options[:crawl_id]
       options.keys.each do |key|
-        @redis.hset "crawl_details", key, options[key]
+        @redis.hset "crawl_details", key, options[key].to_s
       end
     end
     @redis.hset "statistics", "current_status", "Crawl Starting..."

data/spec/cobweb/cobweb_job_spec.rb

@@ -3,7 +3,7 @@ require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
 describe Cobweb, :local_only => true do
 
   before(:all) do
-    #store all existing resque process ids
+    #store all existing resque process ids so we don't kill them afterwards
     @existing_processes = `ps aux | grep resque | grep -v grep | grep -v resque-web | awk '{print $2}'`.split("\n")
   
     # START WORKERS ONLY FOR CRAWL QUEUE SO WE CAN COUNT ENQUEUED PROCESS AND FINISH QUEUES

data/spec/cobweb/cobweb_spec.rb

@@ -18,8 +18,8 @@ describe Cobweb do
     
     options[:follow_redirects].should == true
     options[:redirect_limit].should == 10
-    options[:processing_queue].should == CobwebProcessJob
-    options[:crawl_finished_queue].should == CobwebFinishedJob
+    options[:processing_queue].should == "CobwebProcessJob"
+    options[:crawl_finished_queue].should == "CobwebFinishedJob"
     options[:quiet].should == true
     options[:debug].should == false
     options[:cache].should == 300

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cobweb
 version: !ruby/object:Gem::Version
-  version: 0.0.63
+  version: 0.0.64
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-08-07 00:00:00.000000000 Z
+date: 2012-08-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: resque
-  requirement: &70236896384860 !ruby/object:Gem::Requirement
+  requirement: &70160920762100 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70236896384860
+  version_requirements: *70160920762100
 - !ruby/object:Gem::Dependency
   name: redis
-  requirement: &70236896384380 !ruby/object:Gem::Requirement
+  requirement: &70160920761600 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70236896384380
+  version_requirements: *70160920761600
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &70236896383640 !ruby/object:Gem::Requirement
+  requirement: &70160920760740 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70236896383640
+  version_requirements: *70160920760740
 - !ruby/object:Gem::Dependency
   name: addressable
-  requirement: &70236896382560 !ruby/object:Gem::Requirement
+  requirement: &70160920759800 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70236896382560
+  version_requirements: *70160920759800
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70236896381420 !ruby/object:Gem::Requirement
+  requirement: &70160920758660 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70236896381420
+  version_requirements: *70160920758660
 - !ruby/object:Gem::Dependency
   name: awesome_print
-  requirement: &70236896380260 !ruby/object:Gem::Requirement
+  requirement: &70160920773700 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +76,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70236896380260
+  version_requirements: *70160920773700
 - !ruby/object:Gem::Dependency
   name: sinatra
-  requirement: &70236896395760 !ruby/object:Gem::Requirement
+  requirement: &70160920772960 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +87,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70236896395760
+  version_requirements: *70160920772960
 - !ruby/object:Gem::Dependency
   name: thin
-  requirement: &70236896395220 !ruby/object:Gem::Requirement
+  requirement: &70160920772440 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,10 +98,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70236896395220
+  version_requirements: *70160920772440
 - !ruby/object:Gem::Dependency
   name: haml
-  requirement: &70236896394540 !ruby/object:Gem::Requirement
+  requirement: &70160920771760 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -109,10 +109,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70236896394540
+  version_requirements: *70160920771760
 - !ruby/object:Gem::Dependency
   name: namespaced_redis
-  requirement: &70236896393840 !ruby/object:Gem::Requirement
+  requirement: &70160920771040 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -120,7 +120,7 @@ dependencies:
         version: 1.0.2
   type: :runtime
   prerelease: false
-  version_requirements: *70236896393840
+  version_requirements: *70160920771040
 description: Cobweb is a web crawler that can use resque to cluster crawls to quickly
   crawl extremely large sites which is much more perofmant than multi-threaded crawlers.  It
   is also a standalone crawler that has a sophisticated statistics monitoring interface