cobweb 0.0.36 → 0.0.37

data/README.textile

@@ -1,5 +1,5 @@
 
-h1. Cobweb v0.0.36
+h1. Cobweb v0.0.37
 
 h2. Intro 
   

data/lib/cobweb.rb

@@ -20,7 +20,7 @@ class Cobweb
   # investigate using event machine for single threaded crawling
   
   def self.version
-    "0.0.36"
+    "0.0.37"
   end
   
   def method_missing(method_sym, *arguments, &block)
@@ -34,7 +34,7 @@ class Cobweb
   
   def initialize(options = {})
     @options = options
-    
+    default_use_encoding_safe_process_job_to  false
     default_follow_redirects_to               true
     default_redirect_limit_to                 10
     default_processing_queue_to               CobwebProcessJob

data/lib/crawl_job.rb

@@ -52,9 +52,7 @@ class CrawlJob
         end
         
         # enqueue to processing queue
-        Resque.enqueue(const_get(content_request[:processing_queue]), content.merge({:internal_urls => internal_patterns, :redis_options => content_request[:redis_options], :source_id => content_request[:source_id], :crawl_id => content_request[:crawl_id]}))
-        puts "#{content_request[:url]} has been sent for processing." if content_request[:debug]
-        puts "Crawled: #{@crawl_counter} Limit: #{content_request[:crawl_limit]} Queued: #{@queue_counter}" if content_request[:debug]
+        send_to_processing_queue(content, content_request)
         
         #if the enqueue counter has been requested update that
         if content_request.has_key? :enqueue_counter_key                                                                                  
@@ -63,7 +61,7 @@ class CrawlJob
           enqueue_redis.hset(content_request[:enqueue_counter_key], content_request[:enqueue_counter_field], current_count+1)
         end
 
-        # if the'res nothing left queued or the crawled limit has been reached
+        # if there's nothing left queued or the crawled limit has been reached
         if @queue_counter == 0 || @crawl_counter >= content_request[:crawl_limit].to_i
 
           # finished
@@ -86,6 +84,19 @@ class CrawlJob
 
   end
 
+  def self.send_to_processing_queue(content, content_request)
+    content_to_send = content.merge({:internal_urls => internal_patterns, :redis_options => content_request[:redis_options], :source_id => content_request[:source_id], :crawl_id => content_request[:crawl_id]})
+    if content_request[:use_encoding_safe_process_job]
+      content_to_send[:body] = Base64.encode64(content[:body])
+      content_to_send[:processing_queue] = content_request[:processing_queue]
+      Resque.enqueue(EncodingSafeProcessJob, content_to_send)
+    else
+      Resque.enqueue(const_get(content_request[:processing_queue]), content_to_send)
+    end
+    puts "#{content_request[:url]} has been sent for processing. use_encoding_safe_process_job: #{content_request[:use_encoding_safe_process_job]}" if content_request[:debug]
+    puts "Crawled: #{@crawl_counter} Limit: #{content_request[:crawl_limit]} Queued: #{@queue_counter}" if content_request[:debug]
+  end
+
   private
   
   def self.within_crawl_limits?(crawl_limit)

data/lib/encoding_safe_process_job.rb

@@ -0,0 +1,13 @@
+class EncodingSafeProcessJob
+
+  @queue = :encoding_safe_process_job
+
+  def self.perform(content)
+    clazz = const_get(content["processing_queue"])
+    content["body"] = Base64.decode64(content["body"])
+    clazz.perform(content)
+  end
+end
+
+
+

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cobweb
 version: !ruby/object:Gem::Version
-  version: 0.0.36
+  version: 0.0.37
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-04-19 00:00:00.000000000 Z
+date: 2012-04-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: resque
-  requirement: &70241538594440 !ruby/object:Gem::Requirement
+  requirement: &70275220162200 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70241538594440
+  version_requirements: *70275220162200
 - !ruby/object:Gem::Dependency
   name: redis
-  requirement: &70241538594020 !ruby/object:Gem::Requirement
+  requirement: &70275220161740 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70241538594020
+  version_requirements: *70275220161740
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &70241538593600 !ruby/object:Gem::Requirement
+  requirement: &70275220177640 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70241538593600
+  version_requirements: *70275220177640
 - !ruby/object:Gem::Dependency
   name: addressable
-  requirement: &70241538593180 !ruby/object:Gem::Requirement
+  requirement: &70275220177220 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70241538593180
+  version_requirements: *70275220177220
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70241538609120 !ruby/object:Gem::Requirement
+  requirement: &70275220176640 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70241538609120
+  version_requirements: *70275220176640
 - !ruby/object:Gem::Dependency
   name: awesome_print
-  requirement: &70241538608700 !ruby/object:Gem::Requirement
+  requirement: &70275220176220 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +76,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70241538608700
+  version_requirements: *70275220176220
 - !ruby/object:Gem::Dependency
   name: sinatra
-  requirement: &70241538608280 !ruby/object:Gem::Requirement
+  requirement: &70275220175780 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +87,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70241538608280
+  version_requirements: *70275220175780
 - !ruby/object:Gem::Dependency
   name: thin
-  requirement: &70241538607860 !ruby/object:Gem::Requirement
+  requirement: &70275220175360 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,10 +98,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70241538607860
+  version_requirements: *70275220175360
 - !ruby/object:Gem::Dependency
   name: haml
-  requirement: &70241538607440 !ruby/object:Gem::Requirement
+  requirement: &70275220174940 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -109,10 +109,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70241538607440
+  version_requirements: *70275220174940
 - !ruby/object:Gem::Dependency
   name: namespaced_redis
-  requirement: &70241538607020 !ruby/object:Gem::Requirement
+  requirement: &70275220174520 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -120,7 +120,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70241538607020
+  version_requirements: *70275220174520
 description: Web Crawler that uses resque background job engine to allow you to cluster
   your crawl.
 email: stewart@rockwellcottage.com
@@ -141,6 +141,7 @@ files:
 - lib/cobweb_process_job.rb
 - lib/content_link_parser.rb
 - lib/crawl_job.rb
+- lib/encoding_safe_process_job.rb
 - lib/redirect_error.rb
 - lib/robots.rb
 - lib/stats.rb