cobweb 0.0.19 → 0.0.20

data/lib/cobweb.rb

@@ -29,8 +29,7 @@ class Cobweb
     @options[:debug] = false unless @options.has_key?(:debug)
     @options[:cache] = 300 unless @options.has_key?(:cache)
     @options[:timeout] = 10 unless @options.has_key?(:timeout)
-    @options[:redis_options] = {} unless @options.has_key?(:redis_options)
-    
+    @options[:redis_options] = {} unless @options.has_key?(:redis_options)    
   end
   
   def start(base_url)
@@ -41,33 +40,40 @@ class Cobweb
     }  
     
     request.merge!(@options)
-    redis = NamespacedRedis.new(Redis.new(request[:redis_options]), "cobweb-#{request[:crawl_id]}")
-    redis.hset "statistics", "queued_at", DateTime.now
+    @redis = NamespacedRedis.new(Redis.new(request[:redis_options]), "cobweb-#{VERSION}-#{request[:crawl_id]}")
+    @redis.hset "statistics", "queued_at", DateTime.now
     
     Resque.enqueue(CrawlJob, request)
   end
 
-  def get(url, redirect_limit = @options[:redirect_limit])
+  def get(url, options = @options)
 
-    raise "url cannot be nil" if url.nil?    
+    raise "url cannot be nil" if url.nil?
     
     absolutize = Absolutize.new(url, :output_debug => false, :raise_exceptions => true, :force_escaping => false, :remove_anchors => true)
         
     # get the unique id for this request
     unique_id = Digest::SHA1.hexdigest(url.to_s)
+    redirect_limit = options[:redirect_limit]
     
     # connect to redis
-    redis = NamespacedRedis.new(Redis.new(@options[:redis_options]), "cobweb")
+    ap options
+    if options.has_key? :crawl_id
+      redis = NamespacedRedis.new(Redis.new(@options[:redis_options]), "cobweb-#{VERSION}-#{options[:crawl_id]}")
+    else
+      redis = NamespacedRedis.new(Redis.new(@options[:redis_options]), "cobweb-#{VERSION}")
+    end
     
+    ap "===== HEAD NAMESPACE ====="
+    ap redis.namespace
+    ap "===== HEAD NAMESPACE ====="
+
     content = {}
   
     # check if it has already been cached
     if redis.get(unique_id) and @options[:cache]
       puts "Cache hit for #{url}" unless @options[:quiet]
-      content = JSON.parse(redis.get(unique_id)).deep_symbolize_keys
-      content[:body] = Base64.decode64(content[:body])
-      
-      content
+      content = Marshal.load(redis.get(unique_id)).deep_symbolize_keys
     else
       # this url is valid for processing so lets get on with it
       uri = Addressable::URI.parse(url.strip)
@@ -143,8 +149,7 @@ class Cobweb
         end
         # add content to cache if required
         if @options[:cache]
-          content[:body] = Base64.encode64(content[:body])
-          redis.set(unique_id, content.to_json)
+          redis.set(unique_id, Marshal.dump(content))
           redis.expire unique_id, @options[:cache].to_i
         end
       rescue RedirectError => e
@@ -196,28 +201,32 @@ class Cobweb
     content  
   end
   
-  def head(url, redirect_limit = @options[:redirect_limit])
+  def head(url, options = @options)
     raise "url cannot be nil" if url.nil?    
     
     absolutize = Absolutize.new(url, :output_debug => false, :raise_exceptions => false, :force_escaping => false, :remove_anchors => true)
     
     # get the unique id for this request
     unique_id = Digest::SHA1.hexdigest(url)
+    redirect_limit = options[:redirect_limit]
     
     # connect to redis
-    redis = NamespacedRedis.new(Redis.new(@options[:redis_options]), "cobweb")
+    if options.has_key? :crawl_id
+      redis = NamespacedRedis.new(Redis.new(@options[:redis_options]), "cobweb-#{VERSION}-#{options[:crawl_id]}")
+    else
+      redis = NamespacedRedis.new(Redis.new(@options[:redis_options]), "cobweb-#{VERSION}")
+    end
     
+    ap "===== HEAD NAMESPACE ====="
+    ap redis.namespace
+    ap "===== HEAD NAMESPACE ====="
+
     content = {}
     
     # check if it has already been cached
-    if (redis.get(unique_id) or redis.get("head-#{unique_id}")) and @options[:cache]
+    if redis.get("head-#{unique_id}") and @options[:cache]
       puts "Cache hit for #{url}" unless @options[:quiet]
-      if redis.get(unique_id)
-        content = JSON.parse(redis.get(unique_id)).deep_symbolize_keys
-      else
-        content = JSON.parse(redis.get("head-#{unique_id}")).deep_symbolize_keys
-      end
-      content
+      Marshal.load(redis.get("head-#{unique_id}")).deep_symbolize_keys
     else
       print "Retrieving #{url }... " unless @options[:quiet]
       uri = Addressable::URI.parse(url.strip)
@@ -259,7 +268,7 @@ class Cobweb
           # add content to cache if required
           if @options[:cache]
             puts "Stored in cache [head-#{unique_id}]" if @options[:debug]
-            redis.set("head-#{unique_id}", content.to_json)
+            redis.set("head-#{unique_id}", Marshal.dump(content))
             redis.expire "head-#{unique_id}", @options[:cache].to_i
           else
             puts "Not storing in cache as cache disabled" if @options[:debug]

data/lib/cobweb/version.rb

@@ -0,0 +1 @@
+VERSION = "0.0.20"

data/lib/crawl_job.rb

@@ -28,7 +28,8 @@ class CrawlJob
   def self.perform(content_request)
     # change all hash keys to symbols    
     content_request.deep_symbolize_keys
-    redis = NamespacedRedis.new(Redis.new(content_request[:redis_options]), "cobweb-#{content_request[:crawl_id]}")
+    redis = NamespacedRedis.new(Redis.new(content_request[:redis_options]), "cobweb-#{VERSION}-#{content_request[:crawl_id]}")
+    ap redis.namespace
     @absolutize = Absolutize.new(content_request[:url], :output_debug => false, :raise_exceptions => false, :force_escaping => false, :remove_anchors => true)
 
     # check we haven't crawled this url before
@@ -40,7 +41,7 @@ class CrawlJob
       redis.incr "crawl-counter"
       crawl_counter += 1
       if crawl_counter <= content_request[:crawl_limit].to_i
-        content = Cobweb.new(content_request).get(content_request[:url])
+        content = Cobweb.new(content_request).get(content_request[:url], content_request)
 
         ## update statistics
         if redis.hexists "statistics", "average_response_time"

data/lib/robots.rb

@@ -0,0 +1,9 @@
+class Robots
+  
+  def initialize(url, file_name="robots.txt")
+    uri = URI.parse(url)
+    [uri.scheme, "://", uri.host, ":", uri.port, "/", file_name].join
+    Cobweb.new(:cache => 6000).get([uri.scheme, "://", uri.host, ":", uri.port, "/", file_name].join)
+    
+  end
+end

data/lib/stats.rb

@@ -20,12 +20,15 @@ class Stats < Sinatra::Base
     haml :statistics
   end
   
-end
-
-thread = Thread.new do
-  Stats.run!
   
-  ## we need to manually kill the main thread as sinatra traps the interrupts
-  Thread.main.kill
+  def self.start
+    thread = Thread.new do
+      Stats.run!
+
+      ## we need to manually kill the main thread as sinatra traps the interrupts
+      Thread.main.kill
+    end    
+  end
 end
 
+

data/spec/cobweb/cobweb_spec.rb

@@ -169,19 +169,68 @@ describe Cobweb do
 
         end
       end
+      
+      describe "with cache" do
+        
+        before(:each) do
+          @cobweb = Cobweb.new :quiet => true, :cache => 200
+        end
+        
+        describe "content object" do
+          it "should return the url" do
+            @cobweb.get(@base_url)[:url].should == @base_url
+            @cobweb.get(@base_url)[:url].should == @base_url
+          end
+          it "should return correct content-type" do
+            @mock_http_response.stub!(:content_type).and_return("image/jpeg")
+            @cobweb.get(@base_url)[:mime_type].should == "image/jpeg"
+            @cobweb.get(@base_url)[:mime_type].should == "image/jpeg"
+          end
+          it "should return correct status-code" do
+            @mock_http_response.stub!(:code).and_return(404)
+            @cobweb.get(@base_url)[:status_code].should == 404
+            @cobweb.get(@base_url)[:status_code].should == 404
+          end
+          it "should return correct status-code" do
+            @mock_http_response.stub!(:code).and_return(404)
+            @cobweb.get(@base_url)[:status_code].should == 404
+            @cobweb.get(@base_url)[:status_code].should == 404
+          end
+          it "should return correct character_set" do
+            @cobweb.get(@base_url)[:character_set].should == "UTF-8"
+            @cobweb.get(@base_url)[:character_set].should == "UTF-8"
+          end 
+          it "should return correct content_length" do
+            @cobweb.get(@base_url)[:length].should == 1024
+            @cobweb.get(@base_url)[:length].should == 1024
+          end
+          it "should return correct content_body" do
+            @cobweb.get(@base_url)[:body].should == "asdf"
+            @cobweb.get(@base_url)[:body].should == "asdf"
+          end
+          it "should return correct location" do
+            @cobweb.get(@base_url)[:location].should == nil
+            @cobweb.get(@base_url)[:location].should == nil
+
+            @mock_http_response.stub!(:[]).with("location").and_return("http://google.com/")
+            @cobweb.get(@base_url)[:location].should == "http://google.com/"
+            @cobweb.get(@base_url)[:location].should == "http://google.com/"
+          end
+          it "should return correct headers" do
+            @cobweb.get(@base_url)[:headers].should == @default_headers
+            @cobweb.get(@base_url)[:headers].should == @default_headers
+          end
+          it "should return correct a hash of links" do
+            @cobweb.get(@base_url)[:links].should be_an_instance_of Hash
+            @cobweb.get(@base_url)[:links].should be_an_instance_of Hash
+          end 
+          it "should return the response time for the url" do
+            @cobweb.get(@base_url)[:response_time].should be_an_instance_of Float 
+            @cobweb.get(@base_url)[:response_time].should be_an_instance_of Float 
+          end
+        end
+        
+      end
     end  
   end  
-
-  describe "without mock" do
-    
-    it "should throw exception when server is unavailable" #do
-    #  lambda {@cobweb.get({:url => "http://www.oasdjgoisadjgoisdiog.com"})}.should raise_error URI::InvalidURIError
-    #end
-    
-    it "should return a valid content hash when url doesn't exist on a live server" do
-      status_code = @cobweb.get("http://test.com/laskdjflsdajf")[:status_code]
-      status_code.should == 404
-    end
-    
-  end
 end 

data/spec/spec_helper.rb

@@ -1 +1,13 @@
 require File.expand_path(File.dirname(__FILE__) + '/../lib/cobweb')
+require 'mock_redis'
+
+RSpec.configure do |config|
+  config.before(:each) {
+    #redis_mock = double("redis")
+    #ap redis_mock
+    #redis_mock.stub(:new).and_return(MockRedis.new)
+    
+    Redis.new.flushdb
+  }
+
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cobweb
 version: !ruby/object:Gem::Version
-  version: 0.0.19
+  version: 0.0.20
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-03-04 00:00:00.000000000 Z
+date: 2012-03-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: resque
-  requirement: &70137319788100 !ruby/object:Gem::Requirement
+  requirement: &70234753393720 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70137319788100
+  version_requirements: *70234753393720
 - !ruby/object:Gem::Dependency
   name: redis
-  requirement: &70137319787220 !ruby/object:Gem::Requirement
+  requirement: &70234753392840 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70137319787220
+  version_requirements: *70234753392840
 - !ruby/object:Gem::Dependency
   name: absolutize
-  requirement: &70137319786300 !ruby/object:Gem::Requirement
+  requirement: &70234753392180 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70137319786300
+  version_requirements: *70234753392180
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &70137319785420 !ruby/object:Gem::Requirement
+  requirement: &70234753391520 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70137319785420
+  version_requirements: *70234753391520
 - !ruby/object:Gem::Dependency
   name: addressable
-  requirement: &70137319784540 !ruby/object:Gem::Requirement
+  requirement: &70234753390840 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70137319784540
+  version_requirements: *70234753390840
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70137319783560 !ruby/object:Gem::Requirement
+  requirement: &70234753390200 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +76,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70137319783560
+  version_requirements: *70234753390200
 - !ruby/object:Gem::Dependency
   name: awesome_print
-  requirement: &70137319782700 !ruby/object:Gem::Requirement
+  requirement: &70234753389280 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +87,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70137319782700
+  version_requirements: *70234753389280
 - !ruby/object:Gem::Dependency
   name: sinatra
-  requirement: &70137319781560 !ruby/object:Gem::Requirement
+  requirement: &70234753388360 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,10 +98,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70137319781560
+  version_requirements: *70234753388360
 - !ruby/object:Gem::Dependency
   name: thin
-  requirement: &70137319780360 !ruby/object:Gem::Requirement
+  requirement: &70234753387580 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -109,10 +109,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70137319780360
+  version_requirements: *70234753387580
 - !ruby/object:Gem::Dependency
   name: haml
-  requirement: &70137319779820 !ruby/object:Gem::Requirement
+  requirement: &70234753386980 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -120,7 +120,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70137319779820
+  version_requirements: *70234753386980
 description: 
 email: stewart@rockwellcottage.com
 executables: []
@@ -134,6 +134,7 @@ files:
 - spec/samples/sample_html_links.html
 - spec/spec.opts
 - spec/spec_helper.rb
+- lib/cobweb/version.rb
 - lib/cobweb.rb
 - lib/cobweb_crawler.rb
 - lib/cobweb_finished_job.rb
@@ -143,6 +144,7 @@ files:
 - lib/hash.rb
 - lib/namespaced_redis.rb
 - lib/redirect_error.rb
+- lib/robots.rb
 - lib/stats.rb
 - views/statistics.haml
 - README.textile