cobweb 0.0.33 → 0.0.34

data/README.textile

@@ -1,5 +1,5 @@
 
-h1. Cobweb v0.0.33
+h1. Cobweb v0.0.34
 
 h2. Intro 
   

data/lib/cobweb.rb

@@ -4,6 +4,7 @@ require 'resque'
 require "addressable/uri"
 require 'digest/sha1'
 require 'base64'
+require 'namespaced_redis'
 
 Dir[File.dirname(__FILE__) + '/**/*.rb'].each do |file|
   require file
@@ -19,7 +20,7 @@ class Cobweb
   # investigate using event machine for single threaded crawling
   
   def self.version
-    "0.0.33"
+    "0.0.34"
   end
   
   def method_missing(method_sym, *arguments, &block)
@@ -61,7 +62,7 @@ class Cobweb
     end
     
     request.merge!(@options)
-    @redis = NamespacedRedis.new(Redis.new(request[:redis_options]), "cobweb-#{Cobweb.version}-#{request[:crawl_id]}")
+    @redis = NamespacedRedis.new(request[:redis_options], "cobweb-#{Cobweb.version}-#{request[:crawl_id]}")
     @redis.hset "statistics", "queued_at", DateTime.now
     @redis.set("crawl-counter", 0)
     @redis.set("queue-counter", 1)
@@ -74,9 +75,11 @@ class Cobweb
   end
 
   def get(url, options = @options)
-
     raise "url cannot be nil" if url.nil?
-    
+    uri = Addressable::URI.parse(url)
+    uri.fragment=nil
+    url = uri.to_s
+
     # get the unique id for this request
     unique_id = Digest::SHA1.hexdigest(url.to_s)
     if options.has_key?(:redirect_limit) and !options[:redirect_limit].nil?
@@ -87,9 +90,9 @@ class Cobweb
     
     # connect to redis
     if options.has_key? :crawl_id
-      redis = NamespacedRedis.new(Redis.new(@options[:redis_options]), "cobweb-#{Cobweb.version}-#{options[:crawl_id]}")
+      redis = NamespacedRedis.new(@options[:redis_options], "cobweb-#{Cobweb.version}-#{options[:crawl_id]}")
     else
-      redis = NamespacedRedis.new(Redis.new(@options[:redis_options]), "cobweb-#{Cobweb.version}")
+      redis = NamespacedRedis.new(@options[:redis_options], "cobweb-#{Cobweb.version}")
     end
     
     content = {:base_url => url}
@@ -100,8 +103,8 @@ class Cobweb
       content = deep_symbolize_keys(Marshal.load(redis.get(unique_id)))
     else
       # this url is valid for processing so lets get on with it
-      uri = Addressable::URI.parse(url.strip)
-      
+      #TODO the @http here is different from in head.  Should it be? - in head we are using a method-scoped variable.
+
       # retrieve data
       unless @http && @http.address == uri.host && @http.port == uri.inferred_port
         puts "Creating connection to #{uri.host}..." unless @options[:quiet]
@@ -125,7 +128,7 @@ class Cobweb
           puts "redirected... " unless @options[:quiet]
           
           # get location to redirect to
-          url = Addressable::URI.join(uri, response['location']).to_s
+          url = UriHelper.join_no_fragment(uri, response['location'])
           
           # decrement redirect limit
           redirect_limit = redirect_limit - 1
@@ -231,7 +234,10 @@ class Cobweb
   
   def head(url, options = @options)
     raise "url cannot be nil" if url.nil?    
-    
+    uri = Addressable::URI.parse(url.strip)
+    uri.fragment=nil
+    url = uri.to_s
+
     # get the unique id for this request
     unique_id = Digest::SHA1.hexdigest(url)
     if options.has_key?(:redirect_limit) and !options[:redirect_limit].nil?
@@ -242,9 +248,9 @@ class Cobweb
     
     # connect to redis
     if options.has_key? :crawl_id
-      redis = NamespacedRedis.new(Redis.new(@options[:redis_options]), "cobweb-#{Cobweb.version}-#{options[:crawl_id]}")
+      redis = NamespacedRedis.new(@options[:redis_options], "cobweb-#{Cobweb.version}-#{options[:crawl_id]}")
     else
-      redis = NamespacedRedis.new(Redis.new(@options[:redis_options]), "cobweb-#{Cobweb.version}")
+      redis = NamespacedRedis.new(@options[:redis_options], "cobweb-#{Cobweb.version}")
     end
     
     content = {}
@@ -255,8 +261,7 @@ class Cobweb
       content = deep_symbolize_keys(Marshal.load(redis.get("head-#{unique_id}")))
     else
       print "Retrieving #{url }... " unless @options[:quiet]
-      uri = Addressable::URI.parse(url.strip)
-      
+
       # retrieve data
       http = Net::HTTP.new(uri.host, uri.inferred_port)
       if uri.scheme == "https"
@@ -269,11 +274,12 @@ class Cobweb
       http.open_timeout = @options[:timeout].to_i
       
       begin      
-        response = http.head(uri.to_s)
-        
+        request = Net::HTTP::Head.new uri.request_uri
+        response = http.request request
+
         if @options[:follow_redirects] and response.code.to_i >= 300 and response.code.to_i < 400
           puts "redirected... " unless @options[:quiet]
-          url = Addressable::URI.parse(response['location']).to_s
+          url = UriHelper.join_no_fragment(uri, response['location'])
           redirect_limit = redirect_limit - 1
           options = options.clone
           options[:redirect_limit]=redirect_limit

data/lib/cobweb_crawler.rb

@@ -1,6 +1,7 @@
 require 'digest/md5'
 require 'date'
 require 'ap'
+#require 'namespaced_redis'
 
 class CobwebCrawler
   
@@ -11,8 +12,8 @@ class CobwebCrawler
     
     @options[:redis_options] = {:host => "127.0.0.1"} unless @options.has_key? :redis_options
     crawl_id = Digest::MD5.hexdigest(DateTime.now.inspect.to_s)
-  
-    @redis = NamespacedRedis.new(Redis.new(@options[:redis_options]), "cobweb-#{crawl_id}")
+    
+    @redis = NamespacedRedis.new(@options[:redis_options], "cobweb-#{crawl_id}")
     
     @cobweb = Cobweb.new(@options)
   end

data/lib/crawl_job.rb

@@ -3,6 +3,7 @@ class CrawlJob
   require "net/https"  
   require "uri"
   require "redis"
+  require 'namespaced_redis'
 
   @queue = :cobweb_crawl_job
 
@@ -11,7 +12,7 @@ class CrawlJob
     # change all hash keys to symbols
     content_request = content_request.deep_symbolize_keys
     
-    @redis = NamespacedRedis.new(Redis.new(content_request[:redis_options]), "cobweb-#{Cobweb.version}-#{content_request[:crawl_id]}")
+    @redis = NamespacedRedis.new(content_request[:redis_options], "cobweb-#{Cobweb.version}-#{content_request[:crawl_id]}")
     
     @debug = content_request[:debug]
     
@@ -57,7 +58,7 @@ class CrawlJob
         
         #if the enqueue counter has been requested update that
         if content_request.has_key? :enqueue_counter_key                                                                                  
-          enqueue_redis = NamespacedRedis.new(Redis.new(content_request[:redis_options]), content_request[:enqueue_counter_namespace].to_s)
+          enqueue_redis = NamespacedRedis.new(content_request[:redis_options], content_request[:enqueue_counter_namespace].to_s)
           current_count = enqueue_redis.hget(content_request[:enqueue_counter_key], content_request[:enqueue_counter_field]).to_i
           enqueue_redis.hset(content_request[:enqueue_counter_key], content_request[:enqueue_counter_field], current_count+1)
         end
@@ -126,11 +127,12 @@ class CrawlJob
   def self.all_links_from_content(content)
     links = content[:links].keys.map{|key| content[:links][key]}.flatten
     links.reject!{|link| link.starts_with?("javascript:")}
-    links = links.map{|link| Addressable::URI.join(content[:url], link)}
+    links = links.map{|link| UriHelper.join_no_fragment(content[:url], link) }
     links.select!{|link| link.scheme == "http" || link.scheme == "https"}
+    links.uniq
     links
   end
-  
+
   def self.enqueue_content(content_request, link)
     new_request = content_request.clone
     new_request[:url] = link

data/lib/uri_helper.rb

@@ -0,0 +1,8 @@
+class UriHelper
+  def self.join_no_fragment(content, link)
+    new_link = Addressable::URI.join(content, link)
+    new_link.fragment=nil
+    new_link
+  end
+
+end

data/spec/cobweb/cobweb_spec.rb

@@ -117,7 +117,7 @@ describe Cobweb do
         
       #end
       it "should not follow with redirect disabled" do
-        @cobweb = Cobweb.new(:follow_redirects => false, :cache => nil)
+        @cobweb = Cobweb.new(:follow_redirects => false, :cache => 3)
         @mock_http_client.should_receive(:request).with(@mock_http_redirect_request).and_return(@mock_http_redirect_response)
         
         content = @cobweb.get(@base_url)
@@ -133,7 +133,8 @@ describe Cobweb do
     describe "with cache" do
       
       before(:each) do
-        @cobweb = Cobweb.new :quiet => true, :cache => 200
+        @cobweb = Cobweb.new :quiet => true, :cache => 1
+        Redis.new.flushdb
       end
       
       describe "content object" do
@@ -169,8 +170,8 @@ describe Cobweb do
           @cobweb.get(@base_url)[:body].should == "asdf"
         end
         it "should return correct headers" do
-          @cobweb.get(@base_url)[:headers].should == @default_headers
-          @cobweb.get(@base_url)[:headers].should == @default_headers
+          @cobweb.get(@base_url)[:headers].should == @symbolized_default_headers
+          @cobweb.get(@base_url)[:headers].should == @symbolized_default_headers
         end
         it "should return correct a hash of links" do
           @cobweb.get(@base_url)[:links].should be_an_instance_of Hash
@@ -181,7 +182,29 @@ describe Cobweb do
           @cobweb.get(@base_url)[:response_time].should be_an_instance_of Float 
         end
       end
-      
     end
+    describe "location setting" do
+      it "Get should strip fragments" do
+        Net::HTTP.should_receive(:new).with("www.google.com", 80)
+        Net::HTTP::Get.should_receive(:new).with("/")
+        @cobweb.get("http://www.google.com/#ignore")
+      end
+      it "head should strip fragments" do
+        Net::HTTP.should_receive(:new).with("www.google.com", 80)
+        Net::HTTP::Head.should_receive(:new).with("/").and_return(@mock_http_request)
+        @cobweb.head("http://www.google.com/#ignore")
+      end
+      it "get should not strip path" do
+        Net::HTTP.should_receive(:new).with("www.google.com", 80)
+        Net::HTTP::Get.should_receive(:new).with("/path/to/stuff")
+        @cobweb.get("http://www.google.com/path/to/stuff#ignore")
+      end
+      it "get should not strip query string" do
+        Net::HTTP.should_receive(:new).with("www.google.com", 80)
+        Net::HTTP::Get.should_receive(:new).with("/path/to/stuff?query_string")
+        @cobweb.get("http://www.google.com/path/to/stuff?query_string#ignore")
+      end
+    end
+
   end  
 end 

data/spec/spec_helper.rb

@@ -3,11 +3,10 @@ require 'mock_redis'
 
 RSpec.configure do |config|
   config.before(:each) {
-    #redis_mock = double("redis")
-    #ap redis_mock
-    #redis_mock.stub(:new).and_return(MockRedis.new)
+    redis_mock = double("redis")
+    redis_mock.stub(:new).and_return(@redis_mock_object)
     
-    Redis.new.flushdb
+    #redis_mock.flushdb
     
     
     @default_headers = {"Cache-Control" => "private, max-age=0",
@@ -19,6 +18,15 @@ RSpec.configure do |config|
                         "Server" => "gws",
                         "X-XSS-Protection" => "1; mode=block"}
 
+    @symbolized_default_headers = {:"Cache-Control" => "private, max-age=0",
+                        :"Date" => "Wed, 10 Nov 2010 09:06:17 GMT",
+                        :"Expires" => "-1",
+                        :"Content-Type" => "text/html; charset=UTF-8",
+                        :"Content-Encoding" => "",
+                        :"Transfer-Encoding" => "chunked",
+                        :"Server" => "gws",
+                        :"X-XSS-Protection" => "1; mode=block"}
+
     @mock_http_client = mock(Net::HTTP)
     @mock_http_request = mock(Net::HTTPRequest)
     @mock_http_redirect_request = mock(Net::HTTPRequest)
@@ -33,7 +41,9 @@ RSpec.configure do |config|
     Net::HTTP::Get.stub!(:new).and_return(@mock_http_request)
     Net::HTTP::Get.stub!(:new).with("/redirect.html").and_return(@mock_http_redirect_request)
     Net::HTTP::Get.stub!(:new).with("/redirect2.html").and_return(@mock_http_redirect_request2)
-    
+
+    Net::HTTP::Head.stub!(:new).and_return(@mock_http_request)
+
     @mock_http_client.stub!(:request).with(@mock_http_request).and_return(@mock_http_response)
     @mock_http_client.stub!(:request).with(@mock_http_redirect_request).and_return(@mock_http_redirect_response)      
     @mock_http_client.stub!(:request).with(@mock_http_redirect_request2).and_return(@mock_http_redirect_response2)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cobweb
 version: !ruby/object:Gem::Version
-  version: 0.0.33
+  version: 0.0.34
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-04-12 00:00:00.000000000 Z
+date: 2012-04-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: resque
-  requirement: &70153227362400 !ruby/object:Gem::Requirement
+  requirement: &70240600898440 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70153227362400
+  version_requirements: *70240600898440
 - !ruby/object:Gem::Dependency
   name: redis
-  requirement: &70153227361980 !ruby/object:Gem::Requirement
+  requirement: &70240600898020 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70153227361980
+  version_requirements: *70240600898020
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &70153227361560 !ruby/object:Gem::Requirement
+  requirement: &70240600897600 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70153227361560
+  version_requirements: *70240600897600
 - !ruby/object:Gem::Dependency
   name: addressable
-  requirement: &70153227361140 !ruby/object:Gem::Requirement
+  requirement: &70240600897180 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70153227361140
+  version_requirements: *70240600897180
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70153227360720 !ruby/object:Gem::Requirement
+  requirement: &70240600913120 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70153227360720
+  version_requirements: *70240600913120
 - !ruby/object:Gem::Dependency
   name: awesome_print
-  requirement: &70153227360300 !ruby/object:Gem::Requirement
+  requirement: &70240600912700 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +76,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70153227360300
+  version_requirements: *70240600912700
 - !ruby/object:Gem::Dependency
   name: sinatra
-  requirement: &70153227359880 !ruby/object:Gem::Requirement
+  requirement: &70240600912280 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +87,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70153227359880
+  version_requirements: *70240600912280
 - !ruby/object:Gem::Dependency
   name: thin
-  requirement: &70153227359460 !ruby/object:Gem::Requirement
+  requirement: &70240600911860 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,10 +98,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70153227359460
+  version_requirements: *70240600911860
 - !ruby/object:Gem::Dependency
   name: haml
-  requirement: &70153227359040 !ruby/object:Gem::Requirement
+  requirement: &70240600911440 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -109,7 +109,18 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70153227359040
+  version_requirements: *70240600911440
+- !ruby/object:Gem::Dependency
+  name: namespaced_redis
+  requirement: &70240600911020 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70240600911020
 description: Web Crawler that uses resque background job engine to allow you to cluster
   your crawl.
 email: stewart@rockwellcottage.com
@@ -130,10 +141,10 @@ files:
 - lib/cobweb_process_job.rb
 - lib/content_link_parser.rb
 - lib/crawl_job.rb
-- lib/namespaced_redis.rb
 - lib/redirect_error.rb
 - lib/robots.rb
 - lib/stats.rb
+- lib/uri_helper.rb
 - views/statistics.haml
 - README.textile
 homepage: http://github.com/stewartmckee/cobweb

data/lib/namespaced_redis.rb

@@ -1,88 +0,0 @@
-class NamespacedRedis
-  def initialize(redis, namespace="")
-    raise "redis must be supplied" if redis.nil?
-    @redis = redis
-    @namespace = namespace
-  end
-  
-  def sismember(key, member)
-    @redis.sismember namespaced(key), member
-  end
-  
-  def sadd(key, value)
-    @redis.sadd namespaced(key), value
-  end
-  
-  def srem(key, member)
-    @redis.srem namespaced(key), member
-  end
-
-  def spop(key)
-    @redis.spop namespaced(key)
-  end
-
-  def smembers(key)
-    @redis.smembers namespaced(key)
-  end
-  
-  def scard(key)
-    @redis.scard namespaced(key)
-  end  
-  
-  def get(key)
-    @redis.get namespaced(key)
-  end
-  
-  def incr(key)
-    @redis.incr namespaced(key)
-  end
-  
-  def decr(key)
-    @redis.decr namespaced(key)
-  end
-  
-  def exist(key)
-    @redis.exist namespaced(key)
-  end
-  
-  def set(key, value)
-    @redis.set namespaced(key), value
-  end
-  
-  def hget(key, member)
-    @redis.hget namespaced(key), member
-  end
-  
-  def hgetall(key)
-    @redis.hgetall namespaced(key)
-  end
-  
-  def hset(key, member, value)
-    @redis.hset namespaced(key), member, value
-  end
-  
-  def hexists(key, member)
-    @redis.hexists namespaced(key), member
-  end
-  
-  def del(key)
-    @redis.del namespaced(key)
-  end
-  
-  def expire(key, value)
-    @redis.expire namespaced(key), value
-  end
-  
-  def namespaced(key)
-    "#{@namespace}-#{key}"
-  end
-  
-  def native
-    @redis
-  end
-  
-  def namespace
-    @namespace
-  end
-  
-end