cobweb 0.0.2 → 0.0.5

data/README.textile

@@ -1,5 +1,5 @@
 
-h1. Cobweb v0.0.2
+h1. Cobweb v0.0.3
 
 h2. Intro 
 

data/lib/cobweb.rb

@@ -1,6 +1,7 @@
-require 'rubygems'
+  require 'rubygems'
 require 'uri'
 require 'resque'
+require "addressable/uri"
 require 'digest/sha1'
 
 Dir[File.dirname(__FILE__) + '/*.rb'].each do |file|
@@ -13,9 +14,12 @@ class CobWeb
     @options = options
     @options[:follow_redirects] = true unless @options.has_key?(:follow_redirects)
     @options[:redirect_limit] = 10 unless @options.has_key?(:redirect_limit)
-    @options[:processing_queue] = ContentProcessJob unless @options.has_key?(:processing_queue)
+    @options[:processing_queue] = CobwebProcessJob unless @options.has_key?(:processing_queue)
+    @options[:quiet] = true unless @options.has_key?(:quiet)
     @options[:debug] = false unless @options.has_key?(:debug)
     @options[:cache] = 300 unless @options.has_key?(:cache)
+    @options[:timeout] = 10 unless @options.has_key?(:timeout)
+    @options[:redis_options] = {} unless @options.has_key?(:redis_options)
     
   end
   
@@ -27,7 +31,7 @@ class CobWeb
     }  
     
     request.merge!(@options)
-  
+    
     Resque.enqueue(CrawlJob, request)
   end
 
@@ -35,11 +39,13 @@ class CobWeb
 
     raise "url cannot be nil" if url.nil?    
     
+    absolutize = Absolutize.new(url, :output_debug => false, :raise_exceptions => false, :force_escaping => false, :remove_anchors => true)
+    
     # get the unique id for this request
     unique_id = Digest::SHA1.hexdigest(url)
     
     # connect to redis
-    redis = NamespacedRedis.new(Redis.new, "cobweb")
+    redis = NamespacedRedis.new(Redis.new(@options[:redis_options]), "cobweb")
     
     content = {}
   
@@ -47,61 +53,205 @@ class CobWeb
     if redis.get(unique_id) and @options[:cache]
       puts "Cache hit for #{url}" unless @options[:quiet]
       content = JSON.parse(redis.get(unique_id)).deep_symbolize_keys
-
+      content[:body] = Base64.decode64(content[:body]) unless content[:body].nil? or content[:mime_type].include?("text/html") or content[:mime_type].include?("application/xhtml+xml")
+      
       content
     else
       # this url is valid for processing so lets get on with it
       print "Retrieving #{url }... " unless @options[:quiet]
-      uri = URI.parse(url)
+      uri = Addressable::URI.parse(url.strip)
       
       # retrieve data
       http = Net::HTTP.new(uri.host, uri.port)
       if uri.scheme == "https"
         http.use_ssl = true
         http.verify_mode = OpenSSL::SSL::VERIFY_NONE
-      end 
+      end
+      
       request_time = Time.now.to_f
-      request = Net::HTTP::Get.new(uri.request_uri)
-      response = http.request(request)
-  
-      if @options[:follow_redirects] and response.code.to_i >= 300 and response.code.to_i < 400
-        puts "redirected... " unless @options[:quiet]
-        url = response['location']
-        redirect_limit = redirect_limit - 1
-        content = get(response['location'], redirect_limit)
+      http.read_timeout = @options[:timeout].to_i
+      http.open_timeout = @options[:timeout].to_i
+      begin
+        response = http.start() {|http|
+          response = http.get(uri.request_uri)
+        }
+        
+        if @options[:follow_redirects] and response.code.to_i >= 300 and response.code.to_i < 400
+          puts "redirected... " unless @options[:quiet]
+          url = absolutize.url(response['location']).to_s
+          ap redirect_limit
+          redirect_limit = redirect_limit - 1
+          content = get(url, redirect_limit)
+          content[:url] = uri.to_s
+          content[:redirect_through] = [] if content[:redirect_through].nil?
+          content[:redirect_through].insert(0, url)
+          
+          content[:response_time] = Time.now.to_f - request_time
+        else
+          content[:response_time] = Time.now.to_f - request_time
+          
+          puts "Retrieved." unless @options[:quiet]
+          
+          # create the content container
+          content[:url] = uri.to_s
+          content[:status_code] = response.code.to_i
+          content[:mime_type] = response.content_type.split(";")[0].strip
+          charset = response["Content-Type"][response["Content-Type"].index(";")+2..-1] if !response["Content-Type"].nil? and response["Content-Type"].include?(";")
+          charset = charset[charset.index("=")+1..-1] if charset and charset.include?("=")
+          content[:character_set] = charset 
+          content[:length] = response.content_length
+          if content[:mime_type].include?("text/html") or content[:mime_type].include?("application/xhtml+xml")
+            content[:body] = response.body
+          else 
+            content[:body] = Base64.encode64(response.body)
+          end
+          content[:location] = response["location"]
+          content[:headers] = response.to_hash.symbolize_keys
+          # parse data for links
+          link_parser = ContentLinkParser.new(content[:url], content[:body])
+          content[:links] = link_parser.link_data
+          
+          # add content to cache if required
+          if @options[:cache]
+            content[:body] = Base64.encode64(content[:body]) unless content[:body].nil? or content[:mime_type].include?("text/html") or content[:mime_type].include?("application/xhtml+xml")
+            redis.set(unique_id, content.to_json)
+            redis.expire unique_id, @options[:cache].to_i
+          end
+        end
+      rescue SocketError => e
+        puts "ERROR: #{e.message}"
+        
+        ## generate a blank content
+        content = {}
         content[:url] = uri.to_s
-        content[:redirect_through] = [] if content[:redirect_through].nil?
-        content[:redirect_through].insert(0, response['location'])
+        content[:response_time] = Time.now.to_f - request_time
+        content[:status_code] = 0
+        content[:length] = 0
+        content[:body] = ""
+        content[:error] = e.message
+        content[:mime_type] = "error/dnslookup"
+        content[:headers] = {}
+        content[:links] = {}
+      
+      rescue Timeout::Error => e
+        puts "ERROR: #{e.message}"
         
+        ## generate a blank content
+        content = {}
+        content[:url] = uri.to_s
         content[:response_time] = Time.now.to_f - request_time
+        content[:status_code] = 0
+        content[:length] = 0
+        content[:body] = ""
+        content[:error] = e.message
+        content[:mime_type] = "error/serverdown"
+        content[:headers] = {}
+        content[:links] = {}
+      end
+    end
+    content  
+  end
+  
+  def head(url, redirect_limit = @options[:redirect_limit])
+    raise "url cannot be nil" if url.nil?    
+    
+    absolutize = Absolutize.new(url, :output_debug => false, :raise_exceptions => false, :force_escaping => false, :remove_anchors => true)
+    
+    # get the unique id for this request
+    unique_id = Digest::SHA1.hexdigest(url)
+    
+    # connect to redis
+    redis = NamespacedRedis.new(Redis.new(@options[:redis_options]), "cobweb")
+    
+    content = {}
+    
+    # check if it has already been cached
+    if (redis.get(unique_id) or redis.get("head-#{unique_id}")) and @options[:cache]
+      puts "Cache hit for #{url}" unless @options[:quiet]
+      if redis.get(unique_id)
+        content = JSON.parse(redis.get(unique_id)).deep_symbolize_keys
       else
-        content[:response_time] = Time.now.to_f - request_time
+        content = JSON.parse(redis.get("head-#{unique_id}")).deep_symbolize_keys
+      end
+      content
+    else
+      print "Retrieving #{url }... " unless @options[:quiet]
+      uri = Addressable::URI.parse(url.strip)
+      
+      # retrieve data
+      http = Net::HTTP.new(uri.host, uri.port)
+      if uri.scheme == "https"
+        http.use_ssl = true
+        http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      end 
+      
+      request_time = Time.now.to_f
+      http.read_timeout = @options[:timeout].to_i
+      http.open_timeout = @options[:timeout].to_i
+      
+      begin      
+        response = http.head(uri.to_s)
         
-        puts "Retrieved." unless @options[:quiet]
+        if @options[:follow_redirects] and response.code.to_i >= 300 and response.code.to_i < 400
+          puts "redirected... " unless @options[:quiet]
+          url = absolutize.url(response['location']).to_s
+          redirect_limit = redirect_limit - 1
+          content = head(url, redirect_limit)
+          content[:url] = uri.to_s
+          content[:redirect_through] = [] if content[:redirect_through].nil?
+          content[:redirect_through].insert(0, url)
+        else
+          content[:url] = uri.to_s
+          content[:status_code] = response.code.to_i
+          unless response.content_type.nil?
+            content[:mime_type] = response.content_type.split(";")[0].strip 
+            charset = response["Content-Type"][response["Content-Type"].index(";")+2..-1] if !response["Content-Type"].nil? and response["Content-Type"].include?(";")
+            charset = charset[charset.index("=")+1..-1] if charset and charset.include?("=")
+            content[:character_set] = charset
+          end 
+          
+          # add content to cache if required
+          if @options[:cache]
+            puts "Stored in cache [head-#{unique_id}]" if @options[:debug]
+            redis.set("head-#{unique_id}", content.to_json)
+            redis.expire "head-#{unique_id}", @options[:cache].to_i
+          else
+            puts "Not storing in cache as cache disabled" if @options[:debug]
+          end
+        end
+      rescue SocketError => e
+        puts "ERROR: #{e.message}"
         
-        # create the content container
+        ## generate a blank content
+        content = {}
         content[:url] = uri.to_s
-        content[:status_code] = response.code.to_i
-        content[:content_type] = response.content_type
-        charset = response["Content-Type"][response["Content-Type"].index(";")+2..-1  ] if !response["Content-Type"].nil? and response["Content-Type"].include?(";")
-        charset = charset[charset.index("=")+1..-1] if charset and charset.include?("=")
-        content[:character_set] = charset 
-        content[:content_length] = response.content_length
-        content[:content_body] = response.body
-        content[:location] = response["location"]
-        content[:headers] = response.to_hash.symbolize_keys
-        # parse data for links
-        link_parser = ContentLinkParser.new(content[:url], content[:content_body])
-        content[:links] = link_parser.link_data
+        content[:response_time] = Time.now.to_f - request_time
+        content[:status_code] = 0
+        content[:length] = 0
+        content[:body] = ""
+        content[:error] = e.message
+        content[:mime_type] = "error/dnslookup"
+        content[:headers] = {}
+        content[:links] = {}
+      
+      rescue Timeout::Error => e
+        puts "ERROR: #{e.message}"
         
-        # add content to cache if required
-        if @options[:cache]
-          redis.set(unique_id, content.to_json)
-          redis.expire unique_id, @options[:cache].to_i
-        end
+        ## generate a blank content
+        content = {}
+        content[:url] = uri.to_s
+        content[:response_time] = Time.now.to_f - request_time
+        content[:status_code] = 0
+        content[:length] = 0
+        content[:body] = ""
+        content[:error] = e.message
+        content[:mime_type] = "error/serverdown"
+        content[:headers] = {}
+        content[:links] = {}
       end
+      
+      content
     end
-    content  
   end
 end
 
@@ -126,4 +276,4 @@ class Hash
     end
     self
   end
-end
+end

data/lib/{content_process_job.rb → cobweb_process_job.rb}

@@ -1,4 +1,4 @@
-class ContentProcessJob
+class CobwebProcessJob
   require "ap"
   
   @queue = :cobweb_process_job
@@ -10,4 +10,4 @@ class ContentProcessJob
     #ap content.keys
     
   end
-end
+end

data/lib/crawl_job.rb

@@ -9,7 +9,7 @@ class CrawlJob
   def self.perform(content_request)
     # change all hash keys to symbols    
     content_request.deep_symbolize_keys
-    redis = NamespacedRedis.new(Redis.new, "cobweb-#{content_request[:crawl_id]}")
+    redis = NamespacedRedis.new(Redis.new(content_request[:redis_options]), "cobweb-#{content_request[:crawl_id]}")
     @absolutize = Absolutize.new(content_request[:url], :output_debug => false, :raise_exceptions => false, :force_escaping => false, :remove_anchors => true)
 
     # check we haven't crawled this url before
@@ -19,18 +19,14 @@ class CrawlJob
       redis.incr "crawl-counter"
       crawl_counter = redis.get("crawl-counter").to_i
       queue_counter = redis.get("queue-counter").to_i
-      if crawl_counter <= content_request[:crawl_limit]
-        content = CobWeb.get(content_request)
-        redis.sadd "crawled", content_request[:url] 
+      if crawl_counter <= content_request[:crawl_limit].to_i
+        content = CobWeb.new(content_request).get(content_request[:url])
+        redis.sadd "crawled", content_request[:url]
         set_base_url redis, content, content_request[:base_url]
-        if queue_counter <= content_request[:crawl_limit]
-          ap content[:links]
+        if queue_counter <= content_request[:crawl_limit].to_i
           content[:links].keys.map{|key| content[:links][key]}.flatten.each do |link|
-            ap link
             unless redis.sismember "crawled", link
-              puts redis.get("base_url")
-              puts "---------------------------------"
-              if link.match(Regexp.new("^#{redis.get("base_url")}"))
+              if link.to_s.match(Regexp.new("^#{redis.get("base_url")}"))
                 new_request = content_request.clone
                 new_request[:url] = link
                 new_request[:parent] = content_request[:url]
@@ -42,12 +38,12 @@ class CrawlJob
         end
 
         # enqueue to processing queue
-        Resque.enqueue(const_get(content_request[:processing_queue]), content)
+        Resque.enqueue(const_get(content_request[:processing_queue]), content.merge({:source_id => content_request[:source_id]}))
         puts "#{content_request[:url]} has been sent for processing." if content_request[:debug]
         puts "Crawled: #{crawl_counter} Limit: #{content_request[:crawl_limit]} Queued: #{queue_counter}" if content_request[:debug]
 
       else
-        puts "Crawl Limit Exceeded by #{crawl_counter - content_request[:crawl_limit]} objects" if content_request[:debug]
+        puts "Crawl Limit Exceeded by #{crawl_counter - content_request[:crawl_limit].to_i} objects" if content_request[:debug]
       end
     else
       puts "Already crawled #{content_request[:url]}" if content_request[:debug]
@@ -60,7 +56,7 @@ class CrawlJob
       if content[:status_code] >= 300 and content[:status_code] < 400
         #redirect received for first url
         redis.set("base_url", @absolutize.url(content[:location]).to_s)
-        puts "Warning: base_url given redirects to another location, setting base_url to #{@absolutize.url(content[:location]).to_s}"
+        puts "WARNING: base_url given redirects to another location, setting base_url to #{@absolutize.url(content[:location]).to_s}"
       else
         redis.set("base_url", base_url)
       end

data/spec/cobweb/cobweb_spec.rb

@@ -36,7 +36,10 @@ describe CobWeb do
       
       @mock_http_client.stub!(:request).with(@mock_http_request).and_return(@mock_http_response)
       @mock_http_client.stub!(:request).with(@mock_http_redirect_request).and_return(@mock_http_redirect_response)      
-      @mock_http_client.stub!(:request).with(@mock_http_redirect_request2).and_return(@mock_http_redirect_response2)      
+      @mock_http_client.stub!(:request).with(@mock_http_redirect_request2).and_return(@mock_http_redirect_response2)
+      @mock_http_client.stub!(:read_timeout=).and_return(nil)      
+      @mock_http_client.stub!(:open_timeout=).and_return(nil)      
+      @mock_http_client.stub!(:start).and_return(@mock_http_response)      
       
       @mock_http_response.stub!(:code).and_return(200)
       @mock_http_response.stub!(:content_type).and_return("text/html")

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: cobweb
 version: !ruby/object:Gem::Version 
-  hash: 27
+  hash: 21
   prerelease: false
   segments: 
   - 0
   - 0
-  - 2
-  version: 0.0.2
+  - 5
+  version: 0.0.5
 platform: ruby
 authors: 
 - Stewart McKee
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-11-10 00:00:00 +00:00
+date: 2011-01-28 00:00:00 +00:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -74,6 +74,20 @@ dependencies:
         version: "0"
   type: :runtime
   version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: addressable
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id005
 description: 
 email: stewart@rockwellcottage.com
 executables: []
@@ -90,8 +104,8 @@ files:
 - spec/cobweb/cobweb_spec.rb
 - lib/namespaced_redis.rb
 - lib/cobweb.rb
-- lib/content_process_job.rb
 - lib/content_link_parser.rb
+- lib/cobweb_process_job.rb
 - lib/crawl_job.rb
 - README.textile
 has_rdoc: false