cobweb 0.0.25 → 0.0.26

data/README.textile

@@ -1,5 +1,5 @@
 
-h1. Cobweb v0.0.25
+h1. Cobweb v0.0.26
 
 h2. Intro 
   

data/lib/cobweb.rb

@@ -19,7 +19,7 @@ class Cobweb
   # investigate using event machine for single threaded crawling
   
   def self.version
-    "0.0.25"
+    "0.0.26"
   end
   
   def method_missing(method_sym, *arguments, &block)
@@ -77,8 +77,6 @@ class Cobweb
 
     raise "url cannot be nil" if url.nil?
     
-    absolutize = Absolutize.new(url, :output_debug => false, :raise_exceptions => true, :force_escaping => false, :remove_anchors => true)
-        
     # get the unique id for this request
     unique_id = Digest::SHA1.hexdigest(url.to_s)
     if options.has_key?(:redirect_limit) and !options[:redirect_limit].nil?
@@ -127,7 +125,7 @@ class Cobweb
           puts "redirected... " unless @options[:quiet]
           
           # get location to redirect to
-          url = absolutize.url(response['location']).to_s
+          url = Addressable::URI.parse(response['location']).to_s
           
           # decrement redirect limit
           redirect_limit = redirect_limit - 1
@@ -234,8 +232,6 @@ class Cobweb
   def head(url, options = @options)
     raise "url cannot be nil" if url.nil?    
     
-    absolutize = Absolutize.new(url, :output_debug => false, :raise_exceptions => false, :force_escaping => false, :remove_anchors => true)
-
     # get the unique id for this request
     unique_id = Digest::SHA1.hexdigest(url)
     if options.has_key?(:redirect_limit) and !options[:redirect_limit].nil?
@@ -277,7 +273,7 @@ class Cobweb
         
         if @options[:follow_redirects] and response.code.to_i >= 300 and response.code.to_i < 400
           puts "redirected... " unless @options[:quiet]
-          url = absolutize.url(response['location']).to_s
+          url = Addressable::URI.parse(response['location']).to_s
           redirect_limit = redirect_limit - 1
           options = options.clone
           options[:redirect_limit]=redirect_limit

data/lib/cobweb_crawler.rb

@@ -22,8 +22,6 @@ class CobwebCrawler
     
     @crawl_options = crawl_options
     
-    @absolutize = Absolutize.new(@options[:base_url], :output_debug => false, :raise_exceptions => false, :force_escaping => false, :remove_anchors => true)
-    
     @redis.sadd "queued", base_url
     crawl_counter = @redis.scard("crawled").to_i
     queue_counter = @redis.scard("queued").to_i

data/lib/content_link_parser.rb

@@ -12,7 +12,6 @@ class ContentLinkParser
     if @doc.at("base[href]")
       base_url = @doc.at("base[href]").attr("href").to_s
     end
-    @absolutize = Absolutize.new(base_url, :output_debug => false, :raise_exceptions => false, :force_escaping => false, :remove_anchors => true)
 
     @options[:tags] = {}
     @options[:tags][:links] = [["a[href]", "href"], ["frame[src]", "src"], ["meta[@http-equiv=\"refresh\"]", "content"], ["link[href]:not([rel])", "href"], ["area[href]", "href"]]
@@ -57,15 +56,14 @@ class ContentLinkParser
     if attribute.kind_of? String or attribute.kind_of? Symbol
       @doc.css(selector).each do |tag|
         begin
-          uri = @absolutize.url(tag[attribute])
-          array << uri.to_s
+          array << Addressable::URI.parse(tag[attribute]).to_s
         rescue
         end
       end
     elsif attribute.instance_of? Regexp
       @doc.css(selector).each do |tag|
         begin
-          tag.content.scan(attribute) {|match| array << @absolutize.url(match[0])}
+          tag.content.scan(attribute) {|match| array << Addressable::URI.parse(match[0]).to_s}
         rescue
         end
       end

data/lib/crawl_job.rb

@@ -13,7 +13,6 @@ class CrawlJob
     
     @redis = NamespacedRedis.new(Redis.new(content_request[:redis_options]), "cobweb-#{Cobweb.version}-#{content_request[:crawl_id]}")
     
-    @absolutize = Absolutize.new(content_request[:url], :output_debug => false, :raise_exceptions => false, :force_escaping => false, :remove_anchors => true)
     @debug = content_request[:debug]
     
     refresh_counters

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cobweb
 version: !ruby/object:Gem::Version
-  version: 0.0.25
+  version: 0.0.26
   prerelease: 
 platform: ruby
 authors:
@@ -13,7 +13,7 @@ date: 2012-03-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: resque
-  requirement: &70349719636940 !ruby/object:Gem::Requirement
+  requirement: &70294818287920 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70349719636940
+  version_requirements: *70294818287920
 - !ruby/object:Gem::Dependency
   name: redis
-  requirement: &70349719636520 !ruby/object:Gem::Requirement
+  requirement: &70294818287500 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70349719636520
+  version_requirements: *70294818287500
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &70349719636100 !ruby/object:Gem::Requirement
+  requirement: &70294818287080 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,21 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70349719636100
-- !ruby/object:Gem::Dependency
-  name: absolutize
-  requirement: &70349719635660 !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: *70349719635660
+  version_requirements: *70294818287080
 - !ruby/object:Gem::Dependency
   name: addressable
-  requirement: &70349719635240 !ruby/object:Gem::Requirement
+  requirement: &70294818286640 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70349719635240
+  version_requirements: *70294818286640
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70349719634820 !ruby/object:Gem::Requirement
+  requirement: &70294818286220 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +65,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70349719634820
+  version_requirements: *70294818286220
 - !ruby/object:Gem::Dependency
   name: awesome_print
-  requirement: &70349719634400 !ruby/object:Gem::Requirement
+  requirement: &70294818285800 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +76,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70349719634400
+  version_requirements: *70294818285800
 - !ruby/object:Gem::Dependency
   name: sinatra
-  requirement: &70349719633980 !ruby/object:Gem::Requirement
+  requirement: &70294818285380 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,10 +87,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70349719633980
+  version_requirements: *70294818285380
 - !ruby/object:Gem::Dependency
   name: thin
-  requirement: &70349719633560 !ruby/object:Gem::Requirement
+  requirement: &70294818284960 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -109,10 +98,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70349719633560
+  version_requirements: *70294818284960
 - !ruby/object:Gem::Dependency
   name: haml
-  requirement: &70349719633140 !ruby/object:Gem::Requirement
+  requirement: &70294818284540 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -120,7 +109,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70349719633140
+  version_requirements: *70294818284540
 description: Web Crawler that uses resque background job engine to allow you to cluster
   your crawl.
 email: stewart@rockwellcottage.com