cobweb 0.0.49 → 0.0.50

data/README.textile

@@ -1,5 +1,5 @@
 
-h1. Cobweb v0.0.49
+h1. Cobweb v0.0.50
 
 h2. Intro 
   
@@ -38,6 +38,7 @@ h3. Data Returned
     ** :styles - url's from within link tags with rel of stylesheet and from url() directives with stylesheets
   * :crawl_id - the id used internally for identifying the crawl.  Can be used by the processing job to seperate crawls
   * :internal_urls - an array of urls with * wildcards that represent urls internal to the site (ie pages within the same domain)
+  * :external_urls - an array of urls with * wildcards that represent urls external to the site (overrides internal_urls)
   
   The source for the links can be overridden, contact me for the syntax (don't have time to put it into this documentation, will as soon as i have time!)
 

data/lib/cobweb_links.rb

@@ -10,8 +10,8 @@ class CobwebLinks
     @options[:external_urls] = [] unless @options.has_key? :external_urls
     @options[:debug] = false unless @options.has_key? :debug
     
-    @internal_patterns = @options[:internal_urls].map{|pattern| Regexp.new("^#{pattern.gsub(".", "\\.").gsub("?", "\\?").gsub("*", ".*?")}")}
-    @external_patterns = @options[:external_urls].map{|pattern| Regexp.new("^#{pattern.gsub(".", "\\.").gsub("?", "\\?").gsub("*", ".*?")}")}
+    @internal_patterns = @options[:internal_urls].map{|pattern| Regexp.new("^#{pattern.cobweb_encode_for_regex([".", "?"]).gsub("*", ".*?")}")}
+    @external_patterns = @options[:external_urls].map{|pattern| Regexp.new("^#{pattern.cobweb_encode_for_regex([".", "?"]).gsub("*", ".*?")}")}
     
   end
   
@@ -46,3 +46,11 @@ end
 class InvalidUrlsError < Exception
 end
 
+class String
+  def cobweb_encode_for_regex(characters)
+    characters.map{|character| self.gsub!(character, "\\#{character}") }
+    ap self
+    self
+  end
+end
+

data/lib/cobweb_version.rb

@@ -1,6 +1,6 @@
 class CobwebVersion
   def self.version
-    "0.0.49"
+    "0.0.50"
   end
 
 end

data/spec/cobweb/cobweb_links_spec.rb

@@ -81,7 +81,7 @@ describe CobwebLinks do
       cobweb_links.external?("http://blog.domain_one.com/pageone.html").should be_true
     end
     it "should match external links with querystring parameters" do
-      cobweb_links = CobwebLinks.new(:internal_urls => ["http://www.ford.com/"], :external_urls => ["http://*.ford.com/*?*view=print"], :debug => true)
+      cobweb_links = CobwebLinks.new(:internal_urls => ["http://www.ford.com/"], :external_urls => ["http://*.ford.com/*?*view=print"])
       cobweb_links.external?("http://corporate.ford.com/news-center/press-releases-detail/pr-doug-scott2658-marketing-manager-31039?view=print").should be_true
       cobweb_links.internal?("http://corporate.ford.com/news-center/press-releases-detail/pr-doug-scott2658-marketing-manager-31039?view=print").should be_false
       cobweb_links.external?("http://corporate.ford.com/news-center/view=print/pr-doug-scott2658-marketing-manager-31039").should be_true

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cobweb
 version: !ruby/object:Gem::Version
-  version: 0.0.49
+  version: 0.0.50
   prerelease: 
 platform: ruby
 authors:
@@ -13,7 +13,7 @@ date: 2012-05-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: resque
-  requirement: &70099734277020 !ruby/object:Gem::Requirement
+  requirement: &70235629945140 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70099734277020
+  version_requirements: *70235629945140
 - !ruby/object:Gem::Dependency
   name: redis
-  requirement: &70099734276180 !ruby/object:Gem::Requirement
+  requirement: &70235629943920 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70099734276180
+  version_requirements: *70235629943920
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &70099734289880 !ruby/object:Gem::Requirement
+  requirement: &70235629942920 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70099734289880
+  version_requirements: *70235629942920
 - !ruby/object:Gem::Dependency
   name: addressable
-  requirement: &70099734287280 !ruby/object:Gem::Requirement
+  requirement: &70235629941120 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70099734287280
+  version_requirements: *70235629941120
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70099734286420 !ruby/object:Gem::Requirement
+  requirement: &70235629938800 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70099734286420
+  version_requirements: *70235629938800
 - !ruby/object:Gem::Dependency
   name: awesome_print
-  requirement: &70099734285000 !ruby/object:Gem::Requirement
+  requirement: &70235629961180 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +76,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70099734285000
+  version_requirements: *70235629961180
 - !ruby/object:Gem::Dependency
   name: sinatra
-  requirement: &70099734283680 !ruby/object:Gem::Requirement
+  requirement: &70235629958300 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +87,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70099734283680
+  version_requirements: *70235629958300
 - !ruby/object:Gem::Dependency
   name: thin
-  requirement: &70099734295820 !ruby/object:Gem::Requirement
+  requirement: &70235629970420 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,10 +98,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70099734295820
+  version_requirements: *70235629970420
 - !ruby/object:Gem::Dependency
   name: haml
-  requirement: &70099734292820 !ruby/object:Gem::Requirement
+  requirement: &70235629969420 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -109,10 +109,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70099734292820
+  version_requirements: *70235629969420
 - !ruby/object:Gem::Dependency
   name: namespaced_redis
-  requirement: &70099734300600 !ruby/object:Gem::Requirement
+  requirement: &70235629968260 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -120,7 +120,7 @@ dependencies:
         version: 1.0.2
   type: :runtime
   prerelease: false
-  version_requirements: *70099734300600
+  version_requirements: *70235629968260
 description: Web Crawler that uses resque background job engine to allow you to cluster
   your crawl.
 email: stewart@rockwellcottage.com