cobweb 0.0.28 → 0.0.29

data/README.textile

@@ -1,5 +1,5 @@
 
-h1. Cobweb v0.0.28
+h1. Cobweb v0.0.29
 
 h2. Intro 
   
@@ -37,7 +37,8 @@ h3. Data Returned
     ** :scripts - url's from script tags
     ** :styles - url's from within link tags with rel of stylesheet and from url() directives with stylesheets
   * :crawl_id - the id used internally for identifying the crawl.  Can be used by the processing job to seperate crawls
-    
+  * :internal_urls - an array of urls with * wildcards that represent urls internal to the site (ie pages within the same domain)
+  
   The source for the links can be overridden, contact me for the syntax (don't have time to put it into this documentation, will as soon as i have time!)
 
 h2. Installation

data/lib/cobweb.rb

@@ -19,7 +19,7 @@ class Cobweb
   # investigate using event machine for single threaded crawling
   
   def self.version
-    "0.0.28"
+    "0.0.29"
   end
   
   def method_missing(method_sym, *arguments, &block)

data/lib/crawl_job.rb

@@ -48,7 +48,7 @@ class CrawlJob
         end
 
         # enqueue to processing queue
-        Resque.enqueue(const_get(content_request[:processing_queue]), content.merge({:redis_options => content_request[:redis_options], :source_id => content_request[:source_id], :crawl_id => content_request[:crawl_id]}))
+        Resque.enqueue(const_get(content_request[:processing_queue]), content.merge({:internal_urls => internal_patterns, :redis_options => content_request[:redis_options], :source_id => content_request[:source_id], :crawl_id => content_request[:crawl_id]}))
         puts "#{content_request[:url]} has been sent for processing." if content_request[:debug]
         puts "Crawled: #{@crawl_counter} Limit: #{content_request[:crawl_limit]} Queued: #{@queue_counter}" if content_request[:debug]
 
@@ -89,10 +89,9 @@ class CrawlJob
   end
   
   def self.internal_link?(link)
-    puts "Checking for internal link for: #{link}" if @debug
-    @internal_patterns ||= @redis.smembers("internal_urls").map{|pattern| Regexp.new("^#{pattern.gsub("*", ".*?")}")}
+    puts "Checking internal link for: #{link}" if @debug
     valid_link = true
-    @internal_patterns.each do |pattern|
+    internal_patterns.map{|pattern| Regexp.new("^#{pattern.gsub("*", ".*?")}")}.each do |pattern|
       puts "Matching against #{pattern.source}" if @debug
       if link.match(pattern)
         puts "Matched as internal" if @debug
@@ -102,9 +101,17 @@ class CrawlJob
     puts "Didn't match any pattern so marked as not internal" if @debug
     false
   end
+  
+  def self.internal_patterns
+    @internal_patterns ||= @redis.smembers("internal_urls")
+  end
 
   def self.all_links_from_content(content)
-    content[:links].keys.map{|key| content[:links][key]}.flatten
+    links = content[:links].keys.map{|key| content[:links][key]}.flatten
+    links.reject!{|link| link.starts_with?("javascript:")}
+    links = links.map{|link| Addressable::URI.join(content[:url], link)}
+    links.select!{|link| link.scheme == "http" || link.scheme == "https"}
+    links
   end
   
   def self.enqueue_content(content_request, link)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: cobweb
 version: !ruby/object:Gem::Version
-  version: 0.0.28
+  version: 0.0.29
   prerelease: 
 platform: ruby
 authors:
@@ -13,7 +13,7 @@ date: 2012-03-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: resque
-  requirement: &70349176950540 !ruby/object:Gem::Requirement
+  requirement: &70189333520340 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70349176950540
+  version_requirements: *70189333520340
 - !ruby/object:Gem::Dependency
   name: redis
-  requirement: &70349176950120 !ruby/object:Gem::Requirement
+  requirement: &70189333519900 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70349176950120
+  version_requirements: *70189333519900
 - !ruby/object:Gem::Dependency
   name: nokogiri
-  requirement: &70349176949680 !ruby/object:Gem::Requirement
+  requirement: &70189333519480 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70349176949680
+  version_requirements: *70189333519480
 - !ruby/object:Gem::Dependency
   name: addressable
-  requirement: &70349176949260 !ruby/object:Gem::Requirement
+  requirement: &70189333519060 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70349176949260
+  version_requirements: *70189333519060
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70349176948840 !ruby/object:Gem::Requirement
+  requirement: &70189333518640 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70349176948840
+  version_requirements: *70189333518640
 - !ruby/object:Gem::Dependency
   name: awesome_print
-  requirement: &70349176948420 !ruby/object:Gem::Requirement
+  requirement: &70189333518220 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,10 +76,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70349176948420
+  version_requirements: *70189333518220
 - !ruby/object:Gem::Dependency
   name: sinatra
-  requirement: &70349176948000 !ruby/object:Gem::Requirement
+  requirement: &70189333517800 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -87,10 +87,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70349176948000
+  version_requirements: *70189333517800
 - !ruby/object:Gem::Dependency
   name: thin
-  requirement: &70349176947580 !ruby/object:Gem::Requirement
+  requirement: &70189333517380 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,10 +98,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70349176947580
+  version_requirements: *70189333517380
 - !ruby/object:Gem::Dependency
   name: haml
-  requirement: &70349176947160 !ruby/object:Gem::Requirement
+  requirement: &70189333533320 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -109,7 +109,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70349176947160
+  version_requirements: *70189333533320
 description: Web Crawler that uses resque background job engine to allow you to cluster
   your crawl.
 email: stewart@rockwellcottage.com