krawler 1.0.13 → 1.0.14

data/lib/krawler.rb

@@ -112,7 +112,7 @@ module Krawler
       @mutex.synchronize do
         return if !page.respond_to?(:links)
 
-        recache_invalid_results(page)
+        #recache_invalid_results(page)
 
         page.links.each do |new_link|
           next if new_link.href.nil?
@@ -168,7 +168,13 @@ module Krawler
         query = params_to_hash(uri.query || '')
         query['cache'] = 'false'
         uri.query = hash_to_params(query)
-        @links_to_crawl << uri.to_s
+        if @restrict  # don't crawl outside of our restricted base path
+          if @include && uri.path =~ /#{@include}/ # unless we match our inclusion
+            if !@crawled_links.include?(uri.path) && !@links_to_crawl.include?(uri.path) # don't crawl what we've alread crawled
+              @links_to_crawl << uri.to_s
+            end
+          end
+        end
       end
     end
   end

data/lib/krawler/version.rb

@@ -1,3 +1,3 @@
 module Krawler
-  VERSION = '1.0.13'
+  VERSION = '1.0.14'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: krawler
 version: !ruby/object:Gem::Version
-  version: 1.0.13
+  version: 1.0.14
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-12-10 00:00:00.000000000 Z
+date: 2013-03-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize
-  requirement: !ruby/object:Gem::Requirement
+  requirement: &70187441725840 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,12 +21,7 @@ dependencies:
         version: 2.5.1
   type: :runtime
   prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: 2.5.1
+  version_requirements: *70187441725840
 description: Simple little website crawler.
 email:
 - mike@urlgonomics.com
@@ -66,7 +61,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: krawler
-rubygems_version: 1.8.24
+rubygems_version: 1.8.17
 signing_key: 
 specification_version: 3
 summary: ''