krawler 1.0.9 → 1.0.10

data/lib/krawler.rb

@@ -1,13 +1,17 @@
 require 'krawler/version'
+require 'krawler/authentication'
 require 'mechanize'
 require 'timeout'
 require 'uri'
 require 'thread'
+require 'pry'
 
 module Krawler
 
   class Base
 
+    include Authentication
+
     def initialize(url, options)
       @url              = URI(url)
       @host             = "#{@url.scheme}://#{@url.host}"
@@ -28,7 +32,7 @@ module Krawler
       @agent            = Mechanize.new
       @agent.user_agent = 'Krawler'
       @agent.ssl_version = 'SSLv3'
-      @headers          = { 'Accept-Encoding' => 'gzip, deflate' }
+      @headers           = { 'Accept-Encoding' => 'gzip, deflate' }
       @headers['Cache-Control'] = 'no-cache' if options[:no_cache]
     end
   
@@ -53,33 +57,6 @@ module Krawler
       @suspect_links.each { |link| puts link }
     end
 
-    def authenticate(agent, user, password, login_url)
-      agent.get(login_url) do |page|
-        login_form = page.form
-
-        login_form['user[email]'] = user
-        login_form['user[password]'] = password
-
-        agent.submit(login_form, login_form.buttons.first)
-      end
-    end
-
-    def use_authentication?
-      !@username.nil? || !@password.nil? || !@login_url.nil?
-    end
-
-    def validate_authentication_options
-      any_nil = [@login_url, @username, @password].any? {|v| v.nil?}
-      all_nil = [@login_url, @username, @password].all? {|v| v.nil?}
-      if (any_nil && !all_nil)
-        puts "You must either provide all authentication options" +
-          " (username, password, and loginurl) or provide none."
-        return false
-      else
-        return true
-      end
-    end
-
     def initialize_threads(agent)
       threads = []
       @threads.times do |i|
@@ -95,7 +72,6 @@ module Krawler
                 @links_to_crawl.pop
               end
             }
-
             crawl_page(link, agent)
           end
         end
@@ -135,6 +111,9 @@ module Krawler
   
       @mutex.synchronize do
         return if !page.respond_to?(:links)
+
+        recache_invalid_results(page)
+
         page.links.each do |new_link|
           next if new_link.href.nil?
           next if new_link.rel.include? 'nofollow'
@@ -169,5 +148,28 @@ module Krawler
         end
       end
     end
+
+    protected
+
+    def params_to_hash(params)
+      params = CGI.unescape(params)
+      Hash[ params.split('&').map { |p| p.split('=') } ]
+    end
+
+    def hash_to_params(hash)
+      hash.map { |k, v| "#{k}=#{v}" }.sort * '&'
+    end
+
+    def recache_invalid_results(page)
+      page.search('tr td i.icon-remove').each do |invalid|
+        a = invalid.parent.parent.css('a').first
+        next if a.nil?
+        uri = URI(a['href'])
+        query = params_to_hash(uri.query || '')
+        query['cache'] = 'false'
+        uri.query = hash_to_params(query)
+        @links_to_crawl << uri.to_s
+      end
+    end
   end
 end

data/lib/krawler/authentication.rb

@@ -0,0 +1,33 @@
+module Krawler
+
+  module Authentication
+
+    def authenticate(agent, user, password, login_url)
+      agent.get(login_url) do |page|
+        login_form = page.form
+
+        login_form['user[email]'] = user
+        login_form['user[password]'] = password
+
+        agent.submit(login_form, login_form.buttons.first)
+      end
+    end
+
+    def use_authentication?
+      !@username.nil? || !@password.nil? || !@login_url.nil?
+    end
+
+    def validate_authentication_options
+      any_nil = [@login_url, @username, @password].any? {|v| v.nil?}
+      all_nil = [@login_url, @username, @password].all? {|v| v.nil?}
+      if (any_nil && !all_nil)
+        puts "You must either provide all authentication options" +
+          " (username, password, and loginurl) or provide none."
+        return false
+      else
+        return true
+      end
+    end
+  end
+
+end

data/lib/krawler/version.rb

@@ -1,3 +1,3 @@
 module Krawler
-  VERSION = '1.0.9'
+  VERSION = '1.0.10'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: krawler
 version: !ruby/object:Gem::Version
-  version: 1.0.9
+  version: 1.0.10
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-11-03 00:00:00.000000000 Z
+date: 2012-11-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize
-  requirement: &70292261892960 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,7 +21,12 @@ dependencies:
         version: 2.5.1
   type: :runtime
   prerelease: false
-  version_requirements: *70292261892960
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.5.1
 description: Simple little website crawler.
 email:
 - mike@urlgonomics.com
@@ -38,6 +43,7 @@ files:
 - bin/krawl
 - krawler.gemspec
 - lib/krawler.rb
+- lib/krawler/authentication.rb
 - lib/krawler/version.rb
 - tasks/krawler.rake
 homepage: https://github.com/mje113/krawl
@@ -60,7 +66,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: krawler
-rubygems_version: 1.8.17
+rubygems_version: 1.8.24
 signing_key: 
 specification_version: 3
 summary: ''