krawler 1.0.7 → 1.0.8

data/bin/krawl

@@ -29,6 +29,18 @@ optparse = OptionParser.new do |opts|
     options[:nc] = true
   end
 
+  opts.on('-l[login_url]', '--login_url[=login_url]', 'Login URL') do |login_url|
+    options[:l] = login_url
+  end
+
+  opts.on('-u[username]', '--username[=username]', 'Username') do |username|
+    options[:u] = username
+  end
+
+  opts.on('-p[password]', '--password[=password]', 'Password') do |password|
+    options[:p] = password
+  end
+
   opts.separator ''
 
   opts.on('-h', '--help', 'Show this help message.') { puts opts; exit }
@@ -46,5 +58,8 @@ Krawler::Base.new(ARGV[0] || 'http://localhost:3000/', {
   :restrict  => options[:s],
   :threads   => options[:c],
   :randomize => options[:r],
-  :no_cache  => options[:nc]
+  :no_cache  => options[:nc],
+  :username  => options[:u],
+  :password  => options[:p],
+  :login_url => options[:l]
 }).base

data/krawler.gemspec

@@ -8,7 +8,7 @@ Gem::Specification.new do |gem|
   gem.summary       = %q{}
   gem.homepage      = 'https://github.com/mje113/krawl'
 
-  gem.add_dependency 'mechanize', '~> 2.5.0'
+  gem.add_dependency 'mechanize', '~> 2.5.1'
   gem.rubyforge_project = 'krawler'
 
   gem.files         = `git ls-files`.split($\)

data/lib/krawler/version.rb

@@ -1,3 +1,3 @@
 module Krawler
-  VERSION = '1.0.7'
+  VERSION = '1.0.8'
 end

data/lib/krawler.rb

@@ -3,7 +3,6 @@ require 'mechanize'
 require 'timeout'
 require 'uri'
 require 'thread'
-require 'pry'
 module Krawler
 
   class Base
@@ -20,6 +19,9 @@ module Krawler
       @restrict         = options[:restrict]
       @randomize        = options[:randomize]
       @threads          = options[:threads]   || 1
+      @username         = options[:username]
+      @password         = options[:password]
+      @login_url       = options[:login_url]
       @mutex            = Mutex.new
       @agent            = Mechanize.new
       @agent.user_agent = 'Krawler'
@@ -29,8 +31,14 @@ module Krawler
     end
   
     def base
+      return -1 unless validate_authentication_options
+
       puts "Krawling..."
 
+      if use_authentication?
+        authenticate(@agent, @username, @password, @login_url)
+      end
+
       crawl_page(@url, @agent)
       initialize_threads(@agent)
     
@@ -43,6 +51,33 @@ module Krawler
       @suspect_links.each { |link| puts link }
     end
 
+    def authenticate(agent, user, password, login_url)
+      agent.get(login_url) do |page|
+        login_form = page.form
+
+        login_form['user[email]'] = user
+        login_form['user[password]'] = password
+
+        agent.submit(login_form, login_form.buttons.first)
+      end
+    end
+
+    def use_authentication?
+      !@username.nil? || !@password.nil? || !@login_url.nil?
+    end
+
+    def validate_authentication_options
+      any_nil = [@login_url, @username, @password].any? {|v| v.nil?}
+      all_nil = [@login_url, @username, @password].all? {|v| v.nil?}
+      if (any_nil && !all_nil)
+        puts "You must either provide all authentication options" +
+          " (username, password, and loginurl) or provide none."
+        return false
+      else
+        return true
+      end
+    end
+
     def initialize_threads(agent)
       threads = []
       @threads.times do |i|

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: krawler
 version: !ruby/object:Gem::Version
-  version: 1.0.7
+  version: 1.0.8
   prerelease: 
 platform: ruby
 authors:
@@ -9,19 +9,19 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-09-04 00:00:00.000000000 Z
+date: 2012-10-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mechanize
-  requirement: &70222535947940 !ruby/object:Gem::Requirement
+  requirement: &70166215374240 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 2.5.0
+        version: 2.5.1
   type: :runtime
   prerelease: false
-  version_requirements: *70222535947940
+  version_requirements: *70166215374240
 description: Simple little website crawler.
 email:
 - mike@urlgonomics.com
@@ -65,3 +65,4 @@ signing_key:
 specification_version: 3
 summary: ''
 test_files: []
+has_rdoc: