horseman 0.0.2 → 0.0.3

data/lib/horseman/action.rb

@@ -0,0 +1,20 @@
+require 'horseman/response'
+
+module Horseman
+  class Action
+    attr_reader :uri, :response
+    
+    def initialize(uri, response=nil)
+      @uri = uri
+      @response = Horseman::Response.new(response.body, response.to_hash) unless response.nil?
+    end
+    
+    def url
+      "#{@uri.scheme}://#{@uri.host}#{@uri.path}" + (@uri.query ? "?#{@uri.query}" : "")
+    end
+    
+    def relative_root
+      "#{url.rpartition('/')[0]}/"
+    end
+  end
+end

data/lib/horseman/browser.rb

@@ -1,66 +1,117 @@
-require 'horseman/response'
+require 'horseman/action'
+require 'horseman/connection'
+require 'horseman/cookies'
 require 'securerandom'
 
 module Horseman
-  class Browser
+  class Browser    
+    MaxRedirects = 10
+    
     attr_accessor :base_url
-    attr_reader :cookies, :connection, :last_response, :multipart_boundary
+    attr_reader :cookies, :connection, :last_action, :multipart_boundary
+    
+    def self.with_base_url(base_url)
+      Horseman::Browser.new(Horseman::Connection.new, base_url)
+    end
     
     def initialize(connection, base_url='')
       @connection = connection
       @base_url = base_url
       @cookies = Horseman::Cookies.new
-      @multipart_boundary = "----HorsemanBoundary#{SecureRandom.base64(8)}"
+      @multipart_boundary = "----HorsemanBoundary#{SecureRandom.hex(8)}"
     end
     
     def clear_session
       @cookies.clear
     end
     
-    def get!(path = '/')
-      request = @connection.build_request(:url => "#{@base_url}#{path}", :verb => :get)
-      exec request
+    def get!(path = '/', options = {})
+      url = options[:no_base_url] ? path : "#{@base_url}#{path}"
+      request = @connection.build_request(:url => url, :verb => :get)
+      redirects = options[:redirects] || 0
+      exec(request, redirects)
     end
     
-    def post!(path = '/', form = :form, data = {})
+    def post!(path = '/', options = {})
       get! path
-      selected_form = @last_response.forms.select {|f| f.id.to_sym == form}.first
-      raise "Could not find form #{:form}" if selected_form.nil?
+      
+      form = options[:form] || :form
+      data = options[:data] || {}
+      unchecked = options[:unchecked] || []
+      
+      selected_form = @last_action.response.forms.select {|f| f.id && f.id.to_sym == form}.first
+      raise "Could not find form #{form}" if selected_form.nil?
 
       selected_form.fields.each do |f|
-        data[f.name.to_sym] ||= f.value
+        data[f.name.to_sym] ||= f.value unless unchecked.include? f.name.to_sym
       end
       request_body = build_request_body(data, selected_form.encoding)
-      request = @connection.build_request(:url => "#{@base_url}#{path}", :verb => :post, :body => request_body)
+      
+      if is_absolute_url(selected_form.action)
+        # Absolute action http://www.example.com/action
+        url = selected_form.action
+      elsif selected_form.action == ''
+        # No action, post to same URL as GET request
+        url = "#{@last_action.url}"
+      else
+        # Relative action, use relative root from last action
+        url = "#{@last_action.relative_root}#{selected_form.action}"
+      end
+      
+      request = @connection.build_request(:url => "#{url}", :verb => :post, :body => request_body)
       request['Content-Type'] = case selected_form.encoding
                                 when :multipart
                                   "multipart/form-data; boundary=#{@multipart_boundary}"
                                 else
                                   "application/x-www-form-urlencoded"
                                 end
+      request['Referer'] = @last_action.url
+      
       exec request
     end
     
     private
     
-    def exec(request)
-      request['cookie'] = @cookies.to_s
+    def exec(request, redirects=0)
+      request['Cookie'] = @cookies.to_s
+      request['Content-Length'] = request.body ? request.body.length : 0
+      request['User-Agent'] = 'Horseman'
+      
+      request.each_header {|k,v| puts "#{k}: #{v}"}
+      puts request.body
+      
       response = @connection.exec_request(request)
+      
       @cookies.update(response.get_fields('set-cookie'))
-      @last_response = Horseman::Response.new(response.body)
+      @last_action = Horseman::Action.new(@connection.uri, response)
+
+      code = response.code
+      puts code
+      
+      if ['301','302','303','307'].include? code
+        raise "Redirect limit reached" if redirects >= MaxRedirects
+        
+        redirect_url = response['location']
+        if !is_absolute_url(redirect_url)
+          redirect_url = "#{@last_action.relative_root}#{redirect_url}"
+        end
+        puts redirect_url
+        get!(redirect_url, :redirects => redirects+1, :no_base_url => true)
+      end
     end
     
     def build_request_body(data, encoding=:url)
       if encoding == :multipart
         data.map do |k,v|
-          %{#{@multipart_boundary}
-            Content-Disposition: form-data; name="#{k}"
-            
-            #{v}}
-        end.join("\n") + @multipart_boundary
+          "--#{@multipart_boundary}\r\nContent-Disposition: form-data; name=\"#{k}\"\r\n\r\n#{v}"
+        end.join("\r\n") + "\r\n--#{@multipart_boundary}--"
       else
         data.map {|k,v| "#{k}=#{v}"}.join('&')
       end
     end
+    
+    def is_absolute_url(url)
+      url[/\w+:\/\/.*/]
+    end
   end
 end

data/lib/horseman/connection.rb

@@ -4,7 +4,7 @@ require 'net/https'
 
 module Horseman
   class Connection
-    attr_reader :http
+    attr_reader :http, :uri
 
     def url=(url)
       @uri = URI.parse(url)
@@ -17,6 +17,8 @@ module Horseman
        
     def build_request(options={})
       self.url = options[:url] unless options[:url].nil?
+      
+      puts "#{(options[:verb] || :get).to_s.upcase} #{options[:url] || 'none given'}"
       options[:verb] == (:get || nil) ? build_get_request : build_post_request(options[:body])
     end
     

data/lib/horseman/response.rb

@@ -9,17 +9,18 @@ module Horseman
     end
   end
   class Form < Element
-    attr_accessor :encoding, :fields, :submit
+    attr_accessor :action, :encoding, :fields, :submit
   end
   class FormField < Element
     attr_accessor :type, :value
   end
   
   class Response
-    attr_reader :body, :forms
+    attr_reader :body, :headers, :forms
     
-    def initialize(body)
+    def initialize(body, headers={})
       @body = body
+      @headers = headers
       @forms = []
       
       @field_types = {
@@ -36,23 +37,26 @@ module Horseman
       parse
     end
     
+    def[](key)
+      @headers[key]
+    end
+    
     private
     
     def parse
       doc = Nokogiri::HTML(@body)
-      doc.css('form').each do |f|
+      doc.css('form').select{|f| f.attr('id')}.each do |f|
         form = Form.new(f.attr('id'), f.attr('name'))
+        form.action = f.attr('action') || '/'
         form.encoding = @encoding_types[f.attr('enctype')] || :url
         form.fields = []
-        f.css('input').each do |i|
+        f.css('input').select{|i| i.attr('name')}.each do |i|
           field = FormField.new(i.attr('id'), i.attr('name'))
           field.type = @field_types[i.attr('type')] || :text
           field.value = i.attr('value')
-          if (field.type == :submit)
-            form.submit = field
-          elsif
-            form.fields << field 
-          end
+
+          form.fields << field 
+          form.submit = field if (field.type == :submit)  
         end
         @forms << form
       end

data/lib/horseman/version.rb

@@ -1,3 +1,3 @@
 module Horseman
-  VERSION = "0.0.2"
+  VERSION = "0.0.3"
 end

data/lib/horseman.rb

@@ -1 +1,2 @@
+require 'horseman/browser'
 require 'horseman/version'

data/spec/horseman/action_spec.rb

@@ -0,0 +1,29 @@
+require 'horseman/action'
+require 'uri'
+
+describe Horseman::Action do
+  
+  context "when given a URL without a query string" do
+    subject {described_class.new(URI.parse('http://www.example.com/path/file.html'))}
+  
+    it "provides access to full URL" do
+      subject.url.should eq 'http://www.example.com/path/file.html'
+    end
+  
+    it "provides access to a relative path root" do
+      subject.relative_root.should eq 'http://www.example.com/path/'
+    end
+  end
+  
+  context "when given a URL with a query string" do
+    subject {described_class.new(URI.parse('http://www.example.com/path/file.html?q1=value'))}
+  
+    it "provides access to full URL" do
+      subject.url.should eq 'http://www.example.com/path/file.html?q1=value'
+    end
+  
+    it "provides access to a relative path root" do
+      subject.relative_root.should eq 'http://www.example.com/path/'
+    end
+  end
+end

data/spec/horseman/browser_spec.rb

@@ -30,11 +30,87 @@ describe Horseman::Browser do
   
   it "stores information about the last response" do
     subject.get!
-    subject.last_response.body.should eq html
+    subject.last_action.response.body.should eq html
   end
-  
+
+  context "when processing redirects" do
+    
+    def build_browser(options)
+      code = options[:code] || '200'
+      num_redirects = options[:redirects] || 1
+      location = options[:location]
+      r = response(:location => location)
+      redirects = 0
+      r.stub(:code) do
+        code = (redirects >= num_redirects) ? '200' : code
+        redirects += 1
+        code
+      end
+      
+      c = connection
+      c.stub(:exec_request) { r }
+
+      described_class.new(c, 'http://www.example.com')    
+    end
+    
+    def should_redirect(options)
+      browser = build_browser(options)
+
+      expected_url = options[:expected_url] || 'http://www.anotherdomain.com/path'  
+      redirects = 0
+      browser.connection.should_receive(:build_request).twice do |options|
+        browser.connection.url = options[:url]
+        if redirects > 0
+          options[:url].should eq expected_url
+        end
+        redirects += 1
+        request
+      end
+      browser.get!
+    end
+    
+    it "follows 301s" do
+      should_redirect(:code => '301')
+    end
+    
+    it "follows 302s" do
+      should_redirect(:code => '302')
+    end
+    
+    it "follows 303s" do
+      should_redirect(:code => '303')
+    end
+    
+    it "follows 307s" do
+      should_redirect(:code => '307')
+    end
+    
+    it "follows relative paths" do
+      should_redirect(:code => '301', :location => 'redirect', :expected_url => 'http://www.example.com/redirect')
+    end
+    
+    it "raises an error after 10 consecutive redirects" do
+      good_browser = build_browser(:code => '301', :redirects => 10)
+      good_browser.get!
+      
+      bad_browser = build_browser(:code => '301', :redirects => 11)
+      expect { bad_browser.get! }.to raise_error
+    end
+  end
+    
   context "when posting" do
-    def with_request
+    def describe_url
+      count = 0
+      subject.connection.should_receive(:build_request).twice do |options|
+        subject.connection.url = options[:url]
+        yield options[:url] if (count > 0)
+        count += 1
+        request
+      end
+      post
+    end
+    
+    def describe_request
       subject.connection.should_receive(:exec_request).twice do |request|
         if (request.method == "POST")
           yield request
@@ -43,43 +119,91 @@ describe Horseman::Browser do
       post
     end
     
-    it "raises an exception on invalid form id" do
-      expect { subject.post!('/', :bad_form) }.to raise_error
+    it "raises an error on invalid form id" do
+      expect { subject.post!('/', :form => :bad_form) }.to raise_error
+    end
+    
+    context "with a relative action" do
+      def post
+        subject.post!('/path', :form => :form1)
+      end
+      
+      it "constructs the correct URL" do
+        describe_url do |url|
+          url.should eq 'http://www.example.com/action'
+        end
+      end
+    end
+    
+    context "with an absolute action" do
+      def post
+        subject.post!('/path', :form => :form2)
+      end
+      
+      it "constructs the correct URL" do
+        describe_url do |url|
+          url.should eq 'http://www.anotherdomain.com/action'
+        end
+      end
     end
     
+    context "with no action" do
+      def post
+        subject.post!('/path', :form => :form3)
+      end
+      
+      it "constructs the correct URL" do
+        describe_url do |url|
+          url.should eq 'http://www.example.com/path'
+        end
+      end
+    end
+    
+    context "with unchecked checkboxes" do
+      def post
+        subject.post!('/path', :form => :form1, :data => {:text => "text_value"}, :unchecked => [:check])
+      end
+      
+      it "does not include unchecked checkboxes" do
+        describe_request do |request|
+          request.body.should_not match /name="check"/
+        end
+      end
+    end
+  
     context "multipart data" do
       def post
-        subject.post!('/action', :form1, {:text => "text_value", :check => "checkbox_value"})
+        subject.post!('/', :form => :form1, :data => {:text => "text_value", :check => "checkbox_value"})
       end
       
       it "properly sets content type" do
-        with_request do |request|
+        describe_request do |request|
           request['Content-Type'].should eq "multipart/form-data; boundary=#{subject.multipart_boundary}"
         end
       end
       
       it "properly encodes form data" do
-        with_request do |request|
-          request.body.should match /^#{subject.multipart_boundary}.*#{subject.multipart_boundary}$/m
-          request.body.should match /#{subject.multipart_boundary}.*Content-Disposition: form-data; name="text".*text_value/m
-          request.body.should match /#{subject.multipart_boundary}.*Content-Disposition: form-data; name="check".*checkbox_value/m
+        describe_request do |request|
+          request.body.should match /\A--#{subject.multipart_boundary}.*--#{subject.multipart_boundary}--\Z/m
+          request.body.should match /^--#{subject.multipart_boundary}\r\nContent-Disposition: form-data; name="text"\r\n\r\ntext_value/m
+          request.body.should match /^--#{subject.multipart_boundary}\r\nContent-Disposition: form-data; name="check"\r\n\r\ncheckbox_value/m
         end
       end
     end
 
     context "URL encoded data" do
       def post
-        subject.post!('/action', :form2, {:text1 => "value1", :text2 => "value2"})
+        subject.post!('/', :form => :form2, :data => {:text1 => "value1", :text2 => "value2"})
       end
     
       it "properly sets content type" do
-        with_request do |request|
+        describe_request do |request|
           request['Content-Type'].should eq 'application/x-www-form-urlencoded'
         end
       end
       
       it "properly encodes form data" do
-        with_request do |request|
+        describe_request do |request|
           request.body.should match /\w+=\w+&\w+=\w+/
           request.body.should match /text1=value1/
           request.body.should match /text2=value2/

data/spec/horseman/response_spec.rb

@@ -6,13 +6,20 @@ describe Horseman::Response do
   subject { described_class.new(html) }
   
   it "parses forms" do
-    subject.forms.count.should eq 2
+    subject.forms.count.should eq 3
+
     subject.forms[0].id.should eq 'form1'
+    subject.forms[0].action.should eq 'action'
+
     subject.forms[1].id.should eq 'form2'
+    subject.forms[1].action.should eq 'http://www.anotherdomain.com/action'
+    
+    subject.forms[2].id.should eq 'form3'
+    subject.forms[2].action.should eq ''
   end
   
   it "parses form fields" do
-    subject.forms[0].fields.count.should eq 3
+    subject.forms[0].fields.count.should eq 4
 
     subject.forms[0].fields[0].name.should eq 'text'
     subject.forms[0].fields[0].type.should eq :text
@@ -36,4 +43,20 @@ describe Horseman::Response do
     subject.forms[0].encoding = :multipart
     subject.forms[1].encoding = :url
   end
+  
+  context "with missing form ids" do
+    subject { described_class.new('<form></form><form id="form"></form>')}
+    it "ignores forms without ids" do
+      subject.forms.count.should eq 1
+      subject.forms[0].id.should eq 'form'
+    end
+  end
+  
+  context "with missing field names" do
+    subject { described_class.new('<form id="form"><input name="input1" /><input /></form>')}
+    it "ignores fields without names" do
+      subject.forms[0].fields.count.should eq 1
+      subject.forms[0].fields[0].name.should eq 'input1'
+    end
+  end
 end

data/spec/mocks.rb

@@ -13,11 +13,13 @@ module Mocks
             <input type="hidden" name="hidden" value="value3" />
             <input type="submit" name="submit1" value="OK" />
           </form>
-          <form id="form2" action="http://www.example.com/action">
+          <form id="form2" action="http://www.anotherdomain.com/action">
             <input type="text" name="text1" />
             <input type="text" name="text2" />
             <input type="submit" name="submit2" value="OK" />
           </form>
+          <form id="form3" action="">
+          </form>
         </body>
       </html>
     }
@@ -28,22 +30,34 @@ module Mocks
      'name2=value2; Domain=www.example.com; Path=/path; Expires=Sun, 1-Jan-2012 00:00:00 GMT']
   end
   
-  def response
-    m = double("HttpResponse")
-    m.stub(:[]) do |key|
+  def request
+    r = double("HttpRequest")
+    r.stub(:[]=) {}
+    r.stub(:body) {}
+    r.stub(:each_header) {}
+    r
+  end
+  
+  def response(options={})
+    r = double("HttpResponse")
+    r.stub(:code) { '200' }
+    r.stub(:[]) do |key|
       case key
       when 'set-cookie'
         cookies.join(', ')
+      when 'location'
+        options[:location] || 'http://www.anotherdomain.com/path'
       end
     end
-    m.stub(:get_fields) do |key|
+    r.stub(:to_hash) { Hash.new }
+    r.stub(:get_fields) do |key|
       case key
       when 'set-cookie'
         cookies
       end
     end
-    m.stub(:body) { html }
-    m
+    r.stub(:body) { html }
+    r
   end
   
   def connection

data/vendor/ruby/1.9.1/gems/gemcutter-0.7.1/Gemfile

@@ -0,0 +1,7 @@
+source 'http://rubygems.org'
+
+platforms :jruby do
+  gem 'jruby-openssl', '~> 0.7'
+end
+
+gemspec

data/vendor/ruby/1.9.1/gems/gemcutter-0.7.1/MIT-LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2009, Nick Quaranto
+
+Permission is hereby granted, free of charge, to any person
+obtaining a copy of this software and associated documentation
+files (the "Software"), to deal in the Software without
+restriction, including without limitation the rights to use,
+copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following
+conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.