http_validator 0.1.3 → 0.1.4

data/lib/http_validator.rb

@@ -1,8 +1,15 @@
 require 'curb'
 require 'uri'
 require 'nokogiri'
+require 'platform_helpers'
+
 class HttpValidator
-  VERSION = '0.1.3'
+  VERSION = '0.1.4'
+end
+
+module Constants
+  BASE = File.expand_path(File.dirname(__FILE__) + '/..')
+  USER_AGENTS = File.readlines(BASE + '/config/user_agents')
 end
 
 module HTTP
@@ -10,7 +17,7 @@ module HTTP
     def self.parse(str)
       headers = {}
       str.gsub(/\r/, '').split(/\n/).each { |i|
-        k, v = i.split(':')
+        k, v = i.split(': ')
         headers[k] = v
       }
       headers
@@ -18,14 +25,18 @@ module HTTP
   end
 
   class Browser
-    attr_accessor :base_url, :elements, :element_info
+    attr_accessor :base_url, :elements, :element_info, :browser, :random_user_agent
     def initialize(url='')
+      rand(Time.now.tv_sec)
       @base_url = url
+      @random_user_agent = false
       @browser = Curl::Easy.new
-      #@browser.enable_cookies = true
+      @browser.max_redirects=3
+      @browser.enable_cookies = true
       @browser.cookiejar = 'cookies.txt'
       @browser.headers["User-Agent"] = "Mozilla/5.0 (X11; Linux i686) AppleWebKit/535.1 (KHTML, like Gecko) Chrome/14.0.835.186 Safari/535.1"
-      @element_types = %w(script img link)
+      @element_types = %w(script img link a)
+      @moved_keys = ['HTTP/1.1 302 Moved Temporarily', 'HTTP/1.1 301 Moved Permanently']
       clear
     end
 
@@ -35,14 +46,31 @@ module HTTP
     end
 
     def element_details(e)
-      return if %w(text/css text/javascript before-content data after-content stylesheet top).include?(e)
       if e =~ /^http:\/\/|^https:\/\//
-        @browser.url = e
+        url = e
       else
-        @browser.url = "#{@base_url.gsub(/\/$/, '')}/#{e}"
+        url = "#{@base_url.gsub(/\/$/, '')}/#{e}"
+      end
+      url.gsub!(/\/\//, '/')
+      url.gsub!(/http:\//, 'http://')
+      url.gsub!(/https:\//, 'https://')
+
+      @browser.url = url.strip
+      if @random_user_agent
+        index = rand(Constants::USER_AGENTS.length)
+        @browser.headers["User-Agent"] = Constants::USER_AGENTS[index].gsub(/\n/, '')
       end
       @browser.perform
+
       headers = HTTP::Header.parse(@browser.header_str)
+      @moved_keys.each { |k|
+        if headers.has_key?(k)
+          @elements << headers['Location']
+          @elements.delete(e)
+          @element_info.delete(e)
+          return
+        end
+      }
       if @element_info.has_key?(e)
         @element_info[e] << {:expected_size => headers['Content-Length'].strip.to_i, 
                             :actual_size => @browser.body_str.length, :headers => headers}
@@ -62,16 +90,40 @@ module HTTP
       doc = Nokogiri::HTML(@browser.body_str)
       @element_types.each { |elem|
         e = nil
+
         doc.search("//#{elem}").each { |item| 
-          next unless item.attribute_nodes.length > 0
-          item.attribute_nodes.each { |node|
-            next unless node.value =~ /\//
-            e = node.value
-            break
+          begin
+            if elem == 'a' || elem == 'link'
+              e = item.attributes['href'].value
+            elsif %(img script).include?(elem)
+              e = item.attributes['src'].value
+            end
+          rescue => ex
+          end
+          @elements << e unless e == nil
+        }
+      }
+    end
+  end
+
+  module UserAgent
+    @browser = HTTP::Browser.new.browser
+    def self.get
+      save_to = File.expand_path(File.dirname(__FILE__) + '/../config/user_agents.yaml')
+      agents = []
+      @browser.url = 'http://www.useragentstring.com/pages/All/'
+      @browser.perform
+      doc = Nokogiri::HTML(@browser.body_str)
+
+      fd = File.open(save_to, 'w+')
+      doc.search("//ul").each { |ul|
+        ul.search("//li").each { |li|
+          li.search("//a").each { |a|
+            fd.puts a.text
           }
-          @elements << e
         }
       }
+      fd.close
     end
   end
 
@@ -80,10 +132,10 @@ module HTTP
 
     def self.display_report(element_info, format='plain')
       if format == 'plain'
-        printf "%-100s%15s%15s\n\n", "Element", "Expected Size", "Actual Size"
+        printf "%-90s%15s%15s\n\n", "Element", "Expected Size", "Actual Size"
         element_info.each_pair { |resource, values|
           values.each { |item|
-            printf "%-100s%15s%15s\n", resource, item[:expected_size], item[:actual_size]
+            printf "%-90s%15s%15s\n", resource, item[:expected_size], item[:actual_size]
           }
         }
       elsif format == 'cucumber_example'
@@ -97,10 +149,18 @@ module HTTP
     end
 
     def self.run(url='', params={})
-      p params
       raise ArgumentError unless url =~ /^http:\/\/|https:\/\//
+      params[:random_user_agent] ||= 'false'
       URI.parse(url) # see if parsable, blow up if not
       @browser.base_url = url
+      if params.has_key?(:proxy)
+        @browser.browser.proxy_url = params[:proxy]
+      end
+
+      if params[:random_user_agent].to_bool
+        @browser.random_user_agent=true
+      end
+
       @browser.clear
       @browser.get_elements
       @browser.elements.each { |e| @browser.element_details(e) }
@@ -113,8 +173,10 @@ module HTTP
   end
 end
 
-#HTTP::Validator.run('http://slashdot.org')
-#HTTP::Validator.run('https://github.com')
+#HTTP::Validator.run('http://www.gap.com', {:display_report => {:format => 'plain'}})
+#HTTP::Validator.run('http://127.0.0.1:9090', {:random_user_agent => true, :display_report => {:format => 'plain'}})
+#HTTP::Validator.run('http://127.0.0.1:9090', {:display_report => {:format => 'plain'}})
+#HTTP::Validator.run('https://github.com', {:display_report => {:format => 'plain'}})
 
 #HTTP::Validator.run('http://127.0.0.1:9090', :display_report => {:format => 'cucumber_example'})
 

data/test/features/resource_size.feature

@@ -8,6 +8,6 @@ Feature: Download elements from source html
 
     Examples:
       |page                  |
-      |http://slashdot.org   |
-      |https://github.com    |
-      |http://www.yahoo.com  |
+      #|http://127.0.0.1:9090  |
+      #|http://www.yahoo.com  |
+      |http://www.gap.com  |

data/test/features/step_defn/verify.rb

@@ -1,15 +1,27 @@
 require 'rspec'
-require 'http_validator'
+base = File.expand_path(File.dirname(__FILE__) + '/../../..')
+require base + '/lib/http_validator'
+
+Before do
+  @proxy = nil
+  @params = {:random_user_agent => true}
+end
+
+Given /the user is using a proxy (.+) then set a proxy/ do |proxy|
+  puts "Proxy set to #{proxy}"
+  @proxy = proxy
+end
 
 Given /a user has landed on a (.+)/ do |url|
-  @element_info = HTTP::Validator.run(url)
+  @params[:proxy] = @proxy if @proxy
+  @element_info = HTTP::Validator.run(url, @params)
 end
 
 Then /all resource elements should download with the correct size/ do
   @element_info.each_pair { |resource, values|
     values.each { |item|
-      puts "Verifying #{resource} downloaded size #{item[:actual_size]} matches Content-Length #{item[:expected_size]}\n"
-      item[:actual_size].should equal item[:expected_size]
+      puts "Verifying #{resource} downloaded size #{item[:actual_size]} matches Content-Length #{item[:expected_size]}\n<br>"
+      item[:actual_size].should == item[:expected_size]
     }
   }
 end

metadata

@@ -2,7 +2,7 @@
 name: http_validator
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors: 
 - Cliff Cyphers
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-10-12 00:00:00 Z
+date: 2011-10-14 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: curb
@@ -24,16 +24,27 @@ dependencies:
   type: :runtime
   version_requirements: *id001
 - !ruby/object:Gem::Dependency 
-  name: hoe
+  name: platform_helpers
   prerelease: false
   requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - "="
+      - !ruby/object:Gem::Version 
+        version: 0.1.2
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: hoe
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
       - !ruby/object:Gem::Version 
         version: "2.12"
   type: :development
-  version_requirements: *id002
+  version_requirements: *id003
 description: |-
   Validate http request content length vs actual for 
   elements such as img, link, etc..
@@ -56,6 +67,7 @@ files:
 - lib/http_validator.rb
 - test/features/step_defn/verify.rb
 - test/features/resource_size.feature
+- config/user_agents
 - .gemtest
 homepage: http://github.com/ccyphers
 licenses: []