horseman 0.0.1

data/Gemfile

@@ -0,0 +1,6 @@
+source :rubygems
+
+gemspec
+
+gem "echoe", "~>4.6.3"
+gem "rspec", "~>2.7"

data/Gemfile.lock

@@ -0,0 +1,42 @@
+PATH
+  remote: .
+  specs:
+    horseman (0.0.1)
+      nokogiri (>= 1.5.0)
+
+GEM
+  remote: http://rubygems.org/
+  specs:
+    allison (2.0.3)
+    diff-lcs (1.1.3)
+    echoe (4.6.3)
+      allison (>= 2.0.3)
+      gemcutter (>= 0.7.0)
+      rake (>= 0.9.2)
+      rdoc (>= 3.6.1)
+      rubyforge (>= 2.0.4)
+    gemcutter (0.7.0)
+    json (1.6.2)
+    json_pure (1.6.2)
+    nokogiri (1.5.0)
+    rake (0.9.2.2)
+    rdoc (3.11)
+      json (~> 1.4)
+    rspec (2.7.0)
+      rspec-core (~> 2.7.0)
+      rspec-expectations (~> 2.7.0)
+      rspec-mocks (~> 2.7.0)
+    rspec-core (2.7.1)
+    rspec-expectations (2.7.0)
+      diff-lcs (~> 1.1.2)
+    rspec-mocks (2.7.0)
+    rubyforge (2.0.4)
+      json_pure (>= 1.1.7)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  echoe (~> 4.6.3)
+  horseman!
+  rspec (~> 2.7)

data/Manifest

@@ -0,0 +1,20 @@
+Gemfile
+Gemfile.lock
+Manifest
+README.rdoc
+Rakefile
+horseman.gemspec
+lib/horseman.rb
+lib/horseman/browser.rb
+lib/horseman/connection.rb
+lib/horseman/cookies.rb
+lib/horseman/hidden_fields.rb
+lib/horseman/response.rb
+lib/horseman/version.rb
+spec/horseman/browser_spec.rb
+spec/horseman/connection_spec.rb
+spec/horseman/cookies_spec.rb
+spec/horseman/hidden_fields_spec.rb
+spec/horseman/response_spec.rb
+spec/mocks.rb
+spec/spec_helper.rb

data/README.rdoc

@@ -0,0 +1,3 @@
+= Horseman
+
+Headless HTTP crawler/scraper for web applications built with ASP.NET WebForms.

data/Rakefile

@@ -0,0 +1,24 @@
+require 'rake' 
+require 'rspec/core/rake_task' 
+require 'echoe'
+  
+Echoe.new("horseman", "0.0.1") do |p|  
+  p.description     = "Headless HTTP crawler/scraper for ASP.NET WebForms applications"  
+  p.url             = "http://jarrodpeace.com"  
+  p.author          = "Jarrod Peace"  
+  p.email           = "peace.jarrod@gmail.com"  
+  p.ignore_pattern  = FileList[".gitignore"]  
+  p.development_dependencies = []  
+  p.runtime_dependencies = ["nokogiri >=1.5.0"]
+end  
+  
+Dir["#{File.dirname(__FILE__)}/tasks/*.rake"].sort.each { |ext| load ext }
+
+
+desc "Default task - runs specs"
+task :default => :spec
+
+desc "Run specs"
+RSpec::Core::RakeTask.new do |t|
+  t.rspec_opts = '-cfd'
+end

data/horseman.gemspec

@@ -0,0 +1,32 @@
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = "horseman"
+  s.version = "0.0.1"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Jarrod Peace"]
+  s.date = "2012-01-06"
+  s.description = "Headless HTTP crawler/scraper for ASP.NET WebForms applications"
+  s.email = "peace.jarrod@gmail.com"
+  s.extra_rdoc_files = ["README.rdoc", "lib/horseman.rb", "lib/horseman/browser.rb", "lib/horseman/connection.rb", "lib/horseman/cookies.rb", "lib/horseman/hidden_fields.rb", "lib/horseman/response.rb", "lib/horseman/version.rb"]
+  s.files = ["Gemfile", "Gemfile.lock", "Manifest", "README.rdoc", "Rakefile", "horseman.gemspec", "lib/horseman.rb", "lib/horseman/browser.rb", "lib/horseman/connection.rb", "lib/horseman/cookies.rb", "lib/horseman/hidden_fields.rb", "lib/horseman/response.rb", "lib/horseman/version.rb", "spec/horseman/browser_spec.rb", "spec/horseman/connection_spec.rb", "spec/horseman/cookies_spec.rb", "spec/horseman/hidden_fields_spec.rb", "spec/horseman/response_spec.rb", "spec/mocks.rb", "spec/spec_helper.rb"]
+  s.homepage = "http://jarrodpeace.com"
+  s.rdoc_options = ["--line-numbers", "--inline-source", "--title", "Horseman", "--main", "README.rdoc"]
+  s.require_paths = ["lib"]
+  s.rubyforge_project = "horseman"
+  s.rubygems_version = "1.8.10"
+  s.summary = "Headless HTTP crawler/scraper for ASP.NET WebForms applications"
+
+  if s.respond_to? :specification_version then
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<nokogiri>, [">= 1.5.0"])
+    else
+      s.add_dependency(%q<nokogiri>, [">= 1.5.0"])
+    end
+  else
+    s.add_dependency(%q<nokogiri>, [">= 1.5.0"])
+  end
+end

data/lib/horseman/browser.rb

@@ -0,0 +1,32 @@
+require 'horseman/response'
+
+module Horseman
+  class Browser
+    attr_accessor :base_url
+    attr_reader :cookies, :connection, :last_response
+    
+    def initialize(connection, base_url='')
+      @connection = connection
+      @base_url = base_url
+      @cookies = Horseman::Cookies.new
+    end
+    
+    def clear_session
+      @cookies.clear
+    end
+    
+    def get!(path = '/')
+      request = @connection.build_request(:url => "#{@base_url}#{path}", :verb => :get)
+      exec(request)
+    end
+    
+    private
+    
+    def exec(request)
+      request['cookie'] = @cookies.to_s
+      response = @connection.exec_request(request)
+      @cookies.update(response.get_fields('set-cookie'))
+      @last_response = Horseman::Response.new(response.body)
+    end
+  end
+end

data/lib/horseman/connection.rb

@@ -0,0 +1,42 @@
+require 'uri'
+require 'net/http'
+require 'net/https'
+
+module Horseman
+  class Connection
+    attr_reader :http
+
+    def url=(url)
+      @uri = URI.parse(url)
+      build_http
+    end
+
+    def exec_request(request)
+      @http.request(request)
+    end
+       
+    def build_request(options={})
+      self.url = options[:url] unless options[:url].nil?
+      options[:verb] == (:get || nil) ? build_get_request : build_post_request(options[:form])
+    end
+    
+    private
+    
+    def build_http
+      @http = Net::HTTP.new(@uri.host, @uri.port)
+      if (@uri.port == 443)
+        @http.use_ssl = true
+      end
+    end
+
+    def build_get_request
+      return Net::HTTP::Get.new(@uri.request_uri)
+    end
+    
+    def build_post_request(form)
+      ret = Net::HTTP::Post.new(@uri.request_uri)
+      ret.form_data = form unless form.nil?
+      return ret
+    end
+  end
+end

data/lib/horseman/cookies.rb

@@ -0,0 +1,80 @@
+module Horseman
+  class Cookie
+    attr_reader :value, :domain, :path, :expiration
+    
+    def initialize(value, attributes)
+      @value = value
+      attributes.each {|a| parse_attribute(a)}
+    end
+    
+    private
+    
+    def parse_attribute(attribute)
+      md = /(\w+)=(.*)/.match(attribute)
+      if md
+        case md.captures[0].downcase
+        when 'domain'
+          @domain = md.captures[1]
+        when 'path'
+          @path = md.captures[1]
+        when 'expires'
+          @expiration = DateTime.parse(md.captures[1])
+        when 'max-age'
+          @expiration = DateTime.now + (md.captures[1] / (60 * 60 * 24))
+        end
+      end
+    end    
+  end
+  
+  class Cookies
+    def initialize
+      clear
+    end
+    
+    def [](cookie_name)
+      return @dict[cookie_name].value unless @dict[cookie_name].nil?
+    end
+    
+    def get(cookie_name)
+      return @dict[cookie_name]
+    end
+    
+    def clear
+      @dict = {}
+    end
+    
+    def count
+      @dict.count
+    end
+    
+    def empty?
+      @dict.count == 0
+    end
+    
+    def to_s
+      @dict.map {|k,v| "#{k}=#{v.value}"}.join('; ')
+    end
+    
+    def update(header)
+      if header.is_a?(Array)
+        header.each {|h| parse_header(h)}
+      else
+        parse_header(header) unless header.nil?
+      end
+      self
+    end
+    
+    private
+
+    def parse_header(header)
+      nvp, *attributes = *(header.split(';'))
+      raise ArgumentError if nvp.nil?
+      md = /(\w+)=(.*)/.match(nvp)
+      raise ArgumentError if md.nil?
+      name = md.captures[0]
+      value = md.captures[1]
+
+      @dict.merge!({name => Horseman::Cookie.new(value, attributes)})
+    end
+  end
+end

data/lib/horseman/hidden_fields.rb

@@ -0,0 +1,13 @@
+module Horseman
+  class HiddenFields
+    attr_reader :tokens
+
+    def initialize(html)
+      rx = /<input.* type=["']hidden["'].* name=["'](\S+)["'].* value=["'](\S*)["'].* \/>/
+      @tokens = {}
+      html.scan(rx).each {|field|
+        @tokens[field[0]] = field[1]
+      }
+    end
+  end
+end

data/lib/horseman/response.rb

@@ -0,0 +1,26 @@
+module Horseman
+  class Element
+    attr_accessor :id, :name
+  end
+  class Form < Element
+  end
+  class FormField < Element
+    attr_accessor :type, :value
+  end
+  
+  class Response
+    attr_reader :body, :forms
+    
+    def initialize(body)
+      @body = body
+      @forms = []
+      parse
+    end
+    
+    private
+    
+    def parse
+      
+    end
+  end
+end

data/lib/horseman/version.rb

@@ -0,0 +1,3 @@
+module Horseman
+  VERSION = "0.0.1"
+end

data/lib/horseman.rb

@@ -0,0 +1 @@
+require 'horseman/version'

data/spec/horseman/browser_spec.rb

@@ -0,0 +1,35 @@
+require 'horseman/browser'
+
+describe Horseman::Browser do
+  include Mocks
+  
+  subject {described_class.new(connection, 'http://www.example.com')}
+  
+  it "saves cookies" do
+    subject.cookies.should be_empty
+
+    subject.get!
+    subject.cookies.count.should eq 2
+    subject.cookies['name1'].should eq 'value1'
+    subject.cookies['name2'].should eq 'value2'
+
+    subject.connection.should_receive(:exec_request) do |request|
+      request['cookie'].should match /\w+=\w+; \w+=\w+/
+      request['cookie'].should match /name1=value1/
+      request['cookie'].should match /name2=value2/
+    end
+    subject.get!
+  end
+  
+  it "empties the cookies when the session is cleared" do
+    subject.get!
+    subject.cookies.should_not be_empty
+    subject.clear_session
+    subject.cookies.should be_empty
+  end
+  
+  it "stores information about the last response" do
+    subject.get!
+    subject.last_response.body.should eq html
+  end  
+end

data/spec/horseman/connection_spec.rb

@@ -0,0 +1,64 @@
+require 'horseman/connection'
+require 'net/http'
+
+describe Horseman::Connection do
+  subject do
+    c = described_class.new
+    c.url = 'http://www.example.com/some/path'
+    c
+  end
+  
+  context "when building requests" do  
+    let(:request) {subject.build_request(:verb => :get)}
+    
+    it "uses the proper path" do
+      request.path.should eq '/some/path'
+    end
+    
+    context "using GET" do
+      it "uses the proper request type" do
+        request.class.should be Net::HTTP::Get
+      end
+    end
+    
+    context "using POST" do
+      let(:request) {subject.build_request(:verb => :post)}
+      
+      it "uses the proper request type" do
+        request.class.should be Net::HTTP::Post
+      end
+
+      context "with form data" do
+        let(:request) {subject.build_request(:verb => :post, :form => {:field1=>'value1', :field2=>'value2'})}
+        
+        it "properly sets request body" do
+          request.body.should eq 'field1=value1&field2=value2'
+        end
+      end
+
+      context "without form data" do    
+        it "properly sets request body" do
+          request.body.should be_nil
+        end
+      end
+    end    
+  end
+  
+  context "when accessed using http" do
+    it "does not use SSL" do
+      subject.http.use_ssl?.should be_false
+    end
+  end
+  
+  context "when accessed using https" do
+    subject do
+      c = described_class.new
+      c.url = 'https://www.example.com'
+      c
+    end
+
+    it "uses SSL" do
+      subject.http.use_ssl?.should be_true
+    end
+  end  
+end

data/spec/horseman/cookies_spec.rb

@@ -0,0 +1,74 @@
+require 'horseman/cookies'
+
+class Yo
+  def test
+    pp "yo"
+  end
+end
+
+describe Horseman::Cookies do
+  let(:simple_header) {'name1=value1'}
+  let(:complex_header) {'name2=value2; Domain=www.example.com; Path=/path; Expires=Sun, 1-Jan-2012 00:00:00 GMT'}
+  
+  it "starts empty" do
+    subject.should be_empty
+  end
+  
+  it "accepts a single header" do
+    subject.update(simple_header)['name1'].should eq 'value1' 
+  end
+  
+  it "accepts multiple headers" do
+    subject.update([simple_header, complex_header])
+    subject['name1'].should eq 'value1'
+    subject['name2'].should eq 'value2'
+  end
+  
+  it "captures attributes" do
+    subject.update(complex_header)
+    subject.get('name2').domain.should eq 'www.example.com'
+    subject.get('name2').path.should eq '/path'
+    subject.get('name2').expiration.should eq DateTime.new(2012, 1, 1, 0, 0, 0, 0)
+  end
+  
+  it "accepts an empty array" do
+    subject.update([]).should be_empty
+  end
+  
+  it "accepts nil" do
+    subject.update(nil).should be_empty
+  end
+  
+  it "raises an exception on an unrecognized header" do
+    expect {subject.update('bad header')}.to raise_error(ArgumentError)
+  end
+  
+  it "generates a correct header" do
+    header = subject.update([simple_header, complex_header]).to_s
+    header.should match /\w+=\w+; \w+=\w+/
+    header.should match /name1=value1/
+    header.should match /name2=value2/
+  end
+  
+  context "with prexisting values" do
+    subject do
+      described_class.new.update('name1=other_value')
+    end
+    
+    it "returns nil for uninitialized values" do
+      subject['doesnt_exist'].should be_nil
+    end
+    
+    it "merges new values" do
+      subject.update(complex_header)
+      subject['name1'].should eq 'other_value'
+      subject['name2'].should eq 'value2'
+    end
+    
+    it "overwrites existing values" do
+      subject.update(simple_header)
+      subject['name1'].should eq 'value1'
+    end
+  end
+  
+end

data/spec/horseman/hidden_fields_spec.rb

@@ -0,0 +1,44 @@
+require 'horseman/hidden_fields'
+
+describe Horseman::HiddenFields do
+  
+  it "parses a single simple hidden input field" do
+    html = %{<input type="hidden" name="test" value="test_data" />}
+    cut = described_class.new(html)
+    
+    cut.tokens.size.should == 1
+    cut.tokens['test'].should == 'test_data'
+  end
+  
+  it "parses a single complex hidden input field" do
+    html = %{<input attr0="value0" type="hidden" attr1="value1" name="test" attr2="value2" value="test_data" attr3="value3" />}
+    cut = described_class.new(html)
+    
+    cut.tokens.size.should == 1
+    cut.tokens['test'].should == 'test_data'
+  end
+
+  it "parses multiple hidden input fields" do
+    html = %{
+        <input type="hidden" name="test" value="test_data" />
+        <input type="hidden" name="foo" value="bar" />
+        <some other="tag"></some>
+        <input type="hidden" name="dee" value="dum" />
+            }
+    cut = described_class.new(html)
+    
+    cut.tokens.size.should == 3
+    cut.tokens['test'].should == 'test_data'
+    cut.tokens['foo'].should == 'bar'
+    cut.tokens['dee'].should == 'dum'
+  end
+    
+  it "handles single quotes, too" do
+    html = %{<input type='hidden' name='test' value='test_data' />}
+    cut = described_class.new(html)
+    
+    cut.tokens.size.should == 1
+    cut.tokens['test'].should == 'test_data'
+  end
+  
+end

data/spec/horseman/response_spec.rb

@@ -0,0 +1,13 @@
+require 'horseman/response'
+
+describe Horseman::Response do
+  include Mocks
+  
+  subject { described_class.new(html) }
+  
+  it "parses forms" do
+    subject.forms.count.should eq 2
+    subject.forms[0].id.should eq 'form1'
+    subject.forms[1].id.should eq 'form2'
+  end
+end

data/spec/mocks.rb

@@ -0,0 +1,50 @@
+require 'horseman/connection'
+
+module Mocks
+  
+  def html
+    %{
+      <html>
+        <head></head>
+        <body>
+          <form id="form1">
+            <input type="text" name="name1" value="value1" />
+            <input type="submit" value="OK" />
+          </form>
+          <form id="form2">
+            <input type="text" name="name2" value="value2" />
+            <input type="submit" value="OK" />
+          </form>
+        </body>
+      </html>
+    }
+  end
+  
+  def cookies
+    ['name1=value1; Domain=www.example.com; Path=/path; Expires=Sun, 1-Jan-2012 00:00:00 GMT',
+     'name2=value2; Domain=www.example.com; Path=/path; Expires=Sun, 1-Jan-2012 00:00:00 GMT']
+  end
+  
+  def response
+    m = double("HttpResponse")
+    m.stub(:[]) do |key|
+      case key
+      when 'set-cookie'
+        cookies.join(', ')
+      end
+    end
+    m.stub(:get_fields) do |key|
+      case key
+      when 'set-cookie'
+        cookies
+      end
+    end
+    m.stub(:body) { html }
+    m
+  end
+  
+  def connection
+    Horseman::Connection.any_instance.stub(:exec_request) { response }
+    Horseman::Connection.new
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1 @@
+require 'mocks'

metadata

@@ -0,0 +1,89 @@
+--- !ruby/object:Gem::Specification
+name: horseman
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Jarrod Peace
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-01-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: &70095638639800 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.5.0
+  type: :runtime
+  prerelease: false
+  version_requirements: *70095638639800
+description: Headless HTTP crawler/scraper for ASP.NET WebForms applications
+email: peace.jarrod@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files:
+- README.rdoc
+- lib/horseman.rb
+- lib/horseman/browser.rb
+- lib/horseman/connection.rb
+- lib/horseman/cookies.rb
+- lib/horseman/hidden_fields.rb
+- lib/horseman/response.rb
+- lib/horseman/version.rb
+files:
+- Gemfile
+- Gemfile.lock
+- Manifest
+- README.rdoc
+- Rakefile
+- horseman.gemspec
+- lib/horseman.rb
+- lib/horseman/browser.rb
+- lib/horseman/connection.rb
+- lib/horseman/cookies.rb
+- lib/horseman/hidden_fields.rb
+- lib/horseman/response.rb
+- lib/horseman/version.rb
+- spec/horseman/browser_spec.rb
+- spec/horseman/connection_spec.rb
+- spec/horseman/cookies_spec.rb
+- spec/horseman/hidden_fields_spec.rb
+- spec/horseman/response_spec.rb
+- spec/mocks.rb
+- spec/spec_helper.rb
+homepage: http://jarrodpeace.com
+licenses: []
+post_install_message: 
+rdoc_options:
+- --line-numbers
+- --inline-source
+- --title
+- Horseman
+- --main
+- README.rdoc
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '1.2'
+requirements: []
+rubyforge_project: horseman
+rubygems_version: 1.8.10
+signing_key: 
+specification_version: 3
+summary: Headless HTTP crawler/scraper for ASP.NET WebForms applications
+test_files: []