spidr 0.6.1 → 0.7.0

data/lib/spidr/session_cache.rb

@@ -17,37 +17,39 @@ module Spidr
     #
     # Creates a new session cache.
     #
-    # @param [Hash] options
-    #   Configuration options.
-    #
-    # @option [Hash] :proxy (Spidr.proxy)
+    # @param [Hash] proxy
     #   Proxy options.
     #
-    # @option [Integer] :open_timeout (Spidr.open_timeout)
-    #   Optional open timeout.
+    # @param [Integer] open_timeout
+    #   Optional open connection timeout.
     #
-    # @option [Integer] :ssl_timeout (Spidr.ssl_timeout)
-    #   Optional ssl timeout.
+    # @param [Integer] ssl_timeout
+    #   Optional SSL connection timeout.
     #
-    # @option [Integer] :read_timeout (Spidr.read_timeout)
+    # @param [Integer] read_timeout
     #   Optional read timeout.
     #
-    # @option [Integer] :continue_timeout (Spidr.continue_timeout)
+    # @param [Integer] continue_timeout
     #   Optional `Continue` timeout.
     #
-    # @option [Integer] :keep_alive_timeout (Spidr.keep_alive_timeout)
+    # @param [Integer] keep_alive_timeout
     #   Optional `Keep-Alive` timeout.
     #
     # @since 0.6.0
     #
-    def initialize(options={})
-      @proxy = options.fetch(:proxy,Spidr.proxy)
-
-      @open_timeout       = options.fetch(:open_timeout,Spidr.open_timeout)
-      @ssl_timeout        = options.fetch(:ssl_timeout,Spidr.ssl_timeout)
-      @read_timeout       = options.fetch(:read_timeout,Spidr.read_timeout)
-      @continue_timeout   = options.fetch(:continue_timeout,Spidr.continue_timeout)
-      @keep_alive_timeout = options.fetch(:keep_alive_timeout,Spidr.keep_alive_timeout)
+    def initialize(proxy:              Spidr.proxy,
+                   open_timeout:       Spidr.open_timeout,
+                   ssl_timeout:        Spidr.ssl_timeout,
+                   read_timeout:       Spidr.read_timeout,
+                   continue_timeout:   Spidr.continue_timeout,
+                   keep_alive_timeout: Spidr.keep_alive_timeout)
+      self.proxy = proxy
+
+      self.open_timeout       = open_timeout
+      self.ssl_timeout        = ssl_timeout
+      self.read_timeout       = read_timeout
+      self.continue_timeout   = continue_timeout
+      self.keep_alive_timeout = keep_alive_timeout
 
       @sessions = {}
     end

data/lib/spidr/settings/proxy.rb

@@ -1,5 +1,7 @@
 require 'spidr/proxy'
 
+require 'uri/http'
+
 module Spidr
   module Settings
     #
@@ -21,7 +23,7 @@ module Spidr
       #
       # Sets the proxy information used by Agent objects.
       #
-      # @param [Spidr::Proxy, Hash, nil] new_proxy
+      # @param [Spidr::Proxy, Hash, URI::HTTP, String, nil] new_proxy
       #   The new proxy information.
       #
       # @option new_proxy [String] :host
@@ -41,11 +43,23 @@ module Spidr
       #
       def proxy=(new_proxy)
         @proxy = case new_proxy
-                 when Spidr::Proxy then new_proxy
-                 when Hash         then Spidr::Proxy.new(new_proxy)
-                 when nil          then Spidr::Proxy.new
+                 when Spidr::Proxy
+                   new_proxy
+                 when Hash
+                   Spidr::Proxy.new(**new_proxy)
+                 when String, URI::HTTP
+                   proxy_uri = URI(new_proxy)
+
+                   Spidr::Proxy.new(
+                      host:     proxy_uri.host,
+                      port:     proxy_uri.port,
+                      user:     proxy_uri.user,
+                      password: proxy_uri.password
+                   )
+                 when nil
+                   Spidr::Proxy.new
                  else
-                   raise(TypeError,"#{self.class}#{__method__} only accepts Proxy, Hash or nil")
+                   raise(TypeError,"#{self.class}#{__method__} only accepts Spidr::Proxy, URI::HTTP, Hash, or nil")
                  end
       end
 

data/lib/spidr/spidr.rb

@@ -36,22 +36,31 @@ module Spidr
   #
   # @see Agent.start_at
   #
-  def self.start_at(url,options={},&block)
-    Agent.start_at(url,options,&block)
+  def self.start_at(url,**kwargs,&block)
+    Agent.start_at(url,**kwargs,&block)
   end
 
   #
   # @see Agent.host
   #
-  def self.host(name,options={},&block)
-    Agent.host(name,options,&block)
+  def self.host(name,**kwargs,&block)
+    Agent.host(name,**kwargs,&block)
+  end
+
+  #
+  # @see Agent.domain
+  #
+  # @since 0.7.0
+  #
+  def self.domain(name,options={},&block)
+    Agent.domain(name,options,&block)
   end
 
   #
   # @see Agent.site
   #
-  def self.site(url,options={},&block)
-    Agent.site(url,options,&block)
+  def self.site(url,**kwargs,&block)
+    Agent.site(url,**kwargs,&block)
   end
 
   # 

data/lib/spidr/version.rb

@@ -1,4 +1,4 @@
 module Spidr
   # Spidr version
-  VERSION = '0.6.1'
+  VERSION = '0.7.0'
 end

data/spec/agent_spec.rb

@@ -7,6 +7,250 @@ require 'spidr/agent'
 describe Agent do
   it_should_behave_like "includes Spidr::Settings::UserAgent"
 
+  describe ".start_at" do
+    module TestAgentStartAt
+      class ExampleApp < Sinatra::Base
+
+        set :host, 'example.com'
+        set :port, 80
+
+        get '/' do
+          '<html><body>should not get here</body></html>'
+        end
+
+        get '/entry-point' do
+          <<~HTML
+          <html>
+            <body>
+              <a href="/link1">link1</a>
+              <a href="http://other.com/offsite-link">offsite link</a>
+              <a href="/link2">link2</a>
+            </body>
+          </html>
+          HTML
+        end
+
+        get '/link1' do
+          '<html><body>got here</body></html>'
+        end
+
+        get '/link2' do
+          '<html><body>got here</body></html>'
+        end
+      end
+
+      class OtherApp < Sinatra::Base
+
+        set :host, 'other.com'
+        set :port, 80
+
+        get '/offsite-link' do
+          '<html><body>should not get here</body></html>'
+        end
+
+      end
+    end
+
+    subject { described_class }
+
+    let(:host)       { 'example.com' }
+    let(:other_host) { 'other.com'   }
+    let(:url)        { URI("http://#{host}/entry-point") }
+
+    let(:app)       { TestAgentStartAt::ExampleApp }
+    let(:other_app) { TestAgentStartAt::OtherApp   }
+
+    before do
+      stub_request(:any, /#{Regexp.escape(host)}/).to_rack(app)
+      stub_request(:any, /#{Regexp.escape(other_host)}/).to_rack(other_app)
+    end
+
+    it "must spider the website starting at the given URL" do
+      agent = subject.start_at(url)
+
+      expect(agent.history).to be == Set[
+        URI("http://#{host}/entry-point"),
+        URI("http://#{host}/link1"),
+        URI("http://#{other_host}/offsite-link"),
+        URI("http://#{host}/link2")
+      ]
+    end
+  end
+
+  describe ".site" do
+    module TestAgentSite
+      class ExampleApp < Sinatra::Base
+
+        set :host, 'example.com'
+        set :port, 80
+
+        get '/' do
+          '<html><body>should not get here</body></html>'
+        end
+
+        get '/entry-point' do
+          <<~HTML
+            <html>
+              <body>
+                <a href="/link1">link1</a>
+                <a href="http://other.com/offsite-link">offsite link</a>
+                <a href="/link2">link2</a>
+              </body>
+            </html>
+          HTML
+        end
+
+        get '/link1' do
+          '<html><body>got here</body></html>'
+        end
+
+        get '/link2' do
+          '<html><body>got here</body></html>'
+        end
+
+      end
+    end
+
+    subject { described_class }
+
+    let(:host) { 'example.com' }
+    let(:url)  { URI("http://#{host}/entry-point") }
+
+    let(:app) { TestAgentSite::ExampleApp }
+
+    before do
+      stub_request(:any, /#{Regexp.escape(host)}/).to_rack(app)
+    end
+
+    it "must spider the website starting at the given URL" do
+      agent = subject.site(url)
+
+      expect(agent.history).to be == Set[
+        URI("http://#{host}/entry-point"),
+        URI("http://#{host}/link1"),
+        URI("http://#{host}/link2")
+      ]
+    end
+  end
+
+  describe ".host" do
+    module TestAgentHost
+      class ExampleApp < Sinatra::Base
+
+        set :host, 'example.com'
+        set :port, 80
+
+        get '/' do
+          <<~HTML
+            <html>
+              <body>
+                <a href="/link1">link1</a>
+                <a href="http://other.com/offsite-link">offsite link</a>
+                <a href="/link2">link2</a>
+              </body>
+            </html>
+          HTML
+        end
+
+        get '/link1' do
+          '<html><body>got here</body></html>'
+        end
+
+        get '/link2' do
+          '<html><body>got here</body></html>'
+        end
+
+      end
+    end
+
+    subject { described_class }
+
+    let(:host) { 'example.com' }
+    let(:app)  { TestAgentHost::ExampleApp }
+
+    before do
+      stub_request(:any, /#{Regexp.escape(host)}/).to_rack(app)
+    end
+
+    it "must spider the website starting at the given URL" do
+      agent = subject.host(host)
+
+      # XXX: for some reason Set#== was returning false, so convert to an Array
+      expect(agent.history.to_a).to be == [
+        URI("http://#{host}/"),
+        URI("http://#{host}/link1"),
+        URI("http://#{host}/link2")
+      ]
+    end
+  end
+
+  describe ".domain" do
+    module TestAgentDomain
+      class ExampleApp < Sinatra::Base
+
+        set :host, 'example.com'
+        set :port, 80
+
+        get '/' do
+          <<~HTML
+            <html>
+              <body>
+                <a href="/link1">link1</a>
+                <a href="http://sub.example.com/subdomain-link">subdomain link</a>
+                <a href="/link2">link2</a>
+              </body>
+            </html>
+          HTML
+        end
+
+        get '/link1' do
+          '<html><body>got here</body></html>'
+        end
+
+        get '/link2' do
+          '<html><body>got here</body></html>'
+        end
+
+      end
+
+      class SubDomainApp < Sinatra::Base
+
+        set :host, 'sub.example.com'
+        set :port, 80
+
+        get '/subdomain-link' do
+          '<html><body>should get here</body></html>'
+        end
+
+      end
+    end
+
+    subject { described_class }
+
+    let(:domain)        { 'example.com' }
+    let(:domain_app)    { TestAgentDomain::ExampleApp }
+
+    let(:subdomain)     { 'sub.example.com' }
+    let(:subdomain_app) { TestAgentDomain::SubDomainApp }
+
+    before do
+      stub_request(:any, /#{Regexp.escape(subdomain)}/).to_rack(subdomain_app)
+      stub_request(:any, /#{Regexp.escape(domain)}/).to_rack(domain_app)
+    end
+
+    it "must spider the domain and subdomains starting at the given domain" do
+      agent = subject.domain(domain)
+
+      # XXX: for some reason Set#== was returning false, so convert to an Array
+      expect(agent.history.to_a).to be == [
+        URI("http://#{domain}/"),
+        URI("http://#{domain}/link1"),
+        URI("http://#{subdomain}/subdomain-link"),
+        URI("http://#{domain}/link2")
+      ]
+    end
+  end
+
   describe "#initialize" do
     it "should not be running" do
       expect(subject).to_not be_running
@@ -32,6 +276,68 @@ describe Agent do
       expect(subject.sessions).to be_kind_of(SessionCache)
     end
 
+    context "when the proxy: keyword argument is given" do
+      let(:proxy) do
+        Spidr::Proxy.new(host: 'example.com')
+      end
+
+      subject { described_class.new(proxy: proxy) }
+
+      it "must initialize the #proxy of #session_cache" do
+        expect(subject.sessions.proxy).to be(proxy)
+      end
+    end
+
+    context "when the open_timeout: keyword argument is given" do
+      let(:open_timeout) { 5 }
+
+      subject { described_class.new(open_timeout: open_timeout) }
+
+      it "must initialize the #open_timeout of #session_cache" do
+        expect(subject.sessions.open_timeout).to eq(open_timeout)
+      end
+    end
+
+    context "when the ssl_timeout: keyword argument is given" do
+      let(:ssl_timeout) { 5 }
+
+      subject { described_class.new(ssl_timeout: ssl_timeout) }
+
+      it "must initialize the #ssl_timeout of #session_cache" do
+        expect(subject.sessions.ssl_timeout).to eq(ssl_timeout)
+      end
+    end
+
+    context "when the read_timeout: keyword argument is given" do
+      let(:read_timeout) { 5 }
+
+      subject { described_class.new(read_timeout: read_timeout) }
+
+      it "must initialize the #read_timeout of #session_cache" do
+        expect(subject.sessions.read_timeout).to eq(read_timeout)
+      end
+    end
+
+    context "when the continue_timeout: keyword argument is given" do
+      let(:continue_timeout) { 5 }
+
+      subject { described_class.new(continue_timeout: continue_timeout) }
+
+      it "must initialize the #continue_timeout of #session_cache" do
+        expect(subject.sessions.continue_timeout).to eq(continue_timeout)
+      end
+    end
+
+    context "when the keep_alive_timeout: keyword argument is given" do
+      let(:keep_alive_timeout) { 5 }
+
+      subject { described_class.new(keep_alive_timeout: keep_alive_timeout) }
+
+      it "must initialize the #keep_alive_timeout of #session_cache" do
+        expect(subject.sessions.keep_alive_timeout).to eq(keep_alive_timeout)
+      end
+    end
+
     it "should initialize the #cookie_jar" do
       expect(subject.cookies).to be_kind_of(CookieJar)
     end
@@ -386,7 +692,15 @@ describe Agent do
     context "frames" do
       app do
         get '/' do
-          %{<html><body><frameset><frame src="/frame" /></frameset></body></html>}
+          <<~HTML
+            <html>
+              <body>
+                <frameset>
+                  <frame src="/frame" />
+                </frameset>
+              </body>
+            </html>
+          HTML
         end
 
         get '/frame' do
@@ -614,7 +928,14 @@ describe Agent do
           end
 
           get '/redirect' do
-            %{<html><head><meta http-equiv="refresh" content="0; url=http://#{settings.host}/link" /></head><body>Redirecting...</body></html>}
+            <<~HTML
+              <html>
+                <head>
+                  <meta http-equiv="refresh" content="0; url=http://#{settings.host}/link" />
+                </head>
+                <body>Redirecting...</body>
+              </html>
+            HTML
           end
 
           get '/link' do
@@ -674,7 +995,14 @@ describe Agent do
 
     app do
       get '/' do
-        %{<html><body><a href="http://google.com/">external link</a> <a href="/link">local link</a></body></html>}
+        <<~HTML
+          <html>
+            <body>
+              <a href="http://google.com/">external link</a>
+              <a href="/link">local link</a>
+            </body>
+          </html>
+        HTML
       end
 
       get '/link' do
@@ -726,13 +1054,27 @@ describe Agent do
 
     app do
       get '/' do
-        %{<html><body><a href="/left?d=1">left</a><a href="/right?d=1">right</a></body></html>}
+        <<~HTML
+          <html>
+            <body>
+              <a href="/left?d=1">left</a>
+              <a href="/right?d=1">right</a>
+            </body>
+          </html>
+        HTML
       end
 
-      get %r{^/left|/right} do
+      get %r{/left|/right} do
         d = Integer(params['d'])
 
-        %{<html><body><a href="/left?d=#{d+1}">left</a><a href="/right?d=#{d+1}">right</a></body></html>}
+        <<~HTML
+          <html>
+            <body>
+              <a href="/left?d=#{d+1}">left</a>
+              <a href="/right?d=#{d+1}">right</a>
+            </body>
+          </html>
+        HTML
       end
     end
 
@@ -774,7 +1116,14 @@ describe Agent do
 
     app do
       get '/' do
-        %{<html><body><a href="/secret">don't follow this link</a> <a href="/pub">follow this link</a></body></html>}
+        <<~HTML
+          <html>
+            <body>
+              <a href="/secret">don't follow this link</a>
+              <a href="/pub">follow this link</a>
+            </body>
+          </html>
+        HTML
       end
 
       get '/pub' do

data/spec/example_page.rb

@@ -1,4 +1,6 @@
 require 'rspec'
+require 'net/http'
+require 'uri'
 
 RSpec.shared_context "example Page" do
   let(:code)          { 200 }

data/spec/page/content_types_spec.rb

@@ -147,4 +147,26 @@ describe Page do
   describe "#zip?" do
     include_examples "Content-Type method", :zip?, 'application/zip'
   end
+
+  describe "#png?" do
+    include_examples "Content-Type method", :png?, 'image/png'
+  end
+
+  describe "#gif?" do
+    include_examples "Content-Type method", :gif?, 'image/gif'
+  end
+
+  describe "#jpeg?" do
+    include_examples "Content-Type method", :jpeg?, 'image/jpeg'
+  end
+
+  describe "#ico?" do
+    context "when 'Content-Type' is 'image/x-icon'" do
+      include_examples "Content-Type method", :ico?, 'image/x-icon'
+    end
+
+    context "when 'Content-Type' is 'image/vnd.microsoft.icon'" do
+      include_examples "Content-Type method", :ico?, 'image/vnd.microsoft.icon'
+    end
+  end
 end