ronin-web-spider 0.1.0.beta1

data/spec/agent_spec.rb

@@ -0,0 +1,585 @@
+require 'spec_helper'
+require 'ronin/web/spider/agent'
+
+require 'webmock/rspec'
+require 'sinatra/base'
+
+describe Ronin::Web::Spider::Agent do
+  describe "#initialize" do
+    context "when Ronin::Support::Network::HTTP.proxy is set" do
+      let(:proxy_host) { 'example.com' }
+      let(:proxy_port) { 8080 }
+      let(:proxy_uri)  { URI::HTTP.build(host: proxy_host, port: proxy_port) }
+
+      before { Ronin::Support::Network::HTTP.proxy = proxy_uri }
+
+      it "must parse ENV['RONIN_HTTP_USER_AGENT'] and set #proxy" do
+        expect(subject.proxy).to be_kind_of(Spidr::Proxy)
+        expect(subject.proxy.host).to eq(proxy_host)
+        expect(subject.proxy.port).to eq(proxy_port)
+      end
+
+      after { Ronin::Support::Network::HTTP.proxy = nil }
+    end
+
+    context "when Ronin::Support::Network::HTTP.user_agent is set" do
+      let(:user_agent) { 'Foo Bar' }
+
+      before { Ronin::Support::Network::HTTP.user_agent = user_agent }
+
+      it "must default #user_agent to ENV['RONIN_HTTP_USER_AGENT']" do
+        expect(subject.user_agent).to eq(user_agent)
+      end
+
+      after { Ronin::Support::Network::HTTP.user_agent = nil }
+    end
+
+    context "when given the proxy: keyword argument" do
+      let(:proxy_host) { 'example.com' }
+      let(:proxy_port) { 8080 }
+
+      context "and it's an Addressable::URI" do
+        let(:proxy) { Addressable::URI.new(host: proxy_host, port: proxy_port) }
+
+        subject { described_class.new(proxy: proxy) }
+
+        it "must convert it to a Spidr::Proxy object" do
+          expect(subject.proxy).to be_kind_of(Spidr::Proxy)
+          expect(subject.proxy.host).to eq(proxy_host)
+          expect(subject.proxy.port).to eq(proxy_port)
+        end
+      end
+
+      context "and it's an URI::HTTP" do
+        let(:proxy) { URI::HTTP.build(host: proxy_host, port: proxy_port) }
+
+        subject { described_class.new(proxy: proxy) }
+
+        it "must convert it to a Spidr::Proxy object" do
+          expect(subject.proxy).to be_kind_of(Spidr::Proxy)
+          expect(subject.proxy.host).to eq(proxy_host)
+          expect(subject.proxy.port).to eq(proxy_port)
+        end
+      end
+
+      context "and it's a Hash" do
+        let(:proxy) do
+          {host: proxy_host, port: proxy_port}
+        end
+
+        subject { described_class.new(proxy: proxy) }
+
+        it "must convert it to a Spidr::Proxy object" do
+          expect(subject.proxy).to be_kind_of(Spidr::Proxy)
+          expect(subject.proxy.host).to eq(proxy_host)
+          expect(subject.proxy.port).to eq(proxy_port)
+        end
+      end
+
+      context "and it's a String" do
+        let(:proxy) { "http://#{proxy_host}:#{proxy_port}" }
+
+        subject { described_class.new(proxy: proxy) }
+
+        it "must convert it to a Spidr::Proxy object" do
+          expect(subject.proxy).to be_kind_of(Spidr::Proxy)
+          expect(subject.proxy.host).to eq(proxy_host)
+          expect(subject.proxy.port).to eq(proxy_port)
+        end
+      end
+    end
+
+    context "when given the user_agent: keyword argument" do
+      context "and it's a String" do
+        let(:user_agent) { "test user-agent" }
+
+        subject { described_class.new(user_agent: user_agent) }
+
+        it "must set the #user_agent" do
+          expect(subject.user_agent).to eq(user_agent)
+        end
+      end
+
+      context "and it's a Symbol" do
+        let(:user_agent) { :chrome_linux }
+        let(:expected_user_agent) do
+          Ronin::Support::Network::HTTP::UserAgents[user_agent]
+        end
+
+        subject { described_class.new(user_agent: user_agent) }
+
+        it "must map the Symbol to one of Ronin::Support::Network::HTTP::UserAgents" do
+          expect(subject.user_agent).to eq(expected_user_agent)
+        end
+      end
+    end
+
+    it "must default #visited_hosts to nil" do
+      expect(subject.visited_hosts).to be(nil)
+    end
+  end
+
+  describe "#every_host" do
+    module TestAgentEveryHost
+      class Host1 < Sinatra::Base
+
+        set :host, 'host1.example.com'
+        set :port, 80
+
+        get '/' do
+          <<~HTML
+          <html>
+            <body>
+              <a href="/link1">link1</a>
+              <a href="http://host2.example.com/offsite-link">offsite link</a>
+              <a href="/link2">link2</a>
+            </body>
+          </html>
+          HTML
+        end
+
+        get '/link1' do
+          '<html><body>got here</body></html>'
+        end
+
+        get '/link2' do
+          '<html><body>got here</body></html>'
+        end
+      end
+
+      class Host2 < Sinatra::Base
+
+        set :host, 'host2.example.com'
+        set :port, 80
+
+        get '/offsite-link' do
+          '<html><body>should not get here</body></html>'
+        end
+
+      end
+    end
+
+    let(:host1) { 'host1.example.com' }
+    let(:host2) { 'host2.example.com'   }
+
+    let(:host1_app) { TestAgentEveryHost::Host1 }
+    let(:host2_app) { TestAgentEveryHost::Host2 }
+
+    before do
+      stub_request(:any, /#{Regexp.escape(host1)}/).to_rack(host1_app)
+      stub_request(:any, /#{Regexp.escape(host2)}/).to_rack(host2_app)
+    end
+
+    it "must yield every newly discovered hostname while spidering" do
+      yielded_hosts = []
+
+      subject.every_host do |host|
+        yielded_hosts << host
+      end
+
+      subject.start_at("http://#{host1}/")
+
+      expect(yielded_hosts).to eq([host1, host2])
+    end
+
+    it "must popualte #visited_hosts" do
+      subject.every_host { |host| }
+      subject.start_at("http://#{host1}/")
+
+      expect(subject.visited_hosts).to be_kind_of(Set)
+      expect(subject.visited_hosts.entries).to eq([host1, host2])
+    end
+  end
+
+  # TODO: need to figure out how to test #every_cert using webmock.
+  describe "#every_cert"
+  
+  describe "#every_favicon" do
+    module TestAgentEveryHost
+      class TestApp < Sinatra::Base
+
+        set :host, 'example.com'
+        set :port, 80
+
+        get '/' do
+          <<~HTML
+          <html>
+            <head>
+              <link rel="favicon" href="/favicon1.ico" type="image/x-icon"/>
+            </head>
+            <body>
+              <a href="/link1">link1</a>
+              <a href="http://host2.example.com/offsite-link">offsite link</a>
+              <a href="/link2">link2</a>
+            </body>
+          </html>
+          HTML
+        end
+
+        get '/favicon1.ico' do
+          content_type 'image/x-icon'
+
+          "favicon1"
+        end
+
+        get '/favicon2.ico' do
+          content_type 'image/vnd.microsoft.icon'
+
+          "favicon2"
+        end
+
+        get '/link1' do
+          '<html><body>got here</body></html>'
+        end
+
+        get '/link2' do
+          <<~HTML
+          <html>
+            <head>
+              <link rel="favicon" href="/favicon2.ico" type="image/x-icon"/>
+            </head>
+            <body>got here</body>
+          </html>
+          HTML
+        end
+      end
+    end
+
+    let(:host) { 'example.com' }
+
+    let(:test_app) { TestAgentEveryHost::TestApp }
+
+    before do
+      stub_request(:any, /#{Regexp.escape(host)}/).to_rack(test_app)
+    end
+
+    it "must yield Spidr::Page objects for each encountered .ico file" do
+      yielded_favicons = []
+
+      subject.every_favicon do |favicon|
+        yielded_favicons << favicon
+      end
+
+      subject.start_at("http://#{host}/")
+
+      expect(yielded_favicons).to_not be_empty
+
+      expect(yielded_favicons[0]).to be_kind_of(Spidr::Page)
+      expect(yielded_favicons[0].content_type).to eq('image/x-icon')
+      expect(yielded_favicons[0].url).to eq(URI("http://#{host}/favicon1.ico"))
+
+      expect(yielded_favicons[1]).to be_kind_of(Spidr::Page)
+      expect(yielded_favicons[1].content_type).to eq('image/vnd.microsoft.icon')
+      expect(yielded_favicons[1].url).to eq(URI("http://#{host}/favicon2.ico"))
+    end
+  end
+
+  describe "#every_html_comment" do
+    module TestAgentEveryHTMLComment
+      class TestApp < Sinatra::Base
+
+        set :host, 'example.com'
+        set :port, 80
+
+        get '/' do
+          <<~HTML
+          <html>
+            <head>
+              <!-- comment 1 -->
+            </head>
+            <!-- -->
+            <body>
+              <!-- comment 2 -->
+            </body>
+          </html>
+          HTML
+        end
+      end
+    end
+
+    let(:host) { 'example.com' }
+
+    let(:test_app) { TestAgentEveryHTMLComment::TestApp }
+
+    before do
+      stub_request(:any, /#{Regexp.escape(host)}/).to_rack(test_app)
+    end
+
+    it "must yield every non-empty/non-whitespace HTML comment String" do
+      yielded_comments = []
+
+      subject.every_html_comment do |comment|
+        yielded_comments << comment
+      end
+
+      subject.start_at("http://#{host}/")
+
+      expect(yielded_comments).to match_array(
+        [
+          'comment 1',
+          'comment 2'
+        ]
+      )
+    end
+  end
+
+  describe "#every_javascript" do
+    module TestAgentEveryJavaScript
+      class TestApp < Sinatra::Base
+
+        set :host, 'example.com'
+        set :port, 80
+
+        get '/' do
+          <<~HTML
+          <html>
+            <head>
+              <script type="text/javascript" src="/javascript1.js"></script>
+              <script type="text/javascript">javascript2</script>
+            </head>
+            <body>
+              <a href="/link1">link1</a>
+              <a href="http://host2.example.com/offsite-link">offsite link</a>
+              <a href="/link2">link2</a>
+            </body>
+          </html>
+          HTML
+        end
+
+        get '/javascript1.js' do
+          content_type 'text/javascript'
+          "javascript1"
+        end
+      end
+    end
+
+    let(:host) { 'example.com' }
+
+    let(:test_app) { TestAgentEveryJavaScript::TestApp }
+
+    before do
+      stub_request(:any, /#{Regexp.escape(host)}/).to_rack(test_app)
+    end
+
+    it "must yield both the contents of .js files and inline <script> tags" do
+      yielded_javascripts = []
+
+      subject.every_javascript do |js|
+        yielded_javascripts << js
+      end
+
+      subject.start_at("http://#{host}/")
+
+      expect(yielded_javascripts).to match_array(%w[javascript1 javascript2])
+    end
+  end
+
+  describe "#every_javascript_string" do
+    module TestAgentEveryJavaScriptString
+      class TestApp < Sinatra::Base
+
+        set :host, 'example.com'
+        set :port, 80
+
+        get '/' do
+          <<~HTML
+          <html>
+            <head>
+              <script type="text/javascript" src="/javascript1.js"></script>
+              <script type="text/javascript">
+              var str3 = "string #3";
+              var str4 = 'string #4';
+              </script>
+            </head>
+            <body>
+              <a href="/link1">link1</a>
+              <a href="http://host2.example.com/offsite-link">offsite link</a>
+              <a href="/link2">link2</a>
+            </body>
+          </html>
+          HTML
+        end
+
+        get '/javascript1.js' do
+          content_type 'text/javascript'
+          <<~JS
+          var str1 = "string #1";
+          var str2 = 'string #2';
+          JS
+        end
+      end
+    end
+
+    let(:host) { 'example.com' }
+
+    let(:test_app) { TestAgentEveryJavaScriptString::TestApp }
+
+    before do
+      stub_request(:any, /#{Regexp.escape(host)}/).to_rack(test_app)
+    end
+
+    it "must yield every JavaScript string from any <script> tag" do
+      yielded_javascript_strings = []
+
+      subject.every_javascript_string do |string|
+        yielded_javascript_strings << string
+      end
+
+      subject.start_at("http://#{host}/")
+
+      expect(yielded_javascript_strings).to match_array(
+        [
+          'string #1',
+          'string #2',
+          'string #3',
+          'string #4'
+        ]
+      )
+    end
+  end
+
+  describe "#every_javascript_comment" do
+    module TestAgentEveryJavaScriptComment
+      class TestApp < Sinatra::Base
+
+        set :host, 'example.com'
+        set :port, 80
+
+        get '/' do
+          <<~HTML
+          <html>
+            <head>
+              <script type="text/javascript" src="/javascript1.js"></script>
+              <script type="text/javascript">
+              // comment 3
+              var str3 = "string #3";
+              /*
+                 comment 4
+               */
+              var str4 = 'string #4';
+              </script>
+            </head>
+            <body>
+              <a href="/link1">link1</a>
+              <a href="http://host2.example.com/offsite-link">offsite link</a>
+              <a href="/link2">link2</a>
+            </body>
+          </html>
+          HTML
+        end
+
+        get '/javascript1.js' do
+          content_type 'text/javascript'
+          <<~JS
+          // comment 1
+          var str1 = "string #1";
+          /* comment 2 */
+          var str2 = 'string #2';
+          JS
+        end
+      end
+    end
+
+    let(:host) { 'example.com' }
+
+    let(:test_app) { TestAgentEveryJavaScriptComment::TestApp }
+
+    before do
+      stub_request(:any, /#{Regexp.escape(host)}/).to_rack(test_app)
+    end
+
+    it "must yield every JavaScript comment from any <script> tag" do
+      yielded_javascript_comments = []
+
+      subject.every_javascript_comment do |comment|
+        yielded_javascript_comments << comment
+      end
+
+      subject.start_at("http://#{host}/")
+
+      expect(yielded_javascript_comments).to match_array(
+        [
+          "// comment 1\n",
+          "/* comment 2 */",
+          "// comment 3\n",
+          "/*\n       comment 4\n     */"
+        ]
+      )
+    end
+  end
+
+  describe "#every_comment" do
+    module TestAgentEveryComment
+      class TestApp < Sinatra::Base
+
+        set :host, 'example.com'
+        set :port, 80
+
+        get '/' do
+          <<~HTML
+          <html>
+            <head>
+              <!-- HTML comment 1 -->
+              <script type="text/javascript" src="/javascript1.js"></script>
+              <script type="text/javascript">
+              // JavaScript comment 3
+              var str3 = "string #3";
+              /*
+                 JavaScript comment 4
+               */
+              var str4 = 'string #4';
+              </script>
+            </head>
+            <!-- -->
+            <body>
+              <!-- HTML comment 2 -->
+              <a href="/link1">link1</a>
+              <a href="http://host2.example.com/offsite-link">offsite link</a>
+              <a href="/link2">link2</a>
+            </body>
+          </html>
+          HTML
+        end
+
+        get '/javascript1.js' do
+          content_type 'text/javascript'
+          <<~JS
+          // JavaScript comment 1
+          var str1 = "string #1";
+          /* JavaScript comment 2 */
+          var str2 = 'string #2';
+          JS
+        end
+      end
+    end
+
+    let(:host) { 'example.com' }
+
+    let(:test_app) { TestAgentEveryComment::TestApp }
+
+    before do
+      stub_request(:any, /#{Regexp.escape(host)}/).to_rack(test_app)
+    end
+
+    it "must yield every HTML and JavaScript comment from any <script> tag" do
+      yielded_comments = []
+
+      subject.every_comment do |comment|
+        yielded_comments << comment
+      end
+
+      subject.start_at("http://#{host}/")
+
+      expect(yielded_comments).to match_array(
+        [
+          "HTML comment 1",
+          "// JavaScript comment 1\n",
+          "/* JavaScript comment 2 */",
+          "// JavaScript comment 3\n",
+          "/*\n       JavaScript comment 4\n     */",
+          "HTML comment 2"
+        ]
+      )
+    end
+  end
+end

data/spec/archive_spec.rb

@@ -0,0 +1,91 @@
+require 'spec_helper'
+require 'ronin/web/spider/archive'
+
+require 'tmpdir'
+
+describe Ronin::Web::Spider::Archive do
+  let(:root) { File.join(Dir.mktmpdir('ronin-web-spider')) }
+
+  subject { described_class.new(root) }
+
+  describe "#initialize" do
+    it "must set #root" do
+      expect(subject.root).to eq(root)
+    end
+  end
+
+  describe ".open" do
+    subject { described_class.open(root) }
+
+    it "must return a new #{described_class}" do
+      expect(subject).to be_kind_of(described_class)
+    end
+
+    context "when given a block" do
+      it "must yield the new #{described_class}" do
+        expect { |b|
+          described_class.open(root,&b)
+        }.to yield_with_args(described_class)
+      end
+    end
+
+    context "when the root directory does not exist" do
+      let(:root) { File.join(super(),'does-not-exist-yet') }
+
+      it "must create the given root directory" do
+        described_class.open(root)
+
+        expect(File.directory?(root)).to be(true)
+      end
+    end
+
+    context "when the root directory does exist" do
+      let(:root) { File.join(super(),'does-not-exist-yet') }
+
+      before { FileUtils.mkdir(root) }
+
+      it "must not raise an error" do
+        expect {
+          described_class.open(root)
+        }.to_not raise_error
+      end
+    end
+  end
+
+  describe "#write" do
+    let(:url)  { URI('https://example.com/foo/bar.html') }
+    let(:body) { 'test file' }
+
+    before { subject.write(url,body) }
+
+    it "must automatically create parent directory" do
+      expect(File.directory?(File.join(root,'foo'))).to be(true)
+    end
+
+    it "must write the body into the file" do
+      expect(File.read(File.join(root,'foo','bar.html'))).to eq(body)
+    end
+
+    context "when the URL has a query string" do
+      let(:url) { URI('https://example.com/foo/bar.php?q=1') }
+
+      it "must include the query string as part of the file name" do
+        expect(File.read(File.join(root,'foo','bar.php?q=1'))).to eq(body)
+      end
+    end
+
+    context "when the URL path ends with a '/'" do
+      let(:url) { URI('https://example.com/foo/bar/') }
+
+      it "must write the body to an index.html file within the URL's path" do
+        expect(File.read(File.join(root,'foo','bar','index.html'))).to eq(body)
+      end
+    end
+  end
+
+  describe "#to_s" do
+    it "must return the root directory" do
+      expect(subject.to_s).to eq(root)
+    end
+  end
+end

data/spec/example_app.rb

@@ -0,0 +1,27 @@
+require 'rspec'
+require 'sinatra/base'
+require 'webmock/rspec'
+
+require 'ronin/web/spider/agent'
+
+RSpec.shared_context "example App" do
+  let(:host) { 'example.com' }
+
+  subject { Ronin::Web::Spider::Agent.new(host: host) }
+
+  def self.app(&block)
+    let(:app) do
+      klass = Class.new(Sinatra::Base)
+      klass.set :host, host
+      klass.set :port, 80
+      klass.class_eval(&block)
+      return klass
+    end
+
+    before do
+      stub_request(:any, /#{Regexp.escape(host)}/).to_rack(app)
+
+      subject.start_at("http://#{host}/")
+    end
+  end
+end