ronin-web-spider 0.1.0.beta2 → 0.1.0

data/spec/agent_spec.rb

@@ -1,585 +0,0 @@
-require 'spec_helper'
-require 'ronin/web/spider/agent'
-
-require 'webmock/rspec'
-require 'sinatra/base'
-
-describe Ronin::Web::Spider::Agent do
-  describe "#initialize" do
-    context "when Ronin::Support::Network::HTTP.proxy is set" do
-      let(:proxy_host) { 'example.com' }
-      let(:proxy_port) { 8080 }
-      let(:proxy_uri)  { URI::HTTP.build(host: proxy_host, port: proxy_port) }
-
-      before { Ronin::Support::Network::HTTP.proxy = proxy_uri }
-
-      it "must parse ENV['RONIN_HTTP_USER_AGENT'] and set #proxy" do
-        expect(subject.proxy).to be_kind_of(Spidr::Proxy)
-        expect(subject.proxy.host).to eq(proxy_host)
-        expect(subject.proxy.port).to eq(proxy_port)
-      end
-
-      after { Ronin::Support::Network::HTTP.proxy = nil }
-    end
-
-    context "when Ronin::Support::Network::HTTP.user_agent is set" do
-      let(:user_agent) { 'Foo Bar' }
-
-      before { Ronin::Support::Network::HTTP.user_agent = user_agent }
-
-      it "must default #user_agent to ENV['RONIN_HTTP_USER_AGENT']" do
-        expect(subject.user_agent).to eq(user_agent)
-      end
-
-      after { Ronin::Support::Network::HTTP.user_agent = nil }
-    end
-
-    context "when given the proxy: keyword argument" do
-      let(:proxy_host) { 'example.com' }
-      let(:proxy_port) { 8080 }
-
-      context "and it's an Addressable::URI" do
-        let(:proxy) { Addressable::URI.new(host: proxy_host, port: proxy_port) }
-
-        subject { described_class.new(proxy: proxy) }
-
-        it "must convert it to a Spidr::Proxy object" do
-          expect(subject.proxy).to be_kind_of(Spidr::Proxy)
-          expect(subject.proxy.host).to eq(proxy_host)
-          expect(subject.proxy.port).to eq(proxy_port)
-        end
-      end
-
-      context "and it's an URI::HTTP" do
-        let(:proxy) { URI::HTTP.build(host: proxy_host, port: proxy_port) }
-
-        subject { described_class.new(proxy: proxy) }
-
-        it "must convert it to a Spidr::Proxy object" do
-          expect(subject.proxy).to be_kind_of(Spidr::Proxy)
-          expect(subject.proxy.host).to eq(proxy_host)
-          expect(subject.proxy.port).to eq(proxy_port)
-        end
-      end
-
-      context "and it's a Hash" do
-        let(:proxy) do
-          {host: proxy_host, port: proxy_port}
-        end
-
-        subject { described_class.new(proxy: proxy) }
-
-        it "must convert it to a Spidr::Proxy object" do
-          expect(subject.proxy).to be_kind_of(Spidr::Proxy)
-          expect(subject.proxy.host).to eq(proxy_host)
-          expect(subject.proxy.port).to eq(proxy_port)
-        end
-      end
-
-      context "and it's a String" do
-        let(:proxy) { "http://#{proxy_host}:#{proxy_port}" }
-
-        subject { described_class.new(proxy: proxy) }
-
-        it "must convert it to a Spidr::Proxy object" do
-          expect(subject.proxy).to be_kind_of(Spidr::Proxy)
-          expect(subject.proxy.host).to eq(proxy_host)
-          expect(subject.proxy.port).to eq(proxy_port)
-        end
-      end
-    end
-
-    context "when given the user_agent: keyword argument" do
-      context "and it's a String" do
-        let(:user_agent) { "test user-agent" }
-
-        subject { described_class.new(user_agent: user_agent) }
-
-        it "must set the #user_agent" do
-          expect(subject.user_agent).to eq(user_agent)
-        end
-      end
-
-      context "and it's a Symbol" do
-        let(:user_agent) { :chrome_linux }
-        let(:expected_user_agent) do
-          Ronin::Support::Network::HTTP::UserAgents[user_agent]
-        end
-
-        subject { described_class.new(user_agent: user_agent) }
-
-        it "must map the Symbol to one of Ronin::Support::Network::HTTP::UserAgents" do
-          expect(subject.user_agent).to eq(expected_user_agent)
-        end
-      end
-    end
-
-    it "must default #visited_hosts to nil" do
-      expect(subject.visited_hosts).to be(nil)
-    end
-  end
-
-  describe "#every_host" do
-    module TestAgentEveryHost
-      class Host1 < Sinatra::Base
-
-        set :host, 'host1.example.com'
-        set :port, 80
-
-        get '/' do
-          <<~HTML
-          <html>
-            <body>
-              <a href="/link1">link1</a>
-              <a href="http://host2.example.com/offsite-link">offsite link</a>
-              <a href="/link2">link2</a>
-            </body>
-          </html>
-          HTML
-        end
-
-        get '/link1' do
-          '<html><body>got here</body></html>'
-        end
-
-        get '/link2' do
-          '<html><body>got here</body></html>'
-        end
-      end
-
-      class Host2 < Sinatra::Base
-
-        set :host, 'host2.example.com'
-        set :port, 80
-
-        get '/offsite-link' do
-          '<html><body>should not get here</body></html>'
-        end
-
-      end
-    end
-
-    let(:host1) { 'host1.example.com' }
-    let(:host2) { 'host2.example.com'   }
-
-    let(:host1_app) { TestAgentEveryHost::Host1 }
-    let(:host2_app) { TestAgentEveryHost::Host2 }
-
-    before do
-      stub_request(:any, /#{Regexp.escape(host1)}/).to_rack(host1_app)
-      stub_request(:any, /#{Regexp.escape(host2)}/).to_rack(host2_app)
-    end
-
-    it "must yield every newly discovered hostname while spidering" do
-      yielded_hosts = []
-
-      subject.every_host do |host|
-        yielded_hosts << host
-      end
-
-      subject.start_at("http://#{host1}/")
-
-      expect(yielded_hosts).to eq([host1, host2])
-    end
-
-    it "must popualte #visited_hosts" do
-      subject.every_host { |host| }
-      subject.start_at("http://#{host1}/")
-
-      expect(subject.visited_hosts).to be_kind_of(Set)
-      expect(subject.visited_hosts.entries).to eq([host1, host2])
-    end
-  end
-
-  # TODO: need to figure out how to test #every_cert using webmock.
-  describe "#every_cert"
-  
-  describe "#every_favicon" do
-    module TestAgentEveryHost
-      class TestApp < Sinatra::Base
-
-        set :host, 'example.com'
-        set :port, 80
-
-        get '/' do
-          <<~HTML
-          <html>
-            <head>
-              <link rel="favicon" href="/favicon1.ico" type="image/x-icon"/>
-            </head>
-            <body>
-              <a href="/link1">link1</a>
-              <a href="http://host2.example.com/offsite-link">offsite link</a>
-              <a href="/link2">link2</a>
-            </body>
-          </html>
-          HTML
-        end
-
-        get '/favicon1.ico' do
-          content_type 'image/x-icon'
-
-          "favicon1"
-        end
-
-        get '/favicon2.ico' do
-          content_type 'image/vnd.microsoft.icon'
-
-          "favicon2"
-        end
-
-        get '/link1' do
-          '<html><body>got here</body></html>'
-        end
-
-        get '/link2' do
-          <<~HTML
-          <html>
-            <head>
-              <link rel="favicon" href="/favicon2.ico" type="image/x-icon"/>
-            </head>
-            <body>got here</body>
-          </html>
-          HTML
-        end
-      end
-    end
-
-    let(:host) { 'example.com' }
-
-    let(:test_app) { TestAgentEveryHost::TestApp }
-
-    before do
-      stub_request(:any, /#{Regexp.escape(host)}/).to_rack(test_app)
-    end
-
-    it "must yield Spidr::Page objects for each encountered .ico file" do
-      yielded_favicons = []
-
-      subject.every_favicon do |favicon|
-        yielded_favicons << favicon
-      end
-
-      subject.start_at("http://#{host}/")
-
-      expect(yielded_favicons).to_not be_empty
-
-      expect(yielded_favicons[0]).to be_kind_of(Spidr::Page)
-      expect(yielded_favicons[0].content_type).to eq('image/x-icon')
-      expect(yielded_favicons[0].url).to eq(URI("http://#{host}/favicon1.ico"))
-
-      expect(yielded_favicons[1]).to be_kind_of(Spidr::Page)
-      expect(yielded_favicons[1].content_type).to eq('image/vnd.microsoft.icon')
-      expect(yielded_favicons[1].url).to eq(URI("http://#{host}/favicon2.ico"))
-    end
-  end
-
-  describe "#every_html_comment" do
-    module TestAgentEveryHTMLComment
-      class TestApp < Sinatra::Base
-
-        set :host, 'example.com'
-        set :port, 80
-
-        get '/' do
-          <<~HTML
-          <html>
-            <head>
-              <!-- comment 1 -->
-            </head>
-            <!-- -->
-            <body>
-              <!-- comment 2 -->
-            </body>
-          </html>
-          HTML
-        end
-      end
-    end
-
-    let(:host) { 'example.com' }
-
-    let(:test_app) { TestAgentEveryHTMLComment::TestApp }
-
-    before do
-      stub_request(:any, /#{Regexp.escape(host)}/).to_rack(test_app)
-    end
-
-    it "must yield every non-empty/non-whitespace HTML comment String" do
-      yielded_comments = []
-
-      subject.every_html_comment do |comment|
-        yielded_comments << comment
-      end
-
-      subject.start_at("http://#{host}/")
-
-      expect(yielded_comments).to match_array(
-        [
-          'comment 1',
-          'comment 2'
-        ]
-      )
-    end
-  end
-
-  describe "#every_javascript" do
-    module TestAgentEveryJavaScript
-      class TestApp < Sinatra::Base
-
-        set :host, 'example.com'
-        set :port, 80
-
-        get '/' do
-          <<~HTML
-          <html>
-            <head>
-              <script type="text/javascript" src="/javascript1.js"></script>
-              <script type="text/javascript">javascript2</script>
-            </head>
-            <body>
-              <a href="/link1">link1</a>
-              <a href="http://host2.example.com/offsite-link">offsite link</a>
-              <a href="/link2">link2</a>
-            </body>
-          </html>
-          HTML
-        end
-
-        get '/javascript1.js' do
-          content_type 'text/javascript'
-          "javascript1"
-        end
-      end
-    end
-
-    let(:host) { 'example.com' }
-
-    let(:test_app) { TestAgentEveryJavaScript::TestApp }
-
-    before do
-      stub_request(:any, /#{Regexp.escape(host)}/).to_rack(test_app)
-    end
-
-    it "must yield both the contents of .js files and inline <script> tags" do
-      yielded_javascripts = []
-
-      subject.every_javascript do |js|
-        yielded_javascripts << js
-      end
-
-      subject.start_at("http://#{host}/")
-
-      expect(yielded_javascripts).to match_array(%w[javascript1 javascript2])
-    end
-  end
-
-  describe "#every_javascript_string" do
-    module TestAgentEveryJavaScriptString
-      class TestApp < Sinatra::Base
-
-        set :host, 'example.com'
-        set :port, 80
-
-        get '/' do
-          <<~HTML
-          <html>
-            <head>
-              <script type="text/javascript" src="/javascript1.js"></script>
-              <script type="text/javascript">
-              var str3 = "string #3";
-              var str4 = 'string #4';
-              </script>
-            </head>
-            <body>
-              <a href="/link1">link1</a>
-              <a href="http://host2.example.com/offsite-link">offsite link</a>
-              <a href="/link2">link2</a>
-            </body>
-          </html>
-          HTML
-        end
-
-        get '/javascript1.js' do
-          content_type 'text/javascript'
-          <<~JS
-          var str1 = "string #1";
-          var str2 = 'string #2';
-          JS
-        end
-      end
-    end
-
-    let(:host) { 'example.com' }
-
-    let(:test_app) { TestAgentEveryJavaScriptString::TestApp }
-
-    before do
-      stub_request(:any, /#{Regexp.escape(host)}/).to_rack(test_app)
-    end
-
-    it "must yield every JavaScript string from any <script> tag" do
-      yielded_javascript_strings = []
-
-      subject.every_javascript_string do |string|
-        yielded_javascript_strings << string
-      end
-
-      subject.start_at("http://#{host}/")
-
-      expect(yielded_javascript_strings).to match_array(
-        [
-          'string #1',
-          'string #2',
-          'string #3',
-          'string #4'
-        ]
-      )
-    end
-  end
-
-  describe "#every_javascript_comment" do
-    module TestAgentEveryJavaScriptComment
-      class TestApp < Sinatra::Base
-
-        set :host, 'example.com'
-        set :port, 80
-
-        get '/' do
-          <<~HTML
-          <html>
-            <head>
-              <script type="text/javascript" src="/javascript1.js"></script>
-              <script type="text/javascript">
-              // comment 3
-              var str3 = "string #3";
-              /*
-                 comment 4
-               */
-              var str4 = 'string #4';
-              </script>
-            </head>
-            <body>
-              <a href="/link1">link1</a>
-              <a href="http://host2.example.com/offsite-link">offsite link</a>
-              <a href="/link2">link2</a>
-            </body>
-          </html>
-          HTML
-        end
-
-        get '/javascript1.js' do
-          content_type 'text/javascript'
-          <<~JS
-          // comment 1
-          var str1 = "string #1";
-          /* comment 2 */
-          var str2 = 'string #2';
-          JS
-        end
-      end
-    end
-
-    let(:host) { 'example.com' }
-
-    let(:test_app) { TestAgentEveryJavaScriptComment::TestApp }
-
-    before do
-      stub_request(:any, /#{Regexp.escape(host)}/).to_rack(test_app)
-    end
-
-    it "must yield every JavaScript comment from any <script> tag" do
-      yielded_javascript_comments = []
-
-      subject.every_javascript_comment do |comment|
-        yielded_javascript_comments << comment
-      end
-
-      subject.start_at("http://#{host}/")
-
-      expect(yielded_javascript_comments).to match_array(
-        [
-          "// comment 1\n",
-          "/* comment 2 */",
-          "// comment 3\n",
-          "/*\n       comment 4\n     */"
-        ]
-      )
-    end
-  end
-
-  describe "#every_comment" do
-    module TestAgentEveryComment
-      class TestApp < Sinatra::Base
-
-        set :host, 'example.com'
-        set :port, 80
-
-        get '/' do
-          <<~HTML
-          <html>
-            <head>
-              <!-- HTML comment 1 -->
-              <script type="text/javascript" src="/javascript1.js"></script>
-              <script type="text/javascript">
-              // JavaScript comment 3
-              var str3 = "string #3";
-              /*
-                 JavaScript comment 4
-               */
-              var str4 = 'string #4';
-              </script>
-            </head>
-            <!-- -->
-            <body>
-              <!-- HTML comment 2 -->
-              <a href="/link1">link1</a>
-              <a href="http://host2.example.com/offsite-link">offsite link</a>
-              <a href="/link2">link2</a>
-            </body>
-          </html>
-          HTML
-        end
-
-        get '/javascript1.js' do
-          content_type 'text/javascript'
-          <<~JS
-          // JavaScript comment 1
-          var str1 = "string #1";
-          /* JavaScript comment 2 */
-          var str2 = 'string #2';
-          JS
-        end
-      end
-    end
-
-    let(:host) { 'example.com' }
-
-    let(:test_app) { TestAgentEveryComment::TestApp }
-
-    before do
-      stub_request(:any, /#{Regexp.escape(host)}/).to_rack(test_app)
-    end
-
-    it "must yield every HTML and JavaScript comment from any <script> tag" do
-      yielded_comments = []
-
-      subject.every_comment do |comment|
-        yielded_comments << comment
-      end
-
-      subject.start_at("http://#{host}/")
-
-      expect(yielded_comments).to match_array(
-        [
-          "HTML comment 1",
-          "// JavaScript comment 1\n",
-          "/* JavaScript comment 2 */",
-          "// JavaScript comment 3\n",
-          "/*\n       JavaScript comment 4\n     */",
-          "HTML comment 2"
-        ]
-      )
-    end
-  end
-end

data/spec/archive_spec.rb

@@ -1,91 +0,0 @@
-require 'spec_helper'
-require 'ronin/web/spider/archive'
-
-require 'tmpdir'
-
-describe Ronin::Web::Spider::Archive do
-  let(:root) { File.join(Dir.mktmpdir('ronin-web-spider')) }
-
-  subject { described_class.new(root) }
-
-  describe "#initialize" do
-    it "must set #root" do
-      expect(subject.root).to eq(root)
-    end
-  end
-
-  describe ".open" do
-    subject { described_class.open(root) }
-
-    it "must return a new #{described_class}" do
-      expect(subject).to be_kind_of(described_class)
-    end
-
-    context "when given a block" do
-      it "must yield the new #{described_class}" do
-        expect { |b|
-          described_class.open(root,&b)
-        }.to yield_with_args(described_class)
-      end
-    end
-
-    context "when the root directory does not exist" do
-      let(:root) { File.join(super(),'does-not-exist-yet') }
-
-      it "must create the given root directory" do
-        described_class.open(root)
-
-        expect(File.directory?(root)).to be(true)
-      end
-    end
-
-    context "when the root directory does exist" do
-      let(:root) { File.join(super(),'does-not-exist-yet') }
-
-      before { FileUtils.mkdir(root) }
-
-      it "must not raise an error" do
-        expect {
-          described_class.open(root)
-        }.to_not raise_error
-      end
-    end
-  end
-
-  describe "#write" do
-    let(:url)  { URI('https://example.com/foo/bar.html') }
-    let(:body) { 'test file' }
-
-    before { subject.write(url,body) }
-
-    it "must automatically create parent directory" do
-      expect(File.directory?(File.join(root,'foo'))).to be(true)
-    end
-
-    it "must write the body into the file" do
-      expect(File.read(File.join(root,'foo','bar.html'))).to eq(body)
-    end
-
-    context "when the URL has a query string" do
-      let(:url) { URI('https://example.com/foo/bar.php?q=1') }
-
-      it "must include the query string as part of the file name" do
-        expect(File.read(File.join(root,'foo','bar.php?q=1'))).to eq(body)
-      end
-    end
-
-    context "when the URL path ends with a '/'" do
-      let(:url) { URI('https://example.com/foo/bar/') }
-
-      it "must write the body to an index.html file within the URL's path" do
-        expect(File.read(File.join(root,'foo','bar','index.html'))).to eq(body)
-      end
-    end
-  end
-
-  describe "#to_s" do
-    it "must return the root directory" do
-      expect(subject.to_s).to eq(root)
-    end
-  end
-end

data/spec/example_app.rb

@@ -1,27 +0,0 @@
-require 'rspec'
-require 'sinatra/base'
-require 'webmock/rspec'
-
-require 'ronin/web/spider/agent'
-
-RSpec.shared_context "example App" do
-  let(:host) { 'example.com' }
-
-  subject { Ronin::Web::Spider::Agent.new(host: host) }
-
-  def self.app(&block)
-    let(:app) do
-      klass = Class.new(Sinatra::Base)
-      klass.set :host, host
-      klass.set :port, 80
-      klass.class_eval(&block)
-      return klass
-    end
-
-    before do
-      stub_request(:any, /#{Regexp.escape(host)}/).to_rack(app)
-
-      subject.start_at("http://#{host}/")
-    end
-  end
-end