ronin-web-spider 0.1.0.beta2 → 0.1.1

data/spec/archive_spec.rb

@@ -1,91 +0,0 @@
-require 'spec_helper'
-require 'ronin/web/spider/archive'
-
-require 'tmpdir'
-
-describe Ronin::Web::Spider::Archive do
-  let(:root) { File.join(Dir.mktmpdir('ronin-web-spider')) }
-
-  subject { described_class.new(root) }
-
-  describe "#initialize" do
-    it "must set #root" do
-      expect(subject.root).to eq(root)
-    end
-  end
-
-  describe ".open" do
-    subject { described_class.open(root) }
-
-    it "must return a new #{described_class}" do
-      expect(subject).to be_kind_of(described_class)
-    end
-
-    context "when given a block" do
-      it "must yield the new #{described_class}" do
-        expect { |b|
-          described_class.open(root,&b)
-        }.to yield_with_args(described_class)
-      end
-    end
-
-    context "when the root directory does not exist" do
-      let(:root) { File.join(super(),'does-not-exist-yet') }
-
-      it "must create the given root directory" do
-        described_class.open(root)
-
-        expect(File.directory?(root)).to be(true)
-      end
-    end
-
-    context "when the root directory does exist" do
-      let(:root) { File.join(super(),'does-not-exist-yet') }
-
-      before { FileUtils.mkdir(root) }
-
-      it "must not raise an error" do
-        expect {
-          described_class.open(root)
-        }.to_not raise_error
-      end
-    end
-  end
-
-  describe "#write" do
-    let(:url)  { URI('https://example.com/foo/bar.html') }
-    let(:body) { 'test file' }
-
-    before { subject.write(url,body) }
-
-    it "must automatically create parent directory" do
-      expect(File.directory?(File.join(root,'foo'))).to be(true)
-    end
-
-    it "must write the body into the file" do
-      expect(File.read(File.join(root,'foo','bar.html'))).to eq(body)
-    end
-
-    context "when the URL has a query string" do
-      let(:url) { URI('https://example.com/foo/bar.php?q=1') }
-
-      it "must include the query string as part of the file name" do
-        expect(File.read(File.join(root,'foo','bar.php?q=1'))).to eq(body)
-      end
-    end
-
-    context "when the URL path ends with a '/'" do
-      let(:url) { URI('https://example.com/foo/bar/') }
-
-      it "must write the body to an index.html file within the URL's path" do
-        expect(File.read(File.join(root,'foo','bar','index.html'))).to eq(body)
-      end
-    end
-  end
-
-  describe "#to_s" do
-    it "must return the root directory" do
-      expect(subject.to_s).to eq(root)
-    end
-  end
-end

data/spec/example_app.rb

@@ -1,27 +0,0 @@
-require 'rspec'
-require 'sinatra/base'
-require 'webmock/rspec'
-
-require 'ronin/web/spider/agent'
-
-RSpec.shared_context "example App" do
-  let(:host) { 'example.com' }
-
-  subject { Ronin::Web::Spider::Agent.new(host: host) }
-
-  def self.app(&block)
-    let(:app) do
-      klass = Class.new(Sinatra::Base)
-      klass.set :host, host
-      klass.set :port, 80
-      klass.class_eval(&block)
-      return klass
-    end
-
-    before do
-      stub_request(:any, /#{Regexp.escape(host)}/).to_rack(app)
-
-      subject.start_at("http://#{host}/")
-    end
-  end
-end

data/spec/git_archive_spec.rb

@@ -1,137 +0,0 @@
-require 'spec_helper'
-require 'ronin/web/spider/git_archive'
-
-require 'tmpdir'
-
-describe Ronin::Web::Spider::GitArchive do
-  let(:root) { File.join(Dir.mktmpdir('ronin-web-spider')) }
-
-  describe ".open" do
-    subject { described_class }
-
-    context "when the root directory does not already exist" do
-      let(:root) { File.join(Dir.tmpdir,'ronin-web-spider-new-dir') }
-
-      it "must run `git init` on the new archive directory" do
-        subject.open(root)
-
-        expect(File.directory?(File.join(root,'.git'))).to be(true)
-      end
-
-      after { FileUtils.rm_r(root) }
-    end
-
-    context "when the root directory already exists" do
-      context "but does not contain a .git directory" do
-        it "must run `git init` within the root directory" do
-          subject.open(root)
-
-          expect(File.directory?(File.join(root,'.git'))).to be(true)
-        end
-      end
-    end
-  end
-
-  subject { described_class.open(root) }
-
-  describe "#git?" do
-    subject { described_class.new(root) }
-
-    context "when the archive directory contains a .git directory" do
-      before do
-        FileUtils.mkdir(File.join(root,'.git'))
-      end
-
-      it "must return true" do
-        expect(subject.git?).to be(true)
-      end
-    end
-
-    context "when the archive directory does not contains a .git directory" do
-      it "must return false" do
-        expect(subject.git?).to be(false)
-      end
-    end
-  end
-
-  describe "#init" do
-    it "must run the 'git init' command" do
-      expect(subject).to receive(:system).with('git','-C',root,'init').and_return(true)
-
-      subject.init
-    end
-
-    context "when the 'git init' command fails" do
-      it do
-        allow(subject).to receive(:system).with('git','-C',root,'init').and_return(false)
-
-        expect {
-          subject.init
-        }.to raise_error(Ronin::Web::Spider::GitError,"git command failed: git -C #{root} init")
-      end
-    end
-
-    context "when 'git' is not installed" do
-      it do
-        allow(subject).to receive(:system).with('git','-C',root,'init').and_return(nil)
-
-        expect {
-          subject.init
-        }.to raise_error(Ronin::Web::Spider::GitError,"the git command was not found")
-      end
-    end
-  end
-
-  describe "#write" do
-    let(:url)  { URI('https://example.com/foo/bar.html') }
-    let(:body) { 'test file' }
-
-    it "must automatically create parent directory" do
-      subject.write(url,body)
-
-      expect(File.directory?(File.join(root,'foo'))).to be(true)
-    end
-
-    it "must write the body into the file" do
-      subject.write(url,body)
-
-      expect(File.read(File.join(root,'foo','bar.html'))).to eq(body)
-    end
-
-    it "must add the file using `git add`" do
-      absolute_path = File.join(root,'foo','bar.html')
-
-      expect(subject).to receive(:system).with(
-        'git', '-C', root, 'add', absolute_path
-      ).and_return(true)
-
-      subject.write(url,body)
-    end
-  end
-
-  describe "#commit" do
-    let(:message) { 'commit message' }
-
-    context "when a block is given" do
-      it "must yield control before calling `git commit -m ...` with the commit message" do
-        expect(subject).to receive(:system).with(
-          'git', '-C', root, 'commit', '-m', message
-        ).and_return(true)
-
-        expect { |b|
-          subject.commit(message,&b)
-        }.to yield_with_args(subject)
-      end
-    end
-
-    context "when no block is given" do
-      it "must not yield and call `git commit -m ...` with the commit message" do
-        expect(subject).to receive(:system).with(
-          'git', '-C', root, 'commit', '-m', message
-        ).and_return(true)
-
-        subject.commit(message)
-      end
-    end
-  end
-end

data/spec/spec_helper.rb

@@ -1,4 +0,0 @@
-require 'rspec'
-require 'simplecov'
-
-SimpleCov.start

data/spec/spider_spec.rb

@@ -1,252 +0,0 @@
-require 'spec_helper'
-require 'example_app'
-
-require 'ronin/web/spider'
-
-describe Ronin::Web::Spider do
-  include_context "example App"
-
-  describe ".start_at" do
-    module TestAgentStartAt
-      class ExampleApp < Sinatra::Base
-
-        set :host, 'example.com'
-        set :port, 80
-
-        get '/' do
-          '<html><body>should not get here</body></html>'
-        end
-
-        get '/entry-point' do
-          <<~HTML
-          <html>
-            <body>
-              <a href="/link1">link1</a>
-              <a href="http://other.com/offsite-link">offsite link</a>
-              <a href="/link2">link2</a>
-            </body>
-          </html>
-          HTML
-        end
-
-        get '/link1' do
-          '<html><body>got here</body></html>'
-        end
-
-        get '/link2' do
-          '<html><body>got here</body></html>'
-        end
-      end
-
-      class OtherApp < Sinatra::Base
-
-        set :host, 'other.com'
-        set :port, 80
-
-        get '/offsite-link' do
-          '<html><body>should not get here</body></html>'
-        end
-
-      end
-    end
-
-    subject { described_class }
-
-    let(:host)       { 'example.com' }
-    let(:other_host) { 'other.com'   }
-    let(:url)        { URI("http://#{host}/entry-point") }
-
-    let(:app)       { TestAgentStartAt::ExampleApp }
-    let(:other_app) { TestAgentStartAt::OtherApp   }
-
-    before do
-      stub_request(:any, /#{Regexp.escape(host)}/).to_rack(app)
-      stub_request(:any, /#{Regexp.escape(other_host)}/).to_rack(other_app)
-    end
-
-    it "must spider the website starting at the given URL" do
-      agent = subject.start_at(url)
-
-      expect(agent.history).to be == Set[
-        URI("http://#{host}/entry-point"),
-        URI("http://#{host}/link1"),
-        URI("http://#{other_host}/offsite-link"),
-        URI("http://#{host}/link2")
-      ]
-    end
-  end
-
-  describe ".site" do
-    module TestAgentSite
-      class ExampleApp < Sinatra::Base
-
-        set :host, 'example.com'
-        set :port, 80
-
-        get '/' do
-          '<html><body>should not get here</body></html>'
-        end
-
-        get '/entry-point' do
-          <<~HTML
-            <html>
-              <body>
-                <a href="/link1">link1</a>
-                <a href="http://other.com/offsite-link">offsite link</a>
-                <a href="/link2">link2</a>
-              </body>
-            </html>
-          HTML
-        end
-
-        get '/link1' do
-          '<html><body>got here</body></html>'
-        end
-
-        get '/link2' do
-          '<html><body>got here</body></html>'
-        end
-
-      end
-    end
-
-    subject { described_class }
-
-    let(:host) { 'example.com' }
-    let(:url)  { URI("http://#{host}/entry-point") }
-
-    let(:app) { TestAgentSite::ExampleApp }
-
-    before do
-      stub_request(:any, /#{Regexp.escape(host)}/).to_rack(app)
-    end
-
-    it "must spider the website starting at the given URL" do
-      agent = subject.site(url)
-
-      expect(agent.history).to be == Set[
-        URI("http://#{host}/entry-point"),
-        URI("http://#{host}/link1"),
-        URI("http://#{host}/link2")
-      ]
-    end
-  end
-
-  describe ".host" do
-    module TestAgentHost
-      class ExampleApp < Sinatra::Base
-
-        set :host, 'example.com'
-        set :port, 80
-
-        get '/' do
-          <<~HTML
-            <html>
-              <body>
-                <a href="/link1">link1</a>
-                <a href="http://other.com/offsite-link">offsite link</a>
-                <a href="/link2">link2</a>
-              </body>
-            </html>
-          HTML
-        end
-
-        get '/link1' do
-          '<html><body>got here</body></html>'
-        end
-
-        get '/link2' do
-          '<html><body>got here</body></html>'
-        end
-
-      end
-    end
-
-    subject { described_class }
-
-    let(:host) { 'example.com' }
-    let(:app)  { TestAgentHost::ExampleApp }
-
-    before do
-      stub_request(:any, /#{Regexp.escape(host)}/).to_rack(app)
-    end
-
-    it "must spider the website starting at the given URL" do
-      agent = subject.host(host)
-
-      # XXX: for some reason Set#== was returning false, so convert to an Array
-      expect(agent.history.to_a).to be == [
-        URI("http://#{host}/"),
-        URI("http://#{host}/link1"),
-        URI("http://#{host}/link2")
-      ]
-    end
-  end
-
-  describe ".domain" do
-    module TestAgentDomain
-      class ExampleApp < Sinatra::Base
-
-        set :host, 'example.com'
-        set :port, 80
-
-        get '/' do
-          <<~HTML
-            <html>
-              <body>
-                <a href="/link1">link1</a>
-                <a href="http://sub.example.com/subdomain-link">subdomain link</a>
-                <a href="/link2">link2</a>
-              </body>
-            </html>
-          HTML
-        end
-
-        get '/link1' do
-          '<html><body>got here</body></html>'
-        end
-
-        get '/link2' do
-          '<html><body>got here</body></html>'
-        end
-
-      end
-
-      class SubDomainApp < Sinatra::Base
-
-        set :host, 'sub.example.com'
-        set :port, 80
-
-        get '/subdomain-link' do
-          '<html><body>should get here</body></html>'
-        end
-
-      end
-    end
-
-    subject { described_class }
-
-    let(:domain)        { 'example.com' }
-    let(:domain_app)    { TestAgentDomain::ExampleApp }
-
-    let(:subdomain)     { 'sub.example.com' }
-    let(:subdomain_app) { TestAgentDomain::SubDomainApp }
-
-    before do
-      stub_request(:any, /#{Regexp.escape(subdomain)}/).to_rack(subdomain_app)
-      stub_request(:any, /#{Regexp.escape(domain)}/).to_rack(domain_app)
-    end
-
-    it "must spider the domain and subdomains starting at the given domain" do
-      agent = subject.domain(domain)
-
-      # XXX: for some reason Set#== was returning false, so convert to an Array
-      expect(agent.history.to_a).to be == [
-        URI("http://#{domain}/"),
-        URI("http://#{domain}/link1"),
-        URI("http://#{subdomain}/subdomain-link"),
-        URI("http://#{domain}/link2")
-      ]
-    end
-  end
-end