ronin-web-spider 0.1.0.beta1

data/spec/git_archive_spec.rb

@@ -0,0 +1,137 @@
+require 'spec_helper'
+require 'ronin/web/spider/git_archive'
+
+require 'tmpdir'
+
+describe Ronin::Web::Spider::GitArchive do
+  let(:root) { File.join(Dir.mktmpdir('ronin-web-spider')) }
+
+  describe ".open" do
+    subject { described_class }
+
+    context "when the root directory does not already exist" do
+      let(:root) { File.join(Dir.tmpdir,'ronin-web-spider-new-dir') }
+
+      it "must run `git init` on the new archive directory" do
+        subject.open(root)
+
+        expect(File.directory?(File.join(root,'.git'))).to be(true)
+      end
+
+      after { FileUtils.rm_r(root) }
+    end
+
+    context "when the root directory already exists" do
+      context "but does not contain a .git directory" do
+        it "must run `git init` within the root directory" do
+          subject.open(root)
+
+          expect(File.directory?(File.join(root,'.git'))).to be(true)
+        end
+      end
+    end
+  end
+
+  subject { described_class.open(root) }
+
+  describe "#git?" do
+    subject { described_class.new(root) }
+
+    context "when the archive directory contains a .git directory" do
+      before do
+        FileUtils.mkdir(File.join(root,'.git'))
+      end
+
+      it "must return true" do
+        expect(subject.git?).to be(true)
+      end
+    end
+
+    context "when the archive directory does not contains a .git directory" do
+      it "must return false" do
+        expect(subject.git?).to be(false)
+      end
+    end
+  end
+
+  describe "#init" do
+    it "must run the 'git init' command" do
+      expect(subject).to receive(:system).with('git','-C',root,'init').and_return(true)
+
+      subject.init
+    end
+
+    context "when the 'git init' command fails" do
+      it do
+        allow(subject).to receive(:system).with('git','-C',root,'init').and_return(false)
+
+        expect {
+          subject.init
+        }.to raise_error(Ronin::Web::Spider::GitError,"git command failed: git -C #{root} init")
+      end
+    end
+
+    context "when 'git' is not installed" do
+      it do
+        allow(subject).to receive(:system).with('git','-C',root,'init').and_return(nil)
+
+        expect {
+          subject.init
+        }.to raise_error(Ronin::Web::Spider::GitError,"the git command was not found")
+      end
+    end
+  end
+
+  describe "#write" do
+    let(:url)  { URI('https://example.com/foo/bar.html') }
+    let(:body) { 'test file' }
+
+    it "must automatically create parent directory" do
+      subject.write(url,body)
+
+      expect(File.directory?(File.join(root,'foo'))).to be(true)
+    end
+
+    it "must write the body into the file" do
+      subject.write(url,body)
+
+      expect(File.read(File.join(root,'foo','bar.html'))).to eq(body)
+    end
+
+    it "must add the file using `git add`" do
+      absolute_path = File.join(root,'foo','bar.html')
+
+      expect(subject).to receive(:system).with(
+        'git', '-C', root, 'add', absolute_path
+      ).and_return(true)
+
+      subject.write(url,body)
+    end
+  end
+
+  describe "#commit" do
+    let(:message) { 'commit message' }
+
+    context "when a block is given" do
+      it "must yield control before calling `git commit -m ...` with the commit message" do
+        expect(subject).to receive(:system).with(
+          'git', '-C', root, 'commit', '-m', message
+        ).and_return(true)
+
+        expect { |b|
+          subject.commit(message,&b)
+        }.to yield_with_args(subject)
+      end
+    end
+
+    context "when no block is given" do
+      it "must not yield and call `git commit -m ...` with the commit message" do
+        expect(subject).to receive(:system).with(
+          'git', '-C', root, 'commit', '-m', message
+        ).and_return(true)
+
+        subject.commit(message)
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,4 @@
+require 'rspec'
+require 'simplecov'
+
+SimpleCov.start

data/spec/spider_spec.rb

@@ -0,0 +1,252 @@
+require 'spec_helper'
+require 'example_app'
+
+require 'ronin/web/spider'
+
+describe Ronin::Web::Spider do
+  include_context "example App"
+
+  describe ".start_at" do
+    module TestAgentStartAt
+      class ExampleApp < Sinatra::Base
+
+        set :host, 'example.com'
+        set :port, 80
+
+        get '/' do
+          '<html><body>should not get here</body></html>'
+        end
+
+        get '/entry-point' do
+          <<~HTML
+          <html>
+            <body>
+              <a href="/link1">link1</a>
+              <a href="http://other.com/offsite-link">offsite link</a>
+              <a href="/link2">link2</a>
+            </body>
+          </html>
+          HTML
+        end
+
+        get '/link1' do
+          '<html><body>got here</body></html>'
+        end
+
+        get '/link2' do
+          '<html><body>got here</body></html>'
+        end
+      end
+
+      class OtherApp < Sinatra::Base
+
+        set :host, 'other.com'
+        set :port, 80
+
+        get '/offsite-link' do
+          '<html><body>should not get here</body></html>'
+        end
+
+      end
+    end
+
+    subject { described_class }
+
+    let(:host)       { 'example.com' }
+    let(:other_host) { 'other.com'   }
+    let(:url)        { URI("http://#{host}/entry-point") }
+
+    let(:app)       { TestAgentStartAt::ExampleApp }
+    let(:other_app) { TestAgentStartAt::OtherApp   }
+
+    before do
+      stub_request(:any, /#{Regexp.escape(host)}/).to_rack(app)
+      stub_request(:any, /#{Regexp.escape(other_host)}/).to_rack(other_app)
+    end
+
+    it "must spider the website starting at the given URL" do
+      agent = subject.start_at(url)
+
+      expect(agent.history).to be == Set[
+        URI("http://#{host}/entry-point"),
+        URI("http://#{host}/link1"),
+        URI("http://#{other_host}/offsite-link"),
+        URI("http://#{host}/link2")
+      ]
+    end
+  end
+
+  describe ".site" do
+    module TestAgentSite
+      class ExampleApp < Sinatra::Base
+
+        set :host, 'example.com'
+        set :port, 80
+
+        get '/' do
+          '<html><body>should not get here</body></html>'
+        end
+
+        get '/entry-point' do
+          <<~HTML
+            <html>
+              <body>
+                <a href="/link1">link1</a>
+                <a href="http://other.com/offsite-link">offsite link</a>
+                <a href="/link2">link2</a>
+              </body>
+            </html>
+          HTML
+        end
+
+        get '/link1' do
+          '<html><body>got here</body></html>'
+        end
+
+        get '/link2' do
+          '<html><body>got here</body></html>'
+        end
+
+      end
+    end
+
+    subject { described_class }
+
+    let(:host) { 'example.com' }
+    let(:url)  { URI("http://#{host}/entry-point") }
+
+    let(:app) { TestAgentSite::ExampleApp }
+
+    before do
+      stub_request(:any, /#{Regexp.escape(host)}/).to_rack(app)
+    end
+
+    it "must spider the website starting at the given URL" do
+      agent = subject.site(url)
+
+      expect(agent.history).to be == Set[
+        URI("http://#{host}/entry-point"),
+        URI("http://#{host}/link1"),
+        URI("http://#{host}/link2")
+      ]
+    end
+  end
+
+  describe ".host" do
+    module TestAgentHost
+      class ExampleApp < Sinatra::Base
+
+        set :host, 'example.com'
+        set :port, 80
+
+        get '/' do
+          <<~HTML
+            <html>
+              <body>
+                <a href="/link1">link1</a>
+                <a href="http://other.com/offsite-link">offsite link</a>
+                <a href="/link2">link2</a>
+              </body>
+            </html>
+          HTML
+        end
+
+        get '/link1' do
+          '<html><body>got here</body></html>'
+        end
+
+        get '/link2' do
+          '<html><body>got here</body></html>'
+        end
+
+      end
+    end
+
+    subject { described_class }
+
+    let(:host) { 'example.com' }
+    let(:app)  { TestAgentHost::ExampleApp }
+
+    before do
+      stub_request(:any, /#{Regexp.escape(host)}/).to_rack(app)
+    end
+
+    it "must spider the website starting at the given URL" do
+      agent = subject.host(host)
+
+      # XXX: for some reason Set#== was returning false, so convert to an Array
+      expect(agent.history.to_a).to be == [
+        URI("http://#{host}/"),
+        URI("http://#{host}/link1"),
+        URI("http://#{host}/link2")
+      ]
+    end
+  end
+
+  describe ".domain" do
+    module TestAgentDomain
+      class ExampleApp < Sinatra::Base
+
+        set :host, 'example.com'
+        set :port, 80
+
+        get '/' do
+          <<~HTML
+            <html>
+              <body>
+                <a href="/link1">link1</a>
+                <a href="http://sub.example.com/subdomain-link">subdomain link</a>
+                <a href="/link2">link2</a>
+              </body>
+            </html>
+          HTML
+        end
+
+        get '/link1' do
+          '<html><body>got here</body></html>'
+        end
+
+        get '/link2' do
+          '<html><body>got here</body></html>'
+        end
+
+      end
+
+      class SubDomainApp < Sinatra::Base
+
+        set :host, 'sub.example.com'
+        set :port, 80
+
+        get '/subdomain-link' do
+          '<html><body>should get here</body></html>'
+        end
+
+      end
+    end
+
+    subject { described_class }
+
+    let(:domain)        { 'example.com' }
+    let(:domain_app)    { TestAgentDomain::ExampleApp }
+
+    let(:subdomain)     { 'sub.example.com' }
+    let(:subdomain_app) { TestAgentDomain::SubDomainApp }
+
+    before do
+      stub_request(:any, /#{Regexp.escape(subdomain)}/).to_rack(subdomain_app)
+      stub_request(:any, /#{Regexp.escape(domain)}/).to_rack(domain_app)
+    end
+
+    it "must spider the domain and subdomains starting at the given domain" do
+      agent = subject.domain(domain)
+
+      # XXX: for some reason Set#== was returning false, so convert to an Array
+      expect(agent.history.to_a).to be == [
+        URI("http://#{domain}/"),
+        URI("http://#{domain}/link1"),
+        URI("http://#{subdomain}/subdomain-link"),
+        URI("http://#{domain}/link2")
+      ]
+    end
+  end
+end

metadata

@@ -0,0 +1,122 @@
+--- !ruby/object:Gem::Specification
+name: ronin-web-spider
+version: !ruby/object:Gem::Version
+  version: 0.1.0.beta1
+platform: ruby
+authors:
+- Postmodern
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2023-01-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: spidr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.7'
+- !ruby/object:Gem::Dependency
+  name: ronin-support
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+description: ronin-web-spider is a collection of common web spidering routines using
+  the spidr gem.
+email: postmodern.mod3@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files:
+- COPYING.txt
+- ChangeLog.md
+- README.md
+files:
+- ".document"
+- ".github/workflows/ruby.yml"
+- ".gitignore"
+- ".rspec"
+- ".ruby-version"
+- ".yardopts"
+- COPYING.txt
+- ChangeLog.md
+- Gemfile
+- README.md
+- Rakefile
+- gemspec.yml
+- lib/ronin/web/spider.rb
+- lib/ronin/web/spider/agent.rb
+- lib/ronin/web/spider/archive.rb
+- lib/ronin/web/spider/exceptions.rb
+- lib/ronin/web/spider/git_archive.rb
+- lib/ronin/web/spider/version.rb
+- ronin-web-spider.gemspec
+- spec/agent_spec.rb
+- spec/archive_spec.rb
+- spec/example_app.rb
+- spec/git_archive_spec.rb
+- spec/spec_helper.rb
+- spec/spider_spec.rb
+homepage: https://ronin-rb.dev/
+licenses:
+- LGPL-3.0
+metadata:
+  documentation_uri: https://rubydoc.info/gems/ronin-web-spider
+  source_code_uri: https://github.com/ronin-rb/ronin-web-spider
+  bug_tracker_uri: https://github.com/ronin-rb/ronin-web-spider/issues
+  changelog_uri: https://github.com/ronin-rb/ronin-web-spider/blob/master/ChangeLog.md
+  rubygems_mfa_required: 'true'
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.0.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.3.26
+signing_key:
+specification_version: 4
+summary: collection of common web spidering routines
+test_files:
+- spec/agent_spec.rb
+- spec/archive_spec.rb
+- spec/git_archive_spec.rb
+- spec/spider_spec.rb