ronin-web-spider 0.1.0.beta2 → 0.1.1

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ronin-web-spider
 version: !ruby/object:Gem::Version
-  version: 0.1.0.beta2
+  version: 0.1.1
 platform: ruby
 authors:
 - Postmodern
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-01-01 00:00:00.000000000 Z
+date: 2024-06-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: spidr
@@ -30,14 +30,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.0.beta1
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.0.0.beta1
+        version: '1.0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -66,6 +66,7 @@ files:
 - ".github/workflows/ruby.yml"
 - ".gitignore"
 - ".rspec"
+- ".rubocop.yml"
 - ".ruby-version"
 - ".yardopts"
 - COPYING.txt
@@ -81,20 +82,14 @@ files:
 - lib/ronin/web/spider/git_archive.rb
 - lib/ronin/web/spider/version.rb
 - ronin-web-spider.gemspec
-- spec/agent_spec.rb
-- spec/archive_spec.rb
-- spec/example_app.rb
-- spec/git_archive_spec.rb
-- spec/spec_helper.rb
-- spec/spider_spec.rb
 homepage: https://ronin-rb.dev/
 licenses:
 - LGPL-3.0
 metadata:
-  documentation_uri: https://rubydoc.info/gems/ronin-web-spider
+  documentation_uri: https://ronin-rb.dev/docs/ronin-web-spider
   source_code_uri: https://github.com/ronin-rb/ronin-web-spider
   bug_tracker_uri: https://github.com/ronin-rb/ronin-web-spider/issues
-  changelog_uri: https://github.com/ronin-rb/ronin-web-spider/blob/master/ChangeLog.md
+  changelog_uri: https://github.com/ronin-rb/ronin-web-spider/blob/main/ChangeLog.md
   rubygems_mfa_required: 'true'
 post_install_message:
 rdoc_options: []
@@ -111,12 +106,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.26
+rubygems_version: 3.3.27
 signing_key:
 specification_version: 4
-summary: collection of common web spidering routines
-test_files:
-- spec/agent_spec.rb
-- spec/archive_spec.rb
-- spec/git_archive_spec.rb
-- spec/spider_spec.rb
+summary: A collection of common web spidering routines.
+test_files: []

data/spec/agent_spec.rb

@@ -1,585 +0,0 @@
-require 'spec_helper'
-require 'ronin/web/spider/agent'
-
-require 'webmock/rspec'
-require 'sinatra/base'
-
-describe Ronin::Web::Spider::Agent do
-  describe "#initialize" do
-    context "when Ronin::Support::Network::HTTP.proxy is set" do
-      let(:proxy_host) { 'example.com' }
-      let(:proxy_port) { 8080 }
-      let(:proxy_uri)  { URI::HTTP.build(host: proxy_host, port: proxy_port) }
-
-      before { Ronin::Support::Network::HTTP.proxy = proxy_uri }
-
-      it "must parse ENV['RONIN_HTTP_USER_AGENT'] and set #proxy" do
-        expect(subject.proxy).to be_kind_of(Spidr::Proxy)
-        expect(subject.proxy.host).to eq(proxy_host)
-        expect(subject.proxy.port).to eq(proxy_port)
-      end
-
-      after { Ronin::Support::Network::HTTP.proxy = nil }
-    end
-
-    context "when Ronin::Support::Network::HTTP.user_agent is set" do
-      let(:user_agent) { 'Foo Bar' }
-
-      before { Ronin::Support::Network::HTTP.user_agent = user_agent }
-
-      it "must default #user_agent to ENV['RONIN_HTTP_USER_AGENT']" do
-        expect(subject.user_agent).to eq(user_agent)
-      end
-
-      after { Ronin::Support::Network::HTTP.user_agent = nil }
-    end
-
-    context "when given the proxy: keyword argument" do
-      let(:proxy_host) { 'example.com' }
-      let(:proxy_port) { 8080 }
-
-      context "and it's an Addressable::URI" do
-        let(:proxy) { Addressable::URI.new(host: proxy_host, port: proxy_port) }
-
-        subject { described_class.new(proxy: proxy) }
-
-        it "must convert it to a Spidr::Proxy object" do
-          expect(subject.proxy).to be_kind_of(Spidr::Proxy)
-          expect(subject.proxy.host).to eq(proxy_host)
-          expect(subject.proxy.port).to eq(proxy_port)
-        end
-      end
-
-      context "and it's an URI::HTTP" do
-        let(:proxy) { URI::HTTP.build(host: proxy_host, port: proxy_port) }
-
-        subject { described_class.new(proxy: proxy) }
-
-        it "must convert it to a Spidr::Proxy object" do
-          expect(subject.proxy).to be_kind_of(Spidr::Proxy)
-          expect(subject.proxy.host).to eq(proxy_host)
-          expect(subject.proxy.port).to eq(proxy_port)
-        end
-      end
-
-      context "and it's a Hash" do
-        let(:proxy) do
-          {host: proxy_host, port: proxy_port}
-        end
-
-        subject { described_class.new(proxy: proxy) }
-
-        it "must convert it to a Spidr::Proxy object" do
-          expect(subject.proxy).to be_kind_of(Spidr::Proxy)
-          expect(subject.proxy.host).to eq(proxy_host)
-          expect(subject.proxy.port).to eq(proxy_port)
-        end
-      end
-
-      context "and it's a String" do
-        let(:proxy) { "http://#{proxy_host}:#{proxy_port}" }
-
-        subject { described_class.new(proxy: proxy) }
-
-        it "must convert it to a Spidr::Proxy object" do
-          expect(subject.proxy).to be_kind_of(Spidr::Proxy)
-          expect(subject.proxy.host).to eq(proxy_host)
-          expect(subject.proxy.port).to eq(proxy_port)
-        end
-      end
-    end
-
-    context "when given the user_agent: keyword argument" do
-      context "and it's a String" do
-        let(:user_agent) { "test user-agent" }
-
-        subject { described_class.new(user_agent: user_agent) }
-
-        it "must set the #user_agent" do
-          expect(subject.user_agent).to eq(user_agent)
-        end
-      end
-
-      context "and it's a Symbol" do
-        let(:user_agent) { :chrome_linux }
-        let(:expected_user_agent) do
-          Ronin::Support::Network::HTTP::UserAgents[user_agent]
-        end
-
-        subject { described_class.new(user_agent: user_agent) }
-
-        it "must map the Symbol to one of Ronin::Support::Network::HTTP::UserAgents" do
-          expect(subject.user_agent).to eq(expected_user_agent)
-        end
-      end
-    end
-
-    it "must default #visited_hosts to nil" do
-      expect(subject.visited_hosts).to be(nil)
-    end
-  end
-
-  describe "#every_host" do
-    module TestAgentEveryHost
-      class Host1 < Sinatra::Base
-
-        set :host, 'host1.example.com'
-        set :port, 80
-
-        get '/' do
-          <<~HTML
-          <html>
-            <body>
-              <a href="/link1">link1</a>
-              <a href="http://host2.example.com/offsite-link">offsite link</a>
-              <a href="/link2">link2</a>
-            </body>
-          </html>
-          HTML
-        end
-
-        get '/link1' do
-          '<html><body>got here</body></html>'
-        end
-
-        get '/link2' do
-          '<html><body>got here</body></html>'
-        end
-      end
-
-      class Host2 < Sinatra::Base
-
-        set :host, 'host2.example.com'
-        set :port, 80
-
-        get '/offsite-link' do
-          '<html><body>should not get here</body></html>'
-        end
-
-      end
-    end
-
-    let(:host1) { 'host1.example.com' }
-    let(:host2) { 'host2.example.com'   }
-
-    let(:host1_app) { TestAgentEveryHost::Host1 }
-    let(:host2_app) { TestAgentEveryHost::Host2 }
-
-    before do
-      stub_request(:any, /#{Regexp.escape(host1)}/).to_rack(host1_app)
-      stub_request(:any, /#{Regexp.escape(host2)}/).to_rack(host2_app)
-    end
-
-    it "must yield every newly discovered hostname while spidering" do
-      yielded_hosts = []
-
-      subject.every_host do |host|
-        yielded_hosts << host
-      end
-
-      subject.start_at("http://#{host1}/")
-
-      expect(yielded_hosts).to eq([host1, host2])
-    end
-
-    it "must popualte #visited_hosts" do
-      subject.every_host { |host| }
-      subject.start_at("http://#{host1}/")
-
-      expect(subject.visited_hosts).to be_kind_of(Set)
-      expect(subject.visited_hosts.entries).to eq([host1, host2])
-    end
-  end
-
-  # TODO: need to figure out how to test #every_cert using webmock.
-  describe "#every_cert"
-  
-  describe "#every_favicon" do
-    module TestAgentEveryHost
-      class TestApp < Sinatra::Base
-
-        set :host, 'example.com'
-        set :port, 80
-
-        get '/' do
-          <<~HTML
-          <html>
-            <head>
-              <link rel="favicon" href="/favicon1.ico" type="image/x-icon"/>
-            </head>
-            <body>
-              <a href="/link1">link1</a>
-              <a href="http://host2.example.com/offsite-link">offsite link</a>
-              <a href="/link2">link2</a>
-            </body>
-          </html>
-          HTML
-        end
-
-        get '/favicon1.ico' do
-          content_type 'image/x-icon'
-
-          "favicon1"
-        end
-
-        get '/favicon2.ico' do
-          content_type 'image/vnd.microsoft.icon'
-
-          "favicon2"
-        end
-
-        get '/link1' do
-          '<html><body>got here</body></html>'
-        end
-
-        get '/link2' do
-          <<~HTML
-          <html>
-            <head>
-              <link rel="favicon" href="/favicon2.ico" type="image/x-icon"/>
-            </head>
-            <body>got here</body>
-          </html>
-          HTML
-        end
-      end
-    end
-
-    let(:host) { 'example.com' }
-
-    let(:test_app) { TestAgentEveryHost::TestApp }
-
-    before do
-      stub_request(:any, /#{Regexp.escape(host)}/).to_rack(test_app)
-    end
-
-    it "must yield Spidr::Page objects for each encountered .ico file" do
-      yielded_favicons = []
-
-      subject.every_favicon do |favicon|
-        yielded_favicons << favicon
-      end
-
-      subject.start_at("http://#{host}/")
-
-      expect(yielded_favicons).to_not be_empty
-
-      expect(yielded_favicons[0]).to be_kind_of(Spidr::Page)
-      expect(yielded_favicons[0].content_type).to eq('image/x-icon')
-      expect(yielded_favicons[0].url).to eq(URI("http://#{host}/favicon1.ico"))
-
-      expect(yielded_favicons[1]).to be_kind_of(Spidr::Page)
-      expect(yielded_favicons[1].content_type).to eq('image/vnd.microsoft.icon')
-      expect(yielded_favicons[1].url).to eq(URI("http://#{host}/favicon2.ico"))
-    end
-  end
-
-  describe "#every_html_comment" do
-    module TestAgentEveryHTMLComment
-      class TestApp < Sinatra::Base
-
-        set :host, 'example.com'
-        set :port, 80
-
-        get '/' do
-          <<~HTML
-          <html>
-            <head>
-              <!-- comment 1 -->
-            </head>
-            <!-- -->
-            <body>
-              <!-- comment 2 -->
-            </body>
-          </html>
-          HTML
-        end
-      end
-    end
-
-    let(:host) { 'example.com' }
-
-    let(:test_app) { TestAgentEveryHTMLComment::TestApp }
-
-    before do
-      stub_request(:any, /#{Regexp.escape(host)}/).to_rack(test_app)
-    end
-
-    it "must yield every non-empty/non-whitespace HTML comment String" do
-      yielded_comments = []
-
-      subject.every_html_comment do |comment|
-        yielded_comments << comment
-      end
-
-      subject.start_at("http://#{host}/")
-
-      expect(yielded_comments).to match_array(
-        [
-          'comment 1',
-          'comment 2'
-        ]
-      )
-    end
-  end
-
-  describe "#every_javascript" do
-    module TestAgentEveryJavaScript
-      class TestApp < Sinatra::Base
-
-        set :host, 'example.com'
-        set :port, 80
-
-        get '/' do
-          <<~HTML
-          <html>
-            <head>
-              <script type="text/javascript" src="/javascript1.js"></script>
-              <script type="text/javascript">javascript2</script>
-            </head>
-            <body>
-              <a href="/link1">link1</a>
-              <a href="http://host2.example.com/offsite-link">offsite link</a>
-              <a href="/link2">link2</a>
-            </body>
-          </html>
-          HTML
-        end
-
-        get '/javascript1.js' do
-          content_type 'text/javascript'
-          "javascript1"
-        end
-      end
-    end
-
-    let(:host) { 'example.com' }
-
-    let(:test_app) { TestAgentEveryJavaScript::TestApp }
-
-    before do
-      stub_request(:any, /#{Regexp.escape(host)}/).to_rack(test_app)
-    end
-
-    it "must yield both the contents of .js files and inline <script> tags" do
-      yielded_javascripts = []
-
-      subject.every_javascript do |js|
-        yielded_javascripts << js
-      end
-
-      subject.start_at("http://#{host}/")
-
-      expect(yielded_javascripts).to match_array(%w[javascript1 javascript2])
-    end
-  end
-
-  describe "#every_javascript_string" do
-    module TestAgentEveryJavaScriptString
-      class TestApp < Sinatra::Base
-
-        set :host, 'example.com'
-        set :port, 80
-
-        get '/' do
-          <<~HTML
-          <html>
-            <head>
-              <script type="text/javascript" src="/javascript1.js"></script>
-              <script type="text/javascript">
-              var str3 = "string #3";
-              var str4 = 'string #4';
-              </script>
-            </head>
-            <body>
-              <a href="/link1">link1</a>
-              <a href="http://host2.example.com/offsite-link">offsite link</a>
-              <a href="/link2">link2</a>
-            </body>
-          </html>
-          HTML
-        end
-
-        get '/javascript1.js' do
-          content_type 'text/javascript'
-          <<~JS
-          var str1 = "string #1";
-          var str2 = 'string #2';
-          JS
-        end
-      end
-    end
-
-    let(:host) { 'example.com' }
-
-    let(:test_app) { TestAgentEveryJavaScriptString::TestApp }
-
-    before do
-      stub_request(:any, /#{Regexp.escape(host)}/).to_rack(test_app)
-    end
-
-    it "must yield every JavaScript string from any <script> tag" do
-      yielded_javascript_strings = []
-
-      subject.every_javascript_string do |string|
-        yielded_javascript_strings << string
-      end
-
-      subject.start_at("http://#{host}/")
-
-      expect(yielded_javascript_strings).to match_array(
-        [
-          'string #1',
-          'string #2',
-          'string #3',
-          'string #4'
-        ]
-      )
-    end
-  end
-
-  describe "#every_javascript_comment" do
-    module TestAgentEveryJavaScriptComment
-      class TestApp < Sinatra::Base
-
-        set :host, 'example.com'
-        set :port, 80
-
-        get '/' do
-          <<~HTML
-          <html>
-            <head>
-              <script type="text/javascript" src="/javascript1.js"></script>
-              <script type="text/javascript">
-              // comment 3
-              var str3 = "string #3";
-              /*
-                 comment 4
-               */
-              var str4 = 'string #4';
-              </script>
-            </head>
-            <body>
-              <a href="/link1">link1</a>
-              <a href="http://host2.example.com/offsite-link">offsite link</a>
-              <a href="/link2">link2</a>
-            </body>
-          </html>
-          HTML
-        end
-
-        get '/javascript1.js' do
-          content_type 'text/javascript'
-          <<~JS
-          // comment 1
-          var str1 = "string #1";
-          /* comment 2 */
-          var str2 = 'string #2';
-          JS
-        end
-      end
-    end
-
-    let(:host) { 'example.com' }
-
-    let(:test_app) { TestAgentEveryJavaScriptComment::TestApp }
-
-    before do
-      stub_request(:any, /#{Regexp.escape(host)}/).to_rack(test_app)
-    end
-
-    it "must yield every JavaScript comment from any <script> tag" do
-      yielded_javascript_comments = []
-
-      subject.every_javascript_comment do |comment|
-        yielded_javascript_comments << comment
-      end
-
-      subject.start_at("http://#{host}/")
-
-      expect(yielded_javascript_comments).to match_array(
-        [
-          "// comment 1\n",
-          "/* comment 2 */",
-          "// comment 3\n",
-          "/*\n       comment 4\n     */"
-        ]
-      )
-    end
-  end
-
-  describe "#every_comment" do
-    module TestAgentEveryComment
-      class TestApp < Sinatra::Base
-
-        set :host, 'example.com'
-        set :port, 80
-
-        get '/' do
-          <<~HTML
-          <html>
-            <head>
-              <!-- HTML comment 1 -->
-              <script type="text/javascript" src="/javascript1.js"></script>
-              <script type="text/javascript">
-              // JavaScript comment 3
-              var str3 = "string #3";
-              /*
-                 JavaScript comment 4
-               */
-              var str4 = 'string #4';
-              </script>
-            </head>
-            <!-- -->
-            <body>
-              <!-- HTML comment 2 -->
-              <a href="/link1">link1</a>
-              <a href="http://host2.example.com/offsite-link">offsite link</a>
-              <a href="/link2">link2</a>
-            </body>
-          </html>
-          HTML
-        end
-
-        get '/javascript1.js' do
-          content_type 'text/javascript'
-          <<~JS
-          // JavaScript comment 1
-          var str1 = "string #1";
-          /* JavaScript comment 2 */
-          var str2 = 'string #2';
-          JS
-        end
-      end
-    end
-
-    let(:host) { 'example.com' }
-
-    let(:test_app) { TestAgentEveryComment::TestApp }
-
-    before do
-      stub_request(:any, /#{Regexp.escape(host)}/).to_rack(test_app)
-    end
-
-    it "must yield every HTML and JavaScript comment from any <script> tag" do
-      yielded_comments = []
-
-      subject.every_comment do |comment|
-        yielded_comments << comment
-      end
-
-      subject.start_at("http://#{host}/")
-
-      expect(yielded_comments).to match_array(
-        [
-          "HTML comment 1",
-          "// JavaScript comment 1\n",
-          "/* JavaScript comment 2 */",
-          "// JavaScript comment 3\n",
-          "/*\n       JavaScript comment 4\n     */",
-          "HTML comment 2"
-        ]
-      )
-    end
-  end
-end