RubyGems - spidr - Versions diffs - 0.6.0 → 0.7.0 - Mend

spidr 0.6.0 → 0.7.0

Files changed (35) hide show

checksums.yaml +5 -5
data/.editorconfig +11 -0
data/.github/workflows/ruby.yml +26 -0
data/.gitignore +4 -5
data/ChangeLog.md +17 -0
data/Gemfile +8 -5
data/LICENSE.txt +1 -1
data/README.md +137 -78
data/Rakefile +1 -0
data/gemspec.yml +8 -1
data/lib/spidr/agent/actions.rb +1 -1
data/lib/spidr/agent/events.rb +1 -1
data/lib/spidr/agent/filters.rb +55 -56
data/lib/spidr/agent/sanitizers.rb +6 -9
data/lib/spidr/agent.rb +230 -120
data/lib/spidr/auth_store.rb +10 -6
data/lib/spidr/page/content_types.rb +51 -0
data/lib/spidr/page/html.rb +17 -19
data/lib/spidr/page/status_codes.rb +12 -10
data/lib/spidr/proxy.rb +6 -14
data/lib/spidr/rules.rb +5 -8
data/lib/spidr/session_cache.rb +23 -21
data/lib/spidr/settings/proxy.rb +19 -5
data/lib/spidr/spidr.rb +16 -6
data/lib/spidr/version.rb +1 -1
data/spec/agent_spec.rb +357 -10
data/spec/example_page.rb +2 -0
data/spec/page/content_types_spec.rb +22 -0
data/spec/page/html_spec.rb +255 -51
data/spec/page/status_codes_spec.rb +4 -4
data/spec/proxy_spec.rb +2 -2
data/spec/settings/proxy_examples.rb +31 -11
data/spec/spec_helper.rb +3 -0
metadata +19 -19
data/.travis.yml +0 -14

data/spec/page/html_spec.rb CHANGED Viewed

@@ -297,7 +297,7 @@ describe Page do
     context "when the page contains iframes" do
       let(:iframe1) { '/iframe1' }
       let(:iframe2) { '/iframe2' }
-      let(:body)   { %{<html><body><iframe src="#{iframe1}" /><iframe src="#{iframe2}" /></body></html>} }
+      let(:body)   { %{<html><body><iframe src="#{iframe1}"></iframe><iframe src="#{iframe2}"></iframe></body></html>} }
       it "should yield each iframe/@src value" do
         expect { |b|
@@ -332,32 +332,100 @@ describe Page do
   end
   describe "#links" do
-    context "when the page contains links" do
+    context "when the page contains an 'a' link" do
       let(:link) { '/link' }
+      let(:body) do
+        <<-HTML
+        <html>
+          <body>
+            <a href="#{link}">link</a>
+          </body>
+        </html>
+        HTML
+      end
+      it "should return an Array of links" do
+        expect(subject.links).to be == [
+          link
+        ]
+      end
+    end
+    context "when the page contains a 'frame'" do
       let(:frame) { '/frame' }
+      let(:body) do
+        <<-HTML
+        <html>
+          <frameset>
+            <frame src="#{frame}"></frame>
+          </frameset>
+        </html>
+        HTML
+      end
+      it "should return an Array of links" do
+        expect(subject.links).to be == [
+          frame
+        ]
+      end
+    end
+    context "when the page contains a 'iframe'" do
       let(:iframe) { '/iframe' }
+      let(:body) do
+        <<-HTML
+        <html>
+          <body>
+            <iframe src="#{iframe}"></iframe>
+          </body>
+        </html>
+        HTML
+      end
+      it "should return an Array of links" do
+        expect(subject.links).to be == [
+          iframe
+        ]
+      end
+    end
+    context "when the page contains a 'link' element" do
       let(:stylesheet) { '/stylesheet.css' }
+      let(:body) do
+        <<-HTML
+        <html>
+          <head>
+            <link type="stylesheet" href="#{stylesheet}" />
+          </head>
+          <body>
+          </body>
+        </html>
+        HTML
+      end
+      it "should return an Array of links" do
+        expect(subject.links).to be == [
+          stylesheet
+        ]
+      end
+    end
+    context "when the page contains a 'script' element" do
       let(:javascript) { '/script.js' }
       let(:body) do
-        %{<html>} +
-          %{<head>} +
-            %{<link rel="stylesheet" type="text/css" href="#{stylesheet}" />} +
-            %{<script type="text/javascript" src="#{javascript}"></script>} +
-          %{</head>} +
-          %{<body>} +
-            %{<a href="#{link}">link</a>} +
-            %{<frameset><frame src="#{frame}" /></frameset>} +
-            %{<iframe src="#{iframe}" />} +
-          %{</body>} +
-        %{</html>}
+        <<-HTML
+        <html>
+          <head>
+            <script src="#{javascript}" />
+          </head>
+          <body>
+          </body>
+        </html>
+        HTML
       end
       it "should return an Array of links" do
         expect(subject.links).to be == [
-          link,
-          frame,
-          iframe,
-          stylesheet,
           javascript
         ]
       end
@@ -369,32 +437,100 @@ describe Page do
   end
   describe "#each_url" do
-    context "when the page contains links" do
+    context "when the page contains an 'a' link" do
       let(:link) { '/link' }
+      let(:body) do
+        <<-HTML
+        <html>
+          <body>
+            <a href="#{link}">link</a>
+          </body>
+        </html>
+        HTML
+      end
+      it "should yield successive absolute URIs" do
+        expect { |b| subject.each_url(&b) }.to yield_successive_args(
+          URI("http://#{host}#{link}")
+        )
+      end
+    end
+    context "when the page contains a 'frame'" do
       let(:frame) { '/frame' }
+      let(:body) do
+        <<-HTML
+        <html>
+          <frameset>
+            <frame src="#{frame}"></frame>
+          </frameset>
+        </html>
+        HTML
+      end
+      it "should yield successive absolute URIs" do
+        expect { |b| subject.each_url(&b) }.to yield_successive_args(
+          URI("http://#{host}#{frame}")
+        )
+      end
+    end
+    context "when the page contains a 'iframe'" do
       let(:iframe) { '/iframe' }
+      let(:body) do
+        <<-HTML
+        <html>
+          <body>
+            <iframe src="#{iframe}"></iframe>
+          </body>
+        </html>
+        HTML
+      end
+      it "should yield successive absolute URIs" do
+        expect { |b| subject.each_url(&b) }.to yield_successive_args(
+          URI("http://#{host}#{iframe}")
+        )
+      end
+    end
+    context "when the page contains a 'link' element" do
       let(:stylesheet) { '/stylesheet.css' }
-      let(:javascript) { '/script.js' }
       let(:body) do
-        %{<html>} +
-          %{<head>} +
-            %{<link rel="stylesheet" type="text/css" href="#{stylesheet}" />} +
-            %{<script type="text/javascript" src="#{javascript}"></script>} +
-          %{</head>} +
-          %{<body>} +
-            %{<a href="#{link}">link</a>} +
-            %{<frameset><frame src="#{frame}" /></frameset>} +
-            %{<iframe src="#{iframe}" />} +
-          %{</body>} +
-        %{</html>}
+        <<-HTML
+        <html>
+          <head>
+            <link type="stylesheet" href="#{stylesheet}" />
+          </head>
+          <body>
+          </body>
+        </html>
+        HTML
+      end
+      it "should yield successive absolute URIs" do
+        expect { |b| subject.each_url(&b) }.to yield_successive_args(
+          URI("http://#{host}#{stylesheet}")
+        )
       end
+    end
-      it "should return an Array of absolute URIs" do
+    context "when the page contains a 'script' element" do
+      let(:javascript) { '/script.js' }
+      let(:body) do
+        <<-HTML
+        <html>
+          <head>
+            <script src="#{javascript}" />
+          </head>
+          <body>
+          </body>
+        </html>
+        HTML
+      end
+      it "should yield successive absolute URIs" do
         expect { |b| subject.each_url(&b) }.to yield_successive_args(
-          URI("http://#{host}#{link}"),
-          URI("http://#{host}#{frame}"),
-          URI("http://#{host}#{iframe}"),
-          URI("http://#{host}#{stylesheet}"),
           URI("http://#{host}#{javascript}")
         )
       end
@@ -410,32 +546,100 @@ describe Page do
   end
   describe "#urls" do
-    context "when the page contains links" do
+    context "when the page contains an 'a' link" do
       let(:link) { '/link' }
+      let(:body) do
+        <<-HTML
+        <html>
+          <body>
+            <a href="#{link}">link</a>
+          </body>
+        </html>
+        HTML
+      end
+      it "should return an Array of absolute URIs" do
+        expect(subject.urls).to be == [
+          URI("http://#{host}#{link}")
+        ]
+      end
+    end
+    context "when the page contains a 'frame'" do
       let(:frame) { '/frame' }
+      let(:body) do
+        <<-HTML
+        <html>
+          <frameset>
+            <frame src="#{frame}"></frame>
+          </frameset>
+        </html>
+        HTML
+      end
+      it "should return an Array of absolute URIs" do
+        expect(subject.urls).to be == [
+          URI("http://#{host}#{frame}")
+        ]
+      end
+    end
+    context "when the page contains a 'iframe'" do
       let(:iframe) { '/iframe' }
+      let(:body) do
+        <<-HTML
+        <html>
+          <body>
+            <iframe src="#{iframe}"></iframe>
+          </body>
+        </html>
+        HTML
+      end
+      it "should return an Array of absolute URIs" do
+        expect(subject.urls).to be == [
+          URI("http://#{host}#{iframe}")
+        ]
+      end
+    end
+    context "when the page contains a 'link' element" do
       let(:stylesheet) { '/stylesheet.css' }
+      let(:body) do
+        <<-HTML
+        <html>
+          <head>
+            <link type="stylesheet" href="#{stylesheet}" />
+          </head>
+          <body>
+          </body>
+        </html>
+        HTML
+      end
+      it "should return an Array of absolute URIs" do
+        expect(subject.urls).to be == [
+          URI("http://#{host}#{stylesheet}")
+        ]
+      end
+    end
+    context "when the page contains a 'script' element" do
       let(:javascript) { '/script.js' }
       let(:body) do
-        %{<html>} +
-          %{<head>} +
-            %{<link rel="stylesheet" type="text/css" href="#{stylesheet}" />} +
-            %{<script type="text/javascript" src="#{javascript}"></script>} +
-          %{</head>} +
-          %{<body>} +
-            %{<a href="#{link}">link</a>} +
-            %{<frameset><frame src="#{frame}" /></frameset>} +
-            %{<iframe src="#{iframe}" />} +
-          %{</body>} +
-        %{</html>}
+        <<-HTML
+        <html>
+          <head>
+            <script src="#{javascript}" />
+          </head>
+          <body>
+          </body>
+        </html>
+        HTML
       end
       it "should return an Array of absolute URIs" do
         expect(subject.urls).to be == [
-          URI("http://#{host}#{link}"),
-          URI("http://#{host}#{frame}"),
-          URI("http://#{host}#{iframe}"),
-          URI("http://#{host}#{stylesheet}"),
           URI("http://#{host}#{javascript}")
         ]
       end

data/spec/page/status_codes_spec.rb CHANGED Viewed

@@ -26,10 +26,6 @@ describe Page do
     include_examples "status code method", :is_ok?, {200 => true, 500 => false}
   end
-  describe "#timedout?" do
-    include_examples "status code method", :timedout?, {308 => true, 200 => false}
-  end
   describe "#bad_request?" do
     include_examples "status code method", :bad_request?, {400 => true, 200 => false}
   end
@@ -46,6 +42,10 @@ describe Page do
     include_examples "status code method", :is_missing?, {404 => true, 200 => false}
   end
+  describe "#is_timedout?" do
+    include_examples "status code method", :is_timedout?, {408 => true, 200 => false}
+  end
   describe "#had_internal_server_error?" do
     include_examples "status code method", :had_internal_server_error?, {500 => true, 200 => false}
   end

data/spec/proxy_spec.rb CHANGED Viewed

@@ -26,13 +26,13 @@ describe Spidr::Proxy do
       it { expect(subject.enabled?).to be true }
     end
-    context "when hist is not set" do
+    context "when host is not set" do
       it { expect(subject.enabled?).to be false }
     end
   end
   describe "#disabled?" do
-    context "when hist is not set" do
+    context "when host is not set" do
       it { expect(subject.disabled?).to be true }
     end

data/spec/settings/proxy_examples.rb CHANGED Viewed

@@ -16,7 +16,7 @@ shared_examples "includes Spidr::Settings::Proxy" do
       end
       it "should retain the default value" do
-        expect(subject.proxy.object_id).to be subject.proxy.object_id
+        expect(subject.proxy.object_id).to be(subject.proxy.object_id)
       end
     end
@@ -26,7 +26,7 @@ shared_examples "includes Spidr::Settings::Proxy" do
       end
       it "should return the set @proxy" do
-        expect(subject.proxy).to be proxy
+        expect(subject.proxy).to be(proxy)
       end
     end
   end
@@ -35,12 +35,10 @@ shared_examples "includes Spidr::Settings::Proxy" do
     context "when given a Proxy object" do
       let(:proxy) { Proxy.new(host: proxy_host, port: proxy_port) }
-      before do
-        subject.proxy = proxy
-      end
+      before { subject.proxy = proxy }
       it "should save it" do
-        expect(subject.proxy).to be proxy
+        expect(subject.proxy).to be(proxy)
       end
     end
@@ -51,15 +49,37 @@ shared_examples "includes Spidr::Settings::Proxy" do
       it "should create a new Proxy object" do
         expect(subject.proxy).to be_kind_of(Proxy)
-        expect(subject.proxy[:host]).to be proxy_host
-        expect(subject.proxy[:port]).to be proxy_port
+        expect(subject.proxy[:host]).to be(proxy_host)
+        expect(subject.proxy[:port]).to be(proxy_port)
       end
     end
-    context "when given nil" do
-      before do
-        subject.proxy = nil
+    context "when given a URI::HTTP" do
+      let(:uri) { URI::HTTP.build(host: proxy_host, port: proxy_port) }
+      before { subject.proxy = uri }
+      it "should create a new Proxy object based on the URI" do
+        expect(subject.proxy).to be_kind_of(Proxy)
+        expect(subject.proxy[:host]).to eq(proxy_host)
+        expect(subject.proxy[:port]).to eq(proxy_port)
+      end
+    end
+    context "when given a String" do
+      let(:url) { "http://#{proxy_host}:#{proxy_port}" }
+      before { subject.proxy = url }
+      it "should parse the String as a URI and create a new Proxy object" do
+        expect(subject.proxy).to be_kind_of(Proxy)
+        expect(subject.proxy[:host]).to eq(proxy_host)
+        expect(subject.proxy[:port]).to eq(proxy_port)
       end
+    end
+    context "when given nil" do
+      before { subject.proxy = nil }
       it "should leave an empty proxy" do
         expect(subject.proxy).to be_kind_of(Proxy)

data/spec/spec_helper.rb CHANGED Viewed

@@ -1,4 +1,7 @@
 require 'rspec'
+require 'simplecov'
 require 'spidr/version'
 include Spidr
+SimpleCov.start

metadata CHANGED Viewed

@@ -1,43 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: spidr
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.7.0
 platform: ruby
 authors:
 - Postmodern
-autorequire:
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-08-05 00:00:00.000000000 Z
+date: 2023-01-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '2.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '2.0'
 description: Spidr is a versatile Ruby web spidering library that can spider a site,
   multiple domains, certain links or infinitely. Spidr is designed to be fast and
   easy to use.
@@ -49,10 +49,11 @@ extra_rdoc_files:
 - LICENSE.txt
 - README.md
 files:
-- .gitignore
-- .rspec
-- .travis.yml
-- .yardopts
+- ".editorconfig"
+- ".github/workflows/ruby.yml"
+- ".gitignore"
+- ".rspec"
+- ".yardopts"
 - ChangeLog.md
 - Gemfile
 - LICENSE.txt
@@ -112,24 +113,23 @@ homepage: https://github.com/postmodern/spidr#readme
 licenses:
 - MIT
 metadata: {}
-post_install_message:
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: 2.0.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.0.14.1
-signing_key:
+rubygems_version: 3.3.26
+signing_key:
 specification_version: 4
 summary: A versatile Ruby web spidering library
 test_files: []

data/.travis.yml DELETED Viewed

@@ -1,14 +0,0 @@
----
-language: ruby
-rvm:
-  - 2.0.0
-  - 2.1.9
-  - 2.2.4
-  - 2.3.1
-  - jruby
-  - rbx
-matrix:
-  allow_failures:
-    - rvm: jruby
-    - rvm: rbx
-script: rake spec