RubyGems - spidr - Versions diffs - 0.6.0 → 0.7.0 - Mend

spidr 0.6.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (35) hide show

checksums.yaml +5 -5
data/.editorconfig +11 -0
data/.github/workflows/ruby.yml +26 -0
data/.gitignore +4 -5
data/ChangeLog.md +17 -0
data/Gemfile +8 -5
data/LICENSE.txt +1 -1
data/README.md +137 -78
data/Rakefile +1 -0
data/gemspec.yml +8 -1
data/lib/spidr/agent/actions.rb +1 -1
data/lib/spidr/agent/events.rb +1 -1
data/lib/spidr/agent/filters.rb +55 -56
data/lib/spidr/agent/sanitizers.rb +6 -9
data/lib/spidr/agent.rb +230 -120
data/lib/spidr/auth_store.rb +10 -6
data/lib/spidr/page/content_types.rb +51 -0
data/lib/spidr/page/html.rb +17 -19
data/lib/spidr/page/status_codes.rb +12 -10
data/lib/spidr/proxy.rb +6 -14
data/lib/spidr/rules.rb +5 -8
data/lib/spidr/session_cache.rb +23 -21
data/lib/spidr/settings/proxy.rb +19 -5
data/lib/spidr/spidr.rb +16 -6
data/lib/spidr/version.rb +1 -1
data/spec/agent_spec.rb +357 -10
data/spec/example_page.rb +2 -0
data/spec/page/content_types_spec.rb +22 -0
data/spec/page/html_spec.rb +255 -51
data/spec/page/status_codes_spec.rb +4 -4
data/spec/proxy_spec.rb +2 -2
data/spec/settings/proxy_examples.rb +31 -11
data/spec/spec_helper.rb +3 -0
metadata +19 -19
data/.travis.yml +0 -14

data/spec/page/html_spec.rb CHANGED Viewed

@@ -297,7 +297,7 @@ describe Page do
     context "when the page contains iframes" do
       let(:iframe1) { '/iframe1' }
       let(:iframe2) { '/iframe2' }
-      let(:body)   { %{<html><body><iframe src="#{iframe1}" /><iframe src="#{iframe2}" /></body></html>} }
+      let(:body)   { %{<html><body><iframe src="#{iframe1}"></iframe><iframe src="#{iframe2}"></iframe></body></html>} }
       it "should yield each iframe/@src value" do
         expect { |b|
@@ -332,32 +332,100 @@ describe Page do
   end
   describe "#links" do
-    context "when the page contains links" do
+    context "when the page contains an 'a' link" do
       let(:link) { '/link' }
+      let(:body) do
+        <<-HTML
+        <html>
+          <body>
+            <a href="#{link}">link</a>
+          </body>
+        </html>
+        HTML
+      end
+      it "should return an Array of links" do
+        expect(subject.links).to be == [
+          link
+        ]
+      end
+    end
+    context "when the page contains a 'frame'" do
       let(:frame) { '/frame' }
+      let(:body) do
+        <<-HTML
+        <html>
+          <frameset>
+            <frame src="#{frame}"></frame>
+          </frameset>
+        </html>
+        HTML
+      end
+      it "should return an Array of links" do
+        expect(subject.links).to be == [
+          frame
+        ]
+      end
+    end
+    context "when the page contains a 'iframe'" do
       let(:iframe) { '/iframe' }
+      let(:body) do
+        <<-HTML
+        <html>
+          <body>
+            <iframe src="#{iframe}"></iframe>
+          </body>
+        </html>
+        HTML
+      end
+      it "should return an Array of links" do
+        expect(subject.links).to be == [
+          iframe
+        ]
+      end
+    end
+    context "when the page contains a 'link' element" do
       let(:stylesheet) { '/stylesheet.css' }
+      let(:body) do
+        <<-HTML
+        <html>
+          <head>
+            <link type="stylesheet" href="#{stylesheet}" />
+          </head>
+          <body>
+          </body>
+        </html>
+        HTML
+      end
+      it "should return an Array of links" do
+        expect(subject.links).to be == [
+          stylesheet
+        ]
+      end
+    end
+    context "when the page contains a 'script' element" do
       let(:javascript) { '/script.js' }
       let(:body) do
-        %{<html>} +
-          %{<head>} +
-            %{<link rel="stylesheet" type="text/css" href="#{stylesheet}" />} +
-            %{<script type="text/javascript" src="#{javascript}"></script>} +
-          %{</head>} +
-          %{<body>} +
-            %{<a href="#{link}">link</a>} +
-            %{<frameset><frame src="#{frame}" /></frameset>} +
-            %{<iframe src="#{iframe}" />} +
-          %{</body>} +
-        %{</html>}
+        <<-HTML
+        <html>
+          <head>
+            <script src="#{javascript}" />
+          </head>
+          <body>
+          </body>
+        </html>
+        HTML
       end
       it "should return an Array of links" do
         expect(subject.links).to be == [
-          link,
-          frame,
-          iframe,
-          stylesheet,
           javascript
         ]
       end
@@ -369,32 +437,100 @@ describe Page do
   end
   describe "#each_url" do
-    context "when the page contains links" do
+    context "when the page contains an 'a' link" do
       let(:link) { '/link' }
+      let(:body) do
+        <<-HTML
+        <html>
+          <body>
+            <a href="#{link}">link</a>
+          </body>
+        </html>
+        HTML
+      end
+      it "should yield successive absolute URIs" do
+        expect { |b| subject.each_url(&b) }.to yield_successive_args(
+          URI("http://#{host}#{link}")
+        )
+      end
+    end
+    context "when the page contains a 'frame'" do
       let(:frame) { '/frame' }
+      let(:body) do
+        <<-HTML
+        <html>
+          <frameset>
+            <frame src="#{frame}"></frame>
+          </frameset>
+        </html>
+        HTML
+      end
+      it "should yield successive absolute URIs" do
+        expect { |b| subject.each_url(&b) }.to yield_successive_args(
+          URI("http://#{host}#{frame}")
+        )
+      end
+    end
+    context "when the page contains a 'iframe'" do
       let(:iframe) { '/iframe' }
+      let(:body) do
+        <<-HTML
+        <html>
+          <body>
+            <iframe src="#{iframe}"></iframe>
+          </body>
+        </html>
+        HTML
+      end
+      it "should yield successive absolute URIs" do
+        expect { |b| subject.each_url(&b) }.to yield_successive_args(
+          URI("http://#{host}#{iframe}")
+        )
+      end
+    end
+    context "when the page contains a 'link' element" do
       let(:stylesheet) { '/stylesheet.css' }
-      let(:javascript) { '/script.js' }
       let(:body) do
-        %{<html>} +
-          %{<head>} +
-            %{<link rel="stylesheet" type="text/css" href="#{stylesheet}" />} +
-            %{<script type="text/javascript" src="#{javascript}"></script>} +
-          %{</head>} +
-          %{<body>} +
-            %{<a href="#{link}">link</a>} +
-            %{<frameset><frame src="#{frame}" /></frameset>} +
-            %{<iframe src="#{iframe}" />} +
-          %{</body>} +
-        %{</html>}
+        <<-HTML
+        <html>
+          <head>
+            <link type="stylesheet" href="#{stylesheet}" />
+          </head>
+          <body>
+          </body>
+        </html>
+        HTML
+      end
+      it "should yield successive absolute URIs" do
+        expect { |b| subject.each_url(&b) }.to yield_successive_args(
+          URI("http://#{host}#{stylesheet}")
+        )
       end
+    end
-      it "should return an Array of absolute URIs" do
+    context "when the page contains a 'script' element" do
+      let(:javascript) { '/script.js' }
+      let(:body) do
+        <<-HTML
+        <html>
+          <head>
+            <script src="#{javascript}" />
+          </head>
+          <body>
+          </body>
+        </html>
+        HTML
+      end
+      it "should yield successive absolute URIs" do
         expect { |b| subject.each_url(&b) }.to yield_successive_args(
-          URI("http://#{host}#{link}"),
-          URI("http://#{host}#{frame}"),
-          URI("http://#{host}#{iframe}"),
-          URI("http://#{host}#{stylesheet}"),
           URI("http://#{host}#{javascript}")
         )
       end
@@ -410,32 +546,100 @@ describe Page do
   end
   describe "#urls" do
-    context "when the page contains links" do
+    context "when the page contains an 'a' link" do
       let(:link) { '/link' }
+      let(:body) do
+        <<-HTML
+        <html>
+          <body>
+            <a href="#{link}">link</a>
+          </body>
+        </html>
+        HTML
+      end
+      it "should return an Array of absolute URIs" do
+        expect(subject.urls).to be == [
+          URI("http://#{host}#{link}")
+        ]
+      end
+    end
+    context "when the page contains a 'frame'" do
       let(:frame) { '/frame' }
+      let(:body) do
+        <<-HTML
+        <html>
+          <frameset>
+            <frame src="#{frame}"></frame>
+          </frameset>
+        </html>
+        HTML
+      end
+      it "should return an Array of absolute URIs" do
+        expect(subject.urls).to be == [
+          URI("http://#{host}#{frame}")
+        ]
+      end
+    end
+    context "when the page contains a 'iframe'" do
       let(:iframe) { '/iframe' }
+      let(:body) do
+        <<-HTML
+        <html>
+          <body>
+            <iframe src="#{iframe}"></iframe>
+          </body>
+        </html>
+        HTML
+      end
+      it "should return an Array of absolute URIs" do
+        expect(subject.urls).to be == [
+          URI("http://#{host}#{iframe}")
+        ]
+      end
+    end
+    context "when the page contains a 'link' element" do
       let(:stylesheet) { '/stylesheet.css' }
+      let(:body) do
+        <<-HTML
+        <html>
+          <head>
+            <link type="stylesheet" href="#{stylesheet}" />
+          </head>
+          <body>
+          </body>
+        </html>
+        HTML
+      end
+      it "should return an Array of absolute URIs" do
+        expect(subject.urls).to be == [
+          URI("http://#{host}#{stylesheet}")
+        ]
+      end
+    end
+    context "when the page contains a 'script' element" do
       let(:javascript) { '/script.js' }
       let(:body) do
-        %{<html>} +
-          %{<head>} +
-            %{<link rel="stylesheet" type="text/css" href="#{stylesheet}" />} +
-            %{<script type="text/javascript" src="#{javascript}"></script>} +
-          %{</head>} +
-          %{<body>} +
-            %{<a href="#{link}">link</a>} +
-            %{<frameset><frame src="#{frame}" /></frameset>} +
-            %{<iframe src="#{iframe}" />} +
-          %{</body>} +
-        %{</html>}
+        <<-HTML
+        <html>
+          <head>
+            <script src="#{javascript}" />
+          </head>
+          <body>
+          </body>
+        </html>
+        HTML
       end
       it "should return an Array of absolute URIs" do
         expect(subject.urls).to be == [
-          URI("http://#{host}#{link}"),
-          URI("http://#{host}#{frame}"),
-          URI("http://#{host}#{iframe}"),
-          URI("http://#{host}#{stylesheet}"),
           URI("http://#{host}#{javascript}")
         ]
       end

data/spec/page/status_codes_spec.rb CHANGED Viewed

@@ -26,10 +26,6 @@ describe Page do
     include_examples "status code method", :is_ok?, {200 => true, 500 => false}
   end
-  describe "#timedout?" do
-    include_examples "status code method", :timedout?, {308 => true, 200 => false}
-  end
   describe "#bad_request?" do
     include_examples "status code method", :bad_request?, {400 => true, 200 => false}
   end
@@ -46,6 +42,10 @@ describe Page do
     include_examples "status code method", :is_missing?, {404 => true, 200 => false}
   end
+  describe "#is_timedout?" do
+    include_examples "status code method", :is_timedout?, {408 => true, 200 => false}
+  end
   describe "#had_internal_server_error?" do
     include_examples "status code method", :had_internal_server_error?, {500 => true, 200 => false}
   end

data/spec/proxy_spec.rb CHANGED Viewed

@@ -26,13 +26,13 @@ describe Spidr::Proxy do
       it { expect(subject.enabled?).to be true }
     end
-    context "when hist is not set" do
+    context "when host is not set" do
       it { expect(subject.enabled?).to be false }
     end
   end
   describe "#disabled?" do
-    context "when hist is not set" do
+    context "when host is not set" do
       it { expect(subject.disabled?).to be true }
     end

data/spec/settings/proxy_examples.rb CHANGED Viewed

@@ -16,7 +16,7 @@ shared_examples "includes Spidr::Settings::Proxy" do
       end
       it "should retain the default value" do
-        expect(subject.proxy.object_id).to be subject.proxy.object_id
+        expect(subject.proxy.object_id).to be(subject.proxy.object_id)
       end
     end
@@ -26,7 +26,7 @@ shared_examples "includes Spidr::Settings::Proxy" do
       end
       it "should return the set @proxy" do
-        expect(subject.proxy).to be proxy
+        expect(subject.proxy).to be(proxy)
       end
     end
   end
@@ -35,12 +35,10 @@ shared_examples "includes Spidr::Settings::Proxy" do
     context "when given a Proxy object" do
       let(:proxy) { Proxy.new(host: proxy_host, port: proxy_port) }
-      before do
-        subject.proxy = proxy
-      end
+      before { subject.proxy = proxy }
       it "should save it" do
-        expect(subject.proxy).to be proxy
+        expect(subject.proxy).to be(proxy)
       end
     end
@@ -51,15 +49,37 @@ shared_examples "includes Spidr::Settings::Proxy" do
       it "should create a new Proxy object" do
         expect(subject.proxy).to be_kind_of(Proxy)
-        expect(subject.proxy[:host]).to be proxy_host
-        expect(subject.proxy[:port]).to be proxy_port
+        expect(subject.proxy[:host]).to be(proxy_host)
+        expect(subject.proxy[:port]).to be(proxy_port)
       end
     end
-    context "when given nil" do
-      before do
-        subject.proxy = nil
+    context "when given a URI::HTTP" do
+      let(:uri) { URI::HTTP.build(host: proxy_host, port: proxy_port) }
+      before { subject.proxy = uri }
+      it "should create a new Proxy object based on the URI" do
+        expect(subject.proxy).to be_kind_of(Proxy)
+        expect(subject.proxy[:host]).to eq(proxy_host)
+        expect(subject.proxy[:port]).to eq(proxy_port)
+      end
+    end
+    context "when given a String" do
+      let(:url) { "http://#{proxy_host}:#{proxy_port}" }
+      before { subject.proxy = url }
+      it "should parse the String as a URI and create a new Proxy object" do
+        expect(subject.proxy).to be_kind_of(Proxy)
+        expect(subject.proxy[:host]).to eq(proxy_host)
+        expect(subject.proxy[:port]).to eq(proxy_port)
       end
+    end
+    context "when given nil" do
+      before { subject.proxy = nil }
       it "should leave an empty proxy" do
         expect(subject.proxy).to be_kind_of(Proxy)

data/spec/spec_helper.rb CHANGED Viewed

@@ -1,4 +1,7 @@
 require 'rspec'
+require 'simplecov'
 require 'spidr/version'
 include Spidr
+SimpleCov.start

metadata CHANGED Viewed

@@ -1,43 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: spidr
 version: !ruby/object:Gem::Version
-  version: 0.6.0
+  version: 0.7.0
 platform: ruby
 authors:
 - Postmodern
-autorequire:
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-08-05 00:00:00.000000000 Z
+date: 2023-01-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '2.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '2.0'
 description: Spidr is a versatile Ruby web spidering library that can spider a site,
   multiple domains, certain links or infinitely. Spidr is designed to be fast and
   easy to use.
@@ -49,10 +49,11 @@ extra_rdoc_files:
 - LICENSE.txt
 - README.md
 files:
-- .gitignore
-- .rspec
-- .travis.yml
-- .yardopts
+- ".editorconfig"
+- ".github/workflows/ruby.yml"
+- ".gitignore"
+- ".rspec"
+- ".yardopts"
 - ChangeLog.md
 - Gemfile
 - LICENSE.txt
@@ -112,24 +113,23 @@ homepage: https://github.com/postmodern/spidr#readme
 licenses:
 - MIT
 metadata: {}
-post_install_message:
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: 2.0.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project:
-rubygems_version: 2.0.14.1
-signing_key:
+rubygems_version: 3.3.26
+signing_key:
 specification_version: 4
 summary: A versatile Ruby web spidering library
 test_files: []

data/.travis.yml DELETED Viewed

@@ -1,14 +0,0 @@
----
-language: ruby
-rvm:
-  - 2.0.0
-  - 2.1.9
-  - 2.2.4
-  - 2.3.1
-  - jruby
-  - rbx
-matrix:
-  allow_failures:
-    - rvm: jruby
-    - rvm: rbx
-script: rake spec