RubyGems - spidr - Versions diffs - 0.6.1 → 0.7.1 - Mend

spidr 0.6.1 → 0.7.1

Files changed (47) hide show

checksums.yaml +4 -4
data/.editorconfig +11 -0
data/.github/workflows/ruby.yml +26 -0
data/.gitignore +4 -5
data/ChangeLog.md +19 -1
data/Gemfile +7 -4
data/LICENSE.txt +1 -1
data/README.md +136 -79
data/Rakefile +1 -0
data/gemspec.yml +7 -0
data/lib/spidr/agent/actions.rb +3 -1
data/lib/spidr/agent/events.rb +3 -1
data/lib/spidr/agent/filters.rb +57 -56
data/lib/spidr/agent/robots.rb +2 -0
data/lib/spidr/agent/sanitizers.rb +7 -8
data/lib/spidr/agent.rb +232 -108
data/lib/spidr/auth_credential.rb +2 -0
data/lib/spidr/auth_store.rb +9 -7
data/lib/spidr/cookie_jar.rb +7 -5
data/lib/spidr/extensions/uri.rb +3 -1
data/lib/spidr/extensions.rb +3 -1
data/lib/spidr/page/content_types.rb +53 -0
data/lib/spidr/page/cookies.rb +2 -0
data/lib/spidr/page/html.rb +21 -20
data/lib/spidr/page/status_codes.rb +15 -11
data/lib/spidr/page.rb +3 -1
data/lib/spidr/proxy.rb +8 -14
data/lib/spidr/rules.rb +7 -8
data/lib/spidr/session_cache.rb +26 -22
data/lib/spidr/settings/proxy.rb +22 -6
data/lib/spidr/settings/timeouts.rb +2 -0
data/lib/spidr/settings/user_agent.rb +2 -0
data/lib/spidr/settings.rb +5 -3
data/lib/spidr/spidr.rb +22 -11
data/lib/spidr/version.rb +3 -1
data/lib/spidr.rb +5 -3
data/spec/agent_spec.rb +356 -7
data/spec/example_page.rb +2 -0
data/spec/page/content_types_spec.rb +22 -0
data/spec/page/html_spec.rb +255 -51
data/spec/page/status_codes_spec.rb +4 -4
data/spec/proxy_spec.rb +2 -2
data/spec/settings/proxy_examples.rb +31 -11
data/spec/spec_helper.rb +3 -0
data/spidr.gemspec +1 -4
metadata +8 -7
data/.travis.yml +0 -16

data/spec/page/html_spec.rb CHANGED Viewed

@@ -297,7 +297,7 @@ describe Page do
     context "when the page contains iframes" do
       let(:iframe1) { '/iframe1' }
       let(:iframe2) { '/iframe2' }
-      let(:body)   { %{<html><body><iframe src="#{iframe1}" /><iframe src="#{iframe2}" /></body></html>} }
+      let(:body)   { %{<html><body><iframe src="#{iframe1}"></iframe><iframe src="#{iframe2}"></iframe></body></html>} }
       it "should yield each iframe/@src value" do
         expect { |b|
@@ -332,32 +332,100 @@ describe Page do
   end
   describe "#links" do
-    context "when the page contains links" do
+    context "when the page contains an 'a' link" do
       let(:link) { '/link' }
+      let(:body) do
+        <<-HTML
+        <html>
+          <body>
+            <a href="#{link}">link</a>
+          </body>
+        </html>
+        HTML
+      end
+      it "should return an Array of links" do
+        expect(subject.links).to be == [
+          link
+        ]
+      end
+    end
+    context "when the page contains a 'frame'" do
       let(:frame) { '/frame' }
+      let(:body) do
+        <<-HTML
+        <html>
+          <frameset>
+            <frame src="#{frame}"></frame>
+          </frameset>
+        </html>
+        HTML
+      end
+      it "should return an Array of links" do
+        expect(subject.links).to be == [
+          frame
+        ]
+      end
+    end
+    context "when the page contains a 'iframe'" do
       let(:iframe) { '/iframe' }
+      let(:body) do
+        <<-HTML
+        <html>
+          <body>
+            <iframe src="#{iframe}"></iframe>
+          </body>
+        </html>
+        HTML
+      end
+      it "should return an Array of links" do
+        expect(subject.links).to be == [
+          iframe
+        ]
+      end
+    end
+    context "when the page contains a 'link' element" do
       let(:stylesheet) { '/stylesheet.css' }
+      let(:body) do
+        <<-HTML
+        <html>
+          <head>
+            <link type="stylesheet" href="#{stylesheet}" />
+          </head>
+          <body>
+          </body>
+        </html>
+        HTML
+      end
+      it "should return an Array of links" do
+        expect(subject.links).to be == [
+          stylesheet
+        ]
+      end
+    end
+    context "when the page contains a 'script' element" do
       let(:javascript) { '/script.js' }
       let(:body) do
-        %{<html>} +
-          %{<head>} +
-            %{<link rel="stylesheet" type="text/css" href="#{stylesheet}" />} +
-            %{<script type="text/javascript" src="#{javascript}"></script>} +
-          %{</head>} +
-          %{<body>} +
-            %{<a href="#{link}">link</a>} +
-            %{<frameset><frame src="#{frame}" /></frameset>} +
-            %{<iframe src="#{iframe}" />} +
-          %{</body>} +
-        %{</html>}
+        <<-HTML
+        <html>
+          <head>
+            <script src="#{javascript}" />
+          </head>
+          <body>
+          </body>
+        </html>
+        HTML
       end
       it "should return an Array of links" do
         expect(subject.links).to be == [
-          link,
-          frame,
-          iframe,
-          stylesheet,
           javascript
         ]
       end
@@ -369,32 +437,100 @@ describe Page do
   end
   describe "#each_url" do
-    context "when the page contains links" do
+    context "when the page contains an 'a' link" do
       let(:link) { '/link' }
+      let(:body) do
+        <<-HTML
+        <html>
+          <body>
+            <a href="#{link}">link</a>
+          </body>
+        </html>
+        HTML
+      end
+      it "should yield successive absolute URIs" do
+        expect { |b| subject.each_url(&b) }.to yield_successive_args(
+          URI("http://#{host}#{link}")
+        )
+      end
+    end
+    context "when the page contains a 'frame'" do
       let(:frame) { '/frame' }
+      let(:body) do
+        <<-HTML
+        <html>
+          <frameset>
+            <frame src="#{frame}"></frame>
+          </frameset>
+        </html>
+        HTML
+      end
+      it "should yield successive absolute URIs" do
+        expect { |b| subject.each_url(&b) }.to yield_successive_args(
+          URI("http://#{host}#{frame}")
+        )
+      end
+    end
+    context "when the page contains a 'iframe'" do
       let(:iframe) { '/iframe' }
+      let(:body) do
+        <<-HTML
+        <html>
+          <body>
+            <iframe src="#{iframe}"></iframe>
+          </body>
+        </html>
+        HTML
+      end
+      it "should yield successive absolute URIs" do
+        expect { |b| subject.each_url(&b) }.to yield_successive_args(
+          URI("http://#{host}#{iframe}")
+        )
+      end
+    end
+    context "when the page contains a 'link' element" do
       let(:stylesheet) { '/stylesheet.css' }
-      let(:javascript) { '/script.js' }
       let(:body) do
-        %{<html>} +
-          %{<head>} +
-            %{<link rel="stylesheet" type="text/css" href="#{stylesheet}" />} +
-            %{<script type="text/javascript" src="#{javascript}"></script>} +
-          %{</head>} +
-          %{<body>} +
-            %{<a href="#{link}">link</a>} +
-            %{<frameset><frame src="#{frame}" /></frameset>} +
-            %{<iframe src="#{iframe}" />} +
-          %{</body>} +
-        %{</html>}
+        <<-HTML
+        <html>
+          <head>
+            <link type="stylesheet" href="#{stylesheet}" />
+          </head>
+          <body>
+          </body>
+        </html>
+        HTML
+      end
+      it "should yield successive absolute URIs" do
+        expect { |b| subject.each_url(&b) }.to yield_successive_args(
+          URI("http://#{host}#{stylesheet}")
+        )
       end
+    end
-      it "should return an Array of absolute URIs" do
+    context "when the page contains a 'script' element" do
+      let(:javascript) { '/script.js' }
+      let(:body) do
+        <<-HTML
+        <html>
+          <head>
+            <script src="#{javascript}" />
+          </head>
+          <body>
+          </body>
+        </html>
+        HTML
+      end
+      it "should yield successive absolute URIs" do
         expect { |b| subject.each_url(&b) }.to yield_successive_args(
-          URI("http://#{host}#{link}"),
-          URI("http://#{host}#{frame}"),
-          URI("http://#{host}#{iframe}"),
-          URI("http://#{host}#{stylesheet}"),
           URI("http://#{host}#{javascript}")
         )
       end
@@ -410,32 +546,100 @@ describe Page do
   end
   describe "#urls" do
-    context "when the page contains links" do
+    context "when the page contains an 'a' link" do
       let(:link) { '/link' }
+      let(:body) do
+        <<-HTML
+        <html>
+          <body>
+            <a href="#{link}">link</a>
+          </body>
+        </html>
+        HTML
+      end
+      it "should return an Array of absolute URIs" do
+        expect(subject.urls).to be == [
+          URI("http://#{host}#{link}")
+        ]
+      end
+    end
+    context "when the page contains a 'frame'" do
       let(:frame) { '/frame' }
+      let(:body) do
+        <<-HTML
+        <html>
+          <frameset>
+            <frame src="#{frame}"></frame>
+          </frameset>
+        </html>
+        HTML
+      end
+      it "should return an Array of absolute URIs" do
+        expect(subject.urls).to be == [
+          URI("http://#{host}#{frame}")
+        ]
+      end
+    end
+    context "when the page contains a 'iframe'" do
       let(:iframe) { '/iframe' }
+      let(:body) do
+        <<-HTML
+        <html>
+          <body>
+            <iframe src="#{iframe}"></iframe>
+          </body>
+        </html>
+        HTML
+      end
+      it "should return an Array of absolute URIs" do
+        expect(subject.urls).to be == [
+          URI("http://#{host}#{iframe}")
+        ]
+      end
+    end
+    context "when the page contains a 'link' element" do
       let(:stylesheet) { '/stylesheet.css' }
+      let(:body) do
+        <<-HTML
+        <html>
+          <head>
+            <link type="stylesheet" href="#{stylesheet}" />
+          </head>
+          <body>
+          </body>
+        </html>
+        HTML
+      end
+      it "should return an Array of absolute URIs" do
+        expect(subject.urls).to be == [
+          URI("http://#{host}#{stylesheet}")
+        ]
+      end
+    end
+    context "when the page contains a 'script' element" do
       let(:javascript) { '/script.js' }
       let(:body) do
-        %{<html>} +
-          %{<head>} +
-            %{<link rel="stylesheet" type="text/css" href="#{stylesheet}" />} +
-            %{<script type="text/javascript" src="#{javascript}"></script>} +
-          %{</head>} +
-          %{<body>} +
-            %{<a href="#{link}">link</a>} +
-            %{<frameset><frame src="#{frame}" /></frameset>} +
-            %{<iframe src="#{iframe}" />} +
-          %{</body>} +
-        %{</html>}
+        <<-HTML
+        <html>
+          <head>
+            <script src="#{javascript}" />
+          </head>
+          <body>
+          </body>
+        </html>
+        HTML
       end
       it "should return an Array of absolute URIs" do
         expect(subject.urls).to be == [
-          URI("http://#{host}#{link}"),
-          URI("http://#{host}#{frame}"),
-          URI("http://#{host}#{iframe}"),
-          URI("http://#{host}#{stylesheet}"),
           URI("http://#{host}#{javascript}")
         ]
       end

data/spec/page/status_codes_spec.rb CHANGED Viewed

@@ -26,10 +26,6 @@ describe Page do
     include_examples "status code method", :is_ok?, {200 => true, 500 => false}
   end
-  describe "#timedout?" do
-    include_examples "status code method", :timedout?, {308 => true, 200 => false}
-  end
   describe "#bad_request?" do
     include_examples "status code method", :bad_request?, {400 => true, 200 => false}
   end
@@ -46,6 +42,10 @@ describe Page do
     include_examples "status code method", :is_missing?, {404 => true, 200 => false}
   end
+  describe "#is_timedout?" do
+    include_examples "status code method", :is_timedout?, {408 => true, 200 => false}
+  end
   describe "#had_internal_server_error?" do
     include_examples "status code method", :had_internal_server_error?, {500 => true, 200 => false}
   end

data/spec/proxy_spec.rb CHANGED Viewed

@@ -26,13 +26,13 @@ describe Spidr::Proxy do
       it { expect(subject.enabled?).to be true }
     end
-    context "when hist is not set" do
+    context "when host is not set" do
       it { expect(subject.enabled?).to be false }
     end
   end
   describe "#disabled?" do
-    context "when hist is not set" do
+    context "when host is not set" do
       it { expect(subject.disabled?).to be true }
     end

data/spec/settings/proxy_examples.rb CHANGED Viewed

@@ -16,7 +16,7 @@ shared_examples "includes Spidr::Settings::Proxy" do
       end
       it "should retain the default value" do
-        expect(subject.proxy.object_id).to be subject.proxy.object_id
+        expect(subject.proxy.object_id).to be(subject.proxy.object_id)
       end
     end
@@ -26,7 +26,7 @@ shared_examples "includes Spidr::Settings::Proxy" do
       end
       it "should return the set @proxy" do
-        expect(subject.proxy).to be proxy
+        expect(subject.proxy).to be(proxy)
       end
     end
   end
@@ -35,12 +35,10 @@ shared_examples "includes Spidr::Settings::Proxy" do
     context "when given a Proxy object" do
       let(:proxy) { Proxy.new(host: proxy_host, port: proxy_port) }
-      before do
-        subject.proxy = proxy
-      end
+      before { subject.proxy = proxy }
       it "should save it" do
-        expect(subject.proxy).to be proxy
+        expect(subject.proxy).to be(proxy)
       end
     end
@@ -51,15 +49,37 @@ shared_examples "includes Spidr::Settings::Proxy" do
       it "should create a new Proxy object" do
         expect(subject.proxy).to be_kind_of(Proxy)
-        expect(subject.proxy[:host]).to be proxy_host
-        expect(subject.proxy[:port]).to be proxy_port
+        expect(subject.proxy[:host]).to be(proxy_host)
+        expect(subject.proxy[:port]).to be(proxy_port)
       end
     end
-    context "when given nil" do
-      before do
-        subject.proxy = nil
+    context "when given a URI::HTTP" do
+      let(:uri) { URI::HTTP.build(host: proxy_host, port: proxy_port) }
+      before { subject.proxy = uri }
+      it "should create a new Proxy object based on the URI" do
+        expect(subject.proxy).to be_kind_of(Proxy)
+        expect(subject.proxy[:host]).to eq(proxy_host)
+        expect(subject.proxy[:port]).to eq(proxy_port)
+      end
+    end
+    context "when given a String" do
+      let(:url) { "http://#{proxy_host}:#{proxy_port}" }
+      before { subject.proxy = url }
+      it "should parse the String as a URI and create a new Proxy object" do
+        expect(subject.proxy).to be_kind_of(Proxy)
+        expect(subject.proxy[:host]).to eq(proxy_host)
+        expect(subject.proxy[:port]).to eq(proxy_port)
       end
+    end
+    context "when given nil" do
+      before { subject.proxy = nil }
       it "should leave an empty proxy" do
         expect(subject.proxy).to be_kind_of(Proxy)

data/spec/spec_helper.rb CHANGED Viewed

@@ -1,4 +1,7 @@
 require 'rspec'
+require 'simplecov'
 require 'spidr/version'
 include Spidr
+SimpleCov.start

data/spidr.gemspec CHANGED Viewed

@@ -7,10 +7,7 @@ Gem::Specification.new do |gem|
   gem.name    = gemspec.fetch('name')
   gem.version = gemspec.fetch('version') do
-                  lib_dir = File.join(File.dirname(__FILE__),'lib')
-                  $LOAD_PATH << lib_dir unless $LOAD_PATH.include?(lib_dir)
-                  require 'spidr/version'
+                  require_relative 'lib/spidr/version'
                   Spidr::VERSION
                 end

metadata CHANGED Viewed

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: spidr
 version: !ruby/object:Gem::Version
-  version: 0.6.1
+  version: 0.7.1
 platform: ruby
 authors:
 - Postmodern
-autorequire:
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-10-25 00:00:00.000000000 Z
+date: 2024-01-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -49,9 +49,10 @@ extra_rdoc_files:
 - LICENSE.txt
 - README.md
 files:
+- ".editorconfig"
+- ".github/workflows/ruby.yml"
 - ".gitignore"
 - ".rspec"
-- ".travis.yml"
 - ".yardopts"
 - ChangeLog.md
 - Gemfile
@@ -112,7 +113,7 @@ homepage: https://github.com/postmodern/spidr#readme
 licenses:
 - MIT
 metadata: {}
-post_install_message:
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -127,8 +128,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
-signing_key:
+rubygems_version: 3.4.10
+signing_key:
 specification_version: 4
 summary: A versatile Ruby web spidering library
 test_files: []

data/.travis.yml DELETED Viewed

@@ -1,16 +0,0 @@
----
-before_install:
-  - gem update --system
-  - gem install bundler -v "~> 2.0"
-language: ruby
-sudo: false
-cache:
-  - bundler
-rvm:
-  - 2.5
-  - 2.6
-  - jruby
-matrix:
-  allow_failures:
-    - rvm: jruby
-script: bundle exec rake spec