spidr 0.5.0 → 0.6.0

data/spec/settings/proxy_examples.rb

@@ -0,0 +1,82 @@
+require 'rspec'
+
+shared_examples "includes Spidr::Settings::Proxy" do
+  let(:proxy_host) { 'proxy.example.com' }
+  let(:proxy_port) { 9999 }
+  let(:proxy) { Spidr::Proxy.new(host: proxy_host, port: proxy_port) }
+
+  describe "proxy" do
+    context "when @proxy is not set" do
+      before do
+        subject.instance_variable_set(:"@proxy",nil)
+      end
+
+      it "should return the disabled proxy" do
+        expect(subject.proxy).to be_disabled
+      end
+
+      it "should retain the default value" do
+        expect(subject.proxy.object_id).to be subject.proxy.object_id
+      end
+    end
+
+    context "when @proxy is set" do
+      before do
+        subject.instance_variable_set(:"@proxy",proxy)
+      end
+
+      it "should return the set @proxy" do
+        expect(subject.proxy).to be proxy
+      end
+    end
+  end
+
+  describe "proxy=" do
+    context "when given a Proxy object" do
+      let(:proxy) { Proxy.new(host: proxy_host, port: proxy_port) }
+
+      before do
+        subject.proxy = proxy
+      end
+
+      it "should save it" do
+        expect(subject.proxy).to be proxy
+      end
+    end
+
+    context "when given a Hash" do
+      before do
+        subject.proxy = {host: proxy_host, port: proxy_port}
+      end
+
+      it "should create a new Proxy object" do
+        expect(subject.proxy).to be_kind_of(Proxy)
+        expect(subject.proxy[:host]).to be proxy_host
+        expect(subject.proxy[:port]).to be proxy_port
+      end
+    end
+
+    context "when given nil" do
+      before do
+        subject.proxy = nil
+      end
+
+      it "should leave an empty proxy" do
+        expect(subject.proxy).to be_kind_of(Proxy)
+        expect(subject.proxy[:host]).to be_nil
+      end
+    end
+  end
+
+  describe "disable_proxy!" do
+    before do
+      subject.proxy = proxy
+
+      subject.disable_proxy!
+    end
+
+    it "should reset the proxy" do
+      expect(subject.proxy).to be_disabled
+    end
+  end
+end

data/spec/settings/timeouts_examples.rb

@@ -0,0 +1,93 @@
+require 'rspec'
+
+shared_examples_for "includes Spidr::Settings::Timeouts" do
+  describe "read_timeout" do
+    context "default value" do
+      it { expect(subject.read_timeout).to be_nil }
+    end
+  end
+
+  describe "read_timeout=" do
+    let(:value) { 5 }
+
+    before { subject.read_timeout = value }
+
+    it "should update read_timeout" do
+      expect(subject.read_timeout).to be == value
+    end
+
+    after { subject.read_timeout = nil }
+  end
+
+  describe "open_timeout" do
+    context "default value" do
+      it { expect(subject.open_timeout).to be_nil }
+    end
+  end
+
+  describe "open_timeout=" do
+    let(:value) { 5 }
+
+    before { subject.open_timeout = value }
+
+    it "should update open_timeout" do
+      expect(subject.open_timeout).to be == value
+    end
+
+    after { subject.open_timeout = nil }
+  end
+
+  describe "ssl_timeout" do
+    context "default value" do
+      it { expect(subject.ssl_timeout).to be_nil }
+    end
+  end
+
+  describe "ssl_timeout=" do
+    let(:value) { 5 }
+
+    before { subject.ssl_timeout = value }
+
+    it "should update ssl_timeout" do
+      expect(subject.ssl_timeout).to be == value
+    end
+
+    after { subject.ssl_timeout = nil }
+  end
+
+  describe "continue_timeout" do
+    context "default value" do
+      it { expect(subject.continue_timeout).to be_nil }
+    end
+  end
+
+  describe "continue_timeout=" do
+    let(:value) { 5 }
+
+    before { subject.continue_timeout = value }
+
+    it "should update continue_timeout" do
+      expect(subject.continue_timeout).to be == value
+    end
+
+    after { subject.continue_timeout = nil }
+  end
+
+  describe "keep_alive_timeout" do
+    context "default value" do
+      it { expect(subject.keep_alive_timeout).to be_nil }
+    end
+  end
+
+  describe "keep_alive_timeout=" do
+    let(:value) { 5 }
+
+    before { subject.keep_alive_timeout = value }
+
+    it "should update keep_alive_timeout" do
+      expect(subject.keep_alive_timeout).to be == value
+    end
+
+    after { subject.keep_alive_timeout = nil }
+  end
+end

data/spec/settings/user_agent_examples.rb

@@ -0,0 +1,25 @@
+require 'rspec'
+
+shared_examples_for "includes Spidr::Settings::UserAgent" do
+  describe "user_agent" do
+    context "default value" do
+      it { expect(subject.user_agent).to be_nil }
+    end
+  end
+
+  describe "user_agent=" do
+    let(:user_agent) { 'Mozilla/5.0 (iPad; U; CPU OS 3_2_1 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Mobile/7B405' }
+
+    before do
+      subject.user_agent = user_agent
+    end
+
+    it "should update the user_agent" do
+      expect(subject.user_agent).to be == user_agent
+    end
+
+    after do
+      subject.user_agent = nil
+    end
+  end
+end

data/spec/spidr_spec.rb

@@ -1,39 +1,16 @@
 require 'spidr'
 
 require 'spec_helper'
+require 'settings/proxy_examples'
+require 'settings/timeouts_examples'
+require 'settings/user_agent_examples'
 
 describe Spidr do
   it "should have a VERSION constant" do
     expect(subject.const_defined?('VERSION')).to eq(true)
   end
 
-  describe "proxy" do
-    after(:all) do
-      Spidr.disable_proxy!
-    end
-
-    it "should not have proxy settings by default" do
-      expect(subject.proxy[:host]).to be_nil
-    end
-
-    it "should allow setting new proxy settings" do
-      subject.proxy = {host: 'example.com', port: 8010}
-
-      expect(subject.proxy[:host]).to eq('example.com')
-      expect(subject.proxy[:port]).to eq(8010)
-    end
-
-    it "should default the :port option of new proxy settings" do
-      subject.proxy = {host: 'example.com'}
-
-      expect(subject.proxy[:host]).to eq('example.com')
-      expect(subject.proxy[:port]).to eq(Spidr::COMMON_PROXY_PORT)
-    end
-
-    it "should allow disabling the proxy" do
-      subject.disable_proxy!
-
-      expect(subject.proxy[:host]).to be_nil
-    end
-  end
+  it_should_behave_like "includes Spidr::Settings::Proxy"
+  it_should_behave_like "includes Spidr::Settings::Timeouts"
+  it_should_behave_like "includes Spidr::Settings::UserAgent"
 end

data/spidr.gemspec

@@ -2,130 +2,59 @@
 
 require 'yaml'
 
-Gem::Specification.new do |gemspec|
-  root = File.dirname(__FILE__)
-  lib_dir = File.join(root,'lib')
-  files = if File.directory?('.git')
-            `git ls-files`.split($/)
-          elsif File.directory?('.hg')
-            `hg manifest`.split($/)
-          elsif File.directory?('.svn')
-            `svn ls -R`.split($/).select { |path| File.file?(path) }
-          else
-            Dir['{**/}{.*,*}'].select { |path| File.file?(path) }
-          end
+Gem::Specification.new do |gem|
+  gemspec = YAML.load_file('gemspec.yml')
 
-  filter_files = lambda { |paths|
-    case paths
-    when Array
-      (files & paths)
-    when String
-      (files & Dir[paths])
-    end
-  }
-
-  version = {
-    :file => 'spidr/version',
-    :constant => 'Spidr::VERSION'
-  }
-
-  defaults = {
-    'name' => File.basename(root),
-    'files' => files,
-    'executables' => filter_files['bin/*'].map { |path| File.basename(path) },
-    'test_files' => filter_files['{test/{**/}*_test.rb,spec/{**/}*_spec.rb}'],
-    'extra_doc_files' => filter_files['*.{txt,rdoc,md,markdown,tt,textile}'],
-  }
+  gem.name    = gemspec.fetch('name')
+  gem.version = gemspec.fetch('version') do
+                  lib_dir = File.join(File.dirname(__FILE__),'lib')
+                  $LOAD_PATH << lib_dir unless $LOAD_PATH.include?(lib_dir)
 
-  metadata = defaults.merge(YAML.load_file('gemspec.yml'))
+                  require 'spidr/version'
+                  Spidr::VERSION
+                end
 
-  gemspec.name = metadata.fetch('name',defaults[:name])
-  gemspec.version = if metadata['version']
-                      metadata['version']
-                    else
-                      $LOAD_PATH << lib_dir unless $LOAD_PATH.include?(lib_dir)
+  gem.summary     = gemspec['summary']
+  gem.description = gemspec['description']
+  gem.licenses    = Array(gemspec['license'])
+  gem.authors     = Array(gemspec['authors'])
+  gem.email       = gemspec['email']
+  gem.homepage    = gemspec['homepage']
 
-                      require version[:file]
-                      eval(version[:constant])
-                    end
+  glob = lambda { |patterns| gem.files & Dir[*patterns] }
 
-  gemspec.summary = metadata.fetch('summary',metadata['description'])
-  gemspec.description = metadata.fetch('description',metadata['summary'])
+  gem.files = `git ls-files`.split($/)
+  gem.files = glob[gemspec['files']] if gemspec['files']
 
-  case metadata['license']
-  when Array
-    gemspec.licenses = metadata['license']
-  when String
-    gemspec.license = metadata['license']
+  gem.executables = gemspec.fetch('executables') do
+    glob['bin/*'].map { |path| File.basename(path) }
   end
+  gem.default_executable = gem.executables.first if Gem::VERSION < '1.7.'
 
-  case metadata['authors']
-  when Array
-    gemspec.authors = metadata['authors']
-  when String
-    gemspec.author = metadata['authors']
-  end
-
-  gemspec.email = metadata['email']
-  gemspec.homepage = metadata['homepage']
+  gem.extensions       = glob[gemspec['extensions'] || 'ext/**/extconf.rb']
+  gem.test_files       = glob[gemspec['test_files'] || '{test/{**/}*_test.rb']
+  gem.extra_rdoc_files = glob[gemspec['extra_doc_files'] || '*.{txt,md}']
 
-  case metadata['require_paths']
-  when Array
-    gemspec.require_paths = metadata['require_paths']
-  when String
-    gemspec.require_path = metadata['require_paths']
-  end
+  gem.require_paths = Array(gemspec.fetch('require_paths') {
+    %w[ext lib].select { |dir| File.directory?(dir) }
+  })
 
-  gemspec.files = filter_files[metadata['files']]
+  gem.requirements              = Array(gemspec['requirements'])
+  gem.required_ruby_version     = gemspec['required_ruby_version']
+  gem.required_rubygems_version = gemspec['required_rubygems_version']
+  gem.post_install_message      = gemspec['post_install_message']
 
-  gemspec.executables = metadata['executables']
-  gemspec.extensions = metadata['extensions']
-
-  if Gem::VERSION < '1.7.'
-    gemspec.default_executable = gemspec.executables.first
-  end
-
-  gemspec.test_files = filter_files[metadata['test_files']]
-
-  unless gemspec.files.include?('.document')
-    gemspec.extra_rdoc_files = metadata['extra_doc_files']
-  end
-
-  gemspec.post_install_message = metadata['post_install_message']
-  gemspec.requirements = metadata['requirements']
-
-  if gemspec.respond_to?(:required_ruby_version=)
-    gemspec.required_ruby_version = metadata['required_ruby_version']
-  end
-
-  if gemspec.respond_to?(:required_rubygems_version=)
-    gemspec.required_rubygems_version = metadata['required_rubygems_version']
-  end
-
-  parse_versions = lambda { |versions|
-    case versions
-    when Array
-      versions.map { |v| v.to_s }
-    when String
-      versions.split(/,\s*/)
-    end
-  }
-
-  if metadata['dependencies']
-    metadata['dependencies'].each do |name,versions|
-      gemspec.add_dependency(name,parse_versions[versions])  
-    end
-  end
+  split = lambda { |string| string.split(/,\s*/) }
 
-  if metadata['runtime_dependencies']
-    metadata['runtime_dependencies'].each do |name,versions|
-      gemspec.add_runtime_dependency(name,parse_versions[versions])  
+  if gemspec['dependencies']
+    gemspec['dependencies'].each do |name,versions|
+      gem.add_dependency(name,split[versions])
     end
   end
 
-  if metadata['development_dependencies']
-    metadata['development_dependencies'].each do |name,versions|
-      gemspec.add_development_dependency(name,parse_versions[versions])  
+  if gemspec['development_dependencies']
+    gemspec['development_dependencies'].each do |name,versions|
+      gem.add_development_dependency(name,split[versions])
     end
   end
 end

metadata

@@ -1,41 +1,41 @@
 --- !ruby/object:Gem::Specification
 name: spidr
 version: !ruby/object:Gem::Version
-  version: 0.5.0
+  version: 0.6.0
 platform: ruby
 authors:
 - Postmodern
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-01-04 00:00:00.000000000 Z
+date: 2016-08-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '1.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '1.3'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '1.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ~>
       - !ruby/object:Gem::Version
         version: '1.0'
 description: Spidr is a versatile Ruby web spidering library that can spider a site,
@@ -49,9 +49,10 @@ extra_rdoc_files:
 - LICENSE.txt
 - README.md
 files:
-- ".gitignore"
-- ".rspec"
-- ".yardopts"
+- .gitignore
+- .rspec
+- .travis.yml
+- .yardopts
 - ChangeLog.md
 - Gemfile
 - LICENSE.txt
@@ -63,6 +64,7 @@ files:
 - lib/spidr/agent/actions.rb
 - lib/spidr/agent/events.rb
 - lib/spidr/agent/filters.rb
+- lib/spidr/agent/robots.rb
 - lib/spidr/agent/sanitizers.rb
 - lib/spidr/auth_credential.rb
 - lib/spidr/auth_store.rb
@@ -70,11 +72,17 @@ files:
 - lib/spidr/extensions.rb
 - lib/spidr/extensions/uri.rb
 - lib/spidr/page.rb
-- lib/spidr/page/body.rb
-- lib/spidr/page/headers.rb
-- lib/spidr/page/links.rb
+- lib/spidr/page/content_types.rb
+- lib/spidr/page/cookies.rb
+- lib/spidr/page/html.rb
+- lib/spidr/page/status_codes.rb
+- lib/spidr/proxy.rb
 - lib/spidr/rules.rb
 - lib/spidr/session_cache.rb
+- lib/spidr/settings.rb
+- lib/spidr/settings/proxy.rb
+- lib/spidr/settings/timeouts.rb
+- lib/spidr/settings/user_agent.rb
 - lib/spidr/spidr.rb
 - lib/spidr/version.rb
 - spec/agent/actions_spec.rb
@@ -83,14 +91,20 @@ files:
 - spec/agent_spec.rb
 - spec/auth_store_spec.rb
 - spec/cookie_jar_spec.rb
+- spec/example_app.rb
+- spec/example_page.rb
 - spec/extensions/uri_spec.rb
-- spec/helpers/history.rb
-- spec/helpers/page.rb
-- spec/helpers/wsoc.rb
-- spec/page_examples.rb
+- spec/page/content_types_spec.rb
+- spec/page/cookies_spec.rb
+- spec/page/html_spec.rb
+- spec/page/status_codes_spec.rb
 - spec/page_spec.rb
+- spec/proxy_spec.rb
 - spec/rules_spec.rb
 - spec/session_cache.rb
+- spec/settings/proxy_examples.rb
+- spec/settings/timeouts_examples.rb
+- spec/settings/user_agent_examples.rb
 - spec/spec_helper.rb
 - spec/spidr_spec.rb
 - spidr.gemspec
@@ -104,28 +118,18 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
-      version: 1.9.1
+      version: 2.0.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.7
+rubygems_version: 2.0.14.1
 signing_key: 
 specification_version: 4
 summary: A versatile Ruby web spidering library
-test_files:
-- spec/agent/actions_spec.rb
-- spec/agent/filters_spec.rb
-- spec/agent/sanitizers_spec.rb
-- spec/agent_spec.rb
-- spec/auth_store_spec.rb
-- spec/cookie_jar_spec.rb
-- spec/extensions/uri_spec.rb
-- spec/page_spec.rb
-- spec/rules_spec.rb
-- spec/spidr_spec.rb
+test_files: []