spider 0.4.4 → 0.5.0

data/lib/spider/spider_instance.rb

@@ -1,28 +1,5 @@
 # Specialized spidering rules.
 
-# Copyright 2007-2008 Mike Burns & John Nagro
-# Redistribution and use in source and binary forms, with or without
-# modification, are permitted provided that the following conditions are met:
-#      * Redistributions of source code must retain the above copyright
-#      notice, this list of conditions and the following disclaimer.
-#      * Redistributions in binary form must reproduce the above copyright
-#      notice, this list of conditions and the following disclaimer in the
-#      documentation and/or other materials provided with the distribution.
-#      * Neither the name Mike Burns nor the
-#      names of his contributors may be used to endorse or promote products
-#      derived from this software without specific prior written permission.
-#  
-# THIS SOFTWARE IS PROVIDED BY Mike Burns ``AS IS'' AND ANY
-# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-# DISCLAIMED. IN NO EVENT SHALL Mike Burns BE LIABLE FOR ANY
-# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
-# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
 require File.dirname(__FILE__)+'/robot_rules.rb'
 require 'open-uri'
 require 'uri'
@@ -53,7 +30,7 @@ class SpiderInstance
     @callbacks   = {}
     @next_urls   = [next_urls]
     @seen        = seen
-    @rules       = rules || RobotRules.new('Ruby Spider 0.4.4')
+    @rules       = rules || RobotRules.new("Ruby Spider #{Spider::VERSION}")
     @robots_seen = robots_seen
     @headers     = {}
     @setup       = nil
@@ -64,9 +41,9 @@ class SpiderInstance
   # All predicates must be true in order for a URL to proceed.
   #
   # Takes a block that takes a string and produces a boolean. For example, this
-  # will ensure that the URL starts with 'http://mike-burns.com':
+  # will ensure that the URL starts with 'http://cashcats.biz':
   #
-  #  add_url_check { |a_url| a_url =~ %r{^http://mike-burns.com.*}
+  #  add_url_check { |a_url| a_url =~ %r{^http://cashcats.biz.*}
   def add_url_check(&block)
     @url_checks << block
   end
@@ -126,7 +103,7 @@ class SpiderInstance
   #
   # The arguments to the block are: the URL as a string, an instance of
   # Net::HTTPResponse, and the prior URL as a string.
-  # 
+  #
   #
   # For example:
   #
@@ -183,13 +160,14 @@ class SpiderInstance
     @headers = {}
   end
 
-  def start! #:nodoc: 
+  def start! #:nodoc:
     interrupted = false
-    trap("SIGINT") { interrupted = true } 
+    trap("SIGINT") { interrupted = true }
     begin
       next_urls = @next_urls.pop
       tmp_n_u = {}
       next_urls.each do |prior_url, urls|
+        urls = [urls] unless urls.kind_of?(Array)
         urls.map do |a_url|
           [a_url, (URI.parse(a_url) rescue nil)]
         end.select do |a_url, parsed_url|
@@ -233,7 +211,7 @@ class SpiderInstance
     return false unless @url_checks.map{|url_check|url_check.call(a_url)}.all?
     begin
       unless @robots_seen.include?(u)
-        #open(u, 'User-Agent' => 'Ruby Spider', 
+        #open(u, 'User-Agent' => 'Ruby Spider',
         #  'Accept' => 'text/html,text/xml,application/xml,text/plain', :ssl_verify => false) do |url|
         #  @rules.parse(u, url.read)
         #end
@@ -332,4 +310,3 @@ class SpiderInstance
     end
   end
 end
-

data/spider.gemspec

@@ -1,10 +1,12 @@
 require 'rubygems'
 
+require File.expand_path('../lib/spider', __FILE__)
+
 spec = Gem::Specification.new do |s|
   s.author = 'John Nagro'
   s.email = 'john.nagro@gmail.com'
   s.has_rdoc = true
-  s.homepage = 'http://spider.rubyforge.org/'
+  s.homepage = 'https://github.com/johnnagro/spider'
   s.name = 'spider'
   s.rubyforge_project = 'spider'
   s.summary = 'A Web spidering library'
@@ -14,5 +16,5 @@ spec = Gem::Specification.new do |s|
 A Web spidering library: handles robots.txt, scraping, finding more
 links, and doing it all over again.
 EOF
-  s.version = '0.4.4'
+  s.version = Spider::VERSION
 end

metadata

@@ -1,149 +1,58 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: spider
-version: !ruby/object:Gem::Version 
-  version: 0.4.4
+version: !ruby/object:Gem::Version
+  version: 0.5.0
 platform: ruby
-authors: 
+authors:
 - John Nagro
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2009-05-21 00:00:00 +00:00
-default_executable: 
+date: 2016-05-13 00:00:00.000000000 Z
 dependencies: []
-
-description: "A Web spidering library: handles robots.txt, scraping, finding more links, and doing it all over again."
+description: |
+  A Web spidering library: handles robots.txt, scraping, finding more
+  links, and doing it all over again.
 email: john.nagro@gmail.com
 executables: []
-
 extensions: []
-
 extra_rdoc_files: []
-
-files: 
-- README
-- doc
-- doc/fr_class_index.html
-- doc/rdoc-style.css
-- doc/fr_file_index.html
-- doc/created.rid
-- doc/fr_method_index.html
-- doc/classes
-- doc/classes/RobotRules.html
-- doc/classes/Spider.src
-- doc/classes/Spider.src/M000029.html
-- doc/classes/NextUrlsInSQS.src
-- doc/classes/NextUrlsInSQS.src/M000018.html
-- doc/classes/NextUrlsInSQS.src/M000019.html
-- doc/classes/NextUrlsInSQS.src/M000020.html
-- doc/classes/LoopingServlet.html
-- doc/classes/Spider.html
-- doc/classes/BeStaticServerPages.html
-- doc/classes/SpiderInstance.src
-- doc/classes/SpiderInstance.src/M000021.html
-- doc/classes/SpiderInstance.src/M000023.html
-- doc/classes/SpiderInstance.src/M000027.html
-- doc/classes/SpiderInstance.src/M000025.html
-- doc/classes/SpiderInstance.src/M000026.html
-- doc/classes/SpiderInstance.src/M000028.html
-- doc/classes/SpiderInstance.src/M000022.html
-- doc/classes/SpiderInstance.src/M000024.html
-- doc/classes/LoopingServlet.src
-- doc/classes/LoopingServlet.src/M000037.html
-- doc/classes/IncludedInMemcached.html
-- doc/classes/RobotRules.src
-- doc/classes/RobotRules.src/M000035.html
-- doc/classes/RobotRules.src/M000034.html
-- doc/classes/RobotRules.src/M000036.html
-- doc/classes/BeStaticServerPages.src
-- doc/classes/BeStaticServerPages.src/M000032.html
-- doc/classes/BeStaticServerPages.src/M000033.html
-- doc/classes/BeStaticServerPages.src/M000030.html
-- doc/classes/BeStaticServerPages.src/M000031.html
-- doc/classes/NextUrlsInSQS.html
-- doc/classes/IncludedInMemcached.src
-- doc/classes/IncludedInMemcached.src/M000017.html
-- doc/classes/IncludedInMemcached.src/M000015.html
-- doc/classes/IncludedInMemcached.src/M000016.html
-- doc/classes/SpiderInstance.html
-- doc/classes/QueryServlet.src
-- doc/classes/QueryServlet.src/M000038.html
-- doc/classes/QueryServlet.html
-- doc/files
-- doc/files/README.html
-- doc/files/spec
-- doc/files/spec/spec_helper_rb.src
-- doc/files/spec/spec_helper_rb.src/M000002.html
-- doc/files/spec/spec_helper_rb.src/M000005.html
-- doc/files/spec/spec_helper_rb.src/M000001.html
-- doc/files/spec/spec_helper_rb.src/M000003.html
-- doc/files/spec/spec_helper_rb.src/M000004.html
-- doc/files/spec/spider_spec_rb.src
-- doc/files/spec/spider_spec_rb.src/M000014.html
-- doc/files/spec/spec_helper_rb.html
-- doc/files/spec/spider
-- doc/files/spec/spider/included_in_memcached_spec_rb.src
-- doc/files/spec/spider/included_in_memcached_spec_rb.src/M000007.html
-- doc/files/spec/spider/included_in_memcached_spec_rb.src/M000006.html
-- doc/files/spec/spider/spider_instance_spec_rb.src
-- doc/files/spec/spider/spider_instance_spec_rb.src/M000013.html
-- doc/files/spec/spider/spider_instance_spec_rb.src/M000009.html
-- doc/files/spec/spider/spider_instance_spec_rb.src/M000010.html
-- doc/files/spec/spider/spider_instance_spec_rb.src/M000012.html
-- doc/files/spec/spider/spider_instance_spec_rb.src/M000008.html
-- doc/files/spec/spider/spider_instance_spec_rb.src/M000011.html
-- doc/files/spec/spider/included_in_memcached_spec_rb.html
-- doc/files/spec/spider/spider_instance_spec_rb.html
-- doc/files/spec/spider_spec_rb.html
-- doc/files/lib
-- doc/files/lib/spider_rb.html
-- doc/files/lib/spider
-- doc/files/lib/spider/included_in_memcached_rb.html
-- doc/files/lib/spider/robot_rules_rb.html
-- doc/files/lib/spider/next_urls_in_sqs_rb.html
-- doc/files/lib/spider/spider_instance_rb.html
-- doc/index.html
-- spider.gemspec
-- spec
-- spec/spec_helper.rb
-- spec/spider
-- spec/spider/included_in_memcached_spec.rb
-- spec/spider/spider_instance_spec.rb
-- spec/spider_spec.rb
+files:
+- AUTHORS
 - CHANGES
-- lib
+- LICENSE
+- README.md
 - lib/spider.rb
-- lib/spider
-- lib/spider/robot_rules.rb
+- lib/spider/included_in_memcached.rb
 - lib/spider/next_urls_in_sqs.rb
+- lib/spider/robot_rules.rb
 - lib/spider/spider_instance.rb
-- lib/spider/included_in_memcached.rb
-has_rdoc: true
-homepage: http://spider.rubyforge.org/
+- spec/spec_helper.rb
+- spec/spider/included_in_memcached_spec.rb
+- spec/spider/spider_instance_spec.rb
+- spec/spider_spec.rb
+- spider.gemspec
+homepage: https://github.com/johnnagro/spider
+licenses: []
+metadata: {}
 post_install_message: 
 rdoc_options: []
-
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
-  requirements: 
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
   - - ">="
-    - !ruby/object:Gem::Version 
-      version: "0"
-  version: 
-required_rubygems_version: !ruby/object:Gem::Requirement 
-  requirements: 
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
   - - ">="
-    - !ruby/object:Gem::Version 
-      version: "0"
-  version: 
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: spider
-rubygems_version: 1.3.0
+rubygems_version: 2.5.1
 signing_key: 
-specification_version: 2
+specification_version: 4
 summary: A Web spidering library
 test_files: []
-

data/doc/classes/BeStaticServerPages.html

@@ -1,197 +0,0 @@
-<?xml version="1.0" encoding="iso-8859-1"?>
-<!DOCTYPE html 
-     PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
-     "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
-
-<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
-<head>
-  <title>Class: BeStaticServerPages</title>
-  <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
-  <meta http-equiv="Content-Script-Type" content="text/javascript" />
-  <link rel="stylesheet" href=".././rdoc-style.css" type="text/css" media="screen" />
-  <script type="text/javascript">
-  // <![CDATA[
-
-  function popupCode( url ) {
-    window.open(url, "Code", "resizable=yes,scrollbars=yes,toolbar=no,status=no,height=150,width=400")
-  }
-
-  function toggleCode( id ) {
-    if ( document.getElementById )
-      elem = document.getElementById( id );
-    else if ( document.all )
-      elem = eval( "document.all." + id );
-    else
-      return false;
-
-    elemStyle = elem.style;
-    
-    if ( elemStyle.display != "block" ) {
-      elemStyle.display = "block"
-    } else {
-      elemStyle.display = "none"
-    }
-
-    return true;
-  }
-  
-  // Make codeblocks hidden by default
-  document.writeln( "<style type=\"text/css\">div.method-source-code { display: none }</style>" )
-  
-  // ]]>
-  </script>
-
-</head>
-<body>
-
-
-
-    <div id="classHeader">
-        <table class="header-table">
-        <tr class="top-aligned-row">
-          <td><strong>Class</strong></td>
-          <td class="class-name-in-header">BeStaticServerPages</td>
-        </tr>
-        <tr class="top-aligned-row">
-            <td><strong>In:</strong></td>
-            <td>
-                <a href="../files/spec/spec_helper_rb.html">
-                spec/spec_helper.rb
-                </a>
-        <br />
-            </td>
-        </tr>
-
-        <tr class="top-aligned-row">
-            <td><strong>Parent:</strong></td>
-            <td>
-                Object
-            </td>
-        </tr>
-        </table>
-    </div>
-  <!-- banner header -->
-
-  <div id="bodyContent">
-
-
-
-  <div id="contextContent">
-
-
-
-   </div>
-
-    <div id="method-list">
-      <h3 class="section-bar">Methods</h3>
-
-      <div class="name-list">
-      <a href="#M000033">description</a>&nbsp;&nbsp;
-      <a href="#M000032">failure_message</a>&nbsp;&nbsp;
-      <a href="#M000031">matches?</a>&nbsp;&nbsp;
-      <a href="#M000030">new</a>&nbsp;&nbsp;
-      </div>
-    </div>
-
-  </div>
-
-
-    <!-- if includes -->
-
-    <div id="section">
-
-
-
-
-
-    <div id="attribute-list">
-      <h3 class="section-bar">Attributes</h3>
-
-      <div class="name-list">
-        <table>
-        <tr class="top-aligned-row context-row">
-          <td class="context-item-name">actual</td>
-          <td class="context-item-value">&nbsp;[RW]&nbsp;</td>
-          <td class="context-item-desc"></td>
-        </tr>
-        </table>
-      </div>
-    </div>
-      
-
-
-    <!-- if method_list -->
-    <div id="methods">
-      <h3 class="section-bar">Public Class methods</h3>
-
-      <div id="method-M000030" class="method-detail">
-        <a name="M000030"></a>
-
-        <div class="method-heading">
-          <a href="BeStaticServerPages.src/M000030.html" target="Code" class="method-signature"
-            onclick="popupCode('BeStaticServerPages.src/M000030.html');return false;">
-          <span class="method-name">new</span><span class="method-args">()</span>
-          </a>
-        </div>
-      
-        <div class="method-description">
-        </div>
-      </div>
-
-      <h3 class="section-bar">Public Instance methods</h3>
-
-      <div id="method-M000033" class="method-detail">
-        <a name="M000033"></a>
-
-        <div class="method-heading">
-          <a href="BeStaticServerPages.src/M000033.html" target="Code" class="method-signature"
-            onclick="popupCode('BeStaticServerPages.src/M000033.html');return false;">
-          <span class="method-name">description</span><span class="method-args">()</span>
-          </a>
-        </div>
-      
-        <div class="method-description">
-        </div>
-      </div>
-
-      <div id="method-M000032" class="method-detail">
-        <a name="M000032"></a>
-
-        <div class="method-heading">
-          <a href="BeStaticServerPages.src/M000032.html" target="Code" class="method-signature"
-            onclick="popupCode('BeStaticServerPages.src/M000032.html');return false;">
-          <span class="method-name">failure_message</span><span class="method-args">()</span>
-          </a>
-        </div>
-      
-        <div class="method-description">
-        </div>
-      </div>
-
-      <div id="method-M000031" class="method-detail">
-        <a name="M000031"></a>
-
-        <div class="method-heading">
-          <a href="BeStaticServerPages.src/M000031.html" target="Code" class="method-signature"
-            onclick="popupCode('BeStaticServerPages.src/M000031.html');return false;">
-          <span class="method-name">matches?</span><span class="method-args">(actual)</span>
-          </a>
-        </div>
-      
-        <div class="method-description">
-        </div>
-      </div>
-
-
-    </div>
-
-
-  </div>
-
-
-<div id="validator-badges">
-  <p><small><a href="http://validator.w3.org/check/referer">[Validate]</a></small></p>
-</div>
-
-</body>
-</html>