RubyGems - spider - Versions diffs - 0.4.4 → 0.5.0 - Mend

spider 0.4.4 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (74) hide show

checksums.yaml +7 -0
data/AUTHORS +12 -0
data/CHANGES +6 -0
data/LICENSE +21 -0
data/{README → README.md} +50 -43
data/lib/spider.rb +12 -29
data/lib/spider/included_in_memcached.rb +1 -24
data/lib/spider/next_urls_in_sqs.rb +6 -29
data/lib/spider/robot_rules.rb +61 -57
data/lib/spider/spider_instance.rb +8 -31
data/spider.gemspec +4 -2
metadata +33 -124
data/doc/classes/BeStaticServerPages.html +0 -197
data/doc/classes/BeStaticServerPages.src/M000030.html +0 -19
data/doc/classes/BeStaticServerPages.src/M000031.html +0 -19
data/doc/classes/BeStaticServerPages.src/M000032.html +0 -18
data/doc/classes/BeStaticServerPages.src/M000033.html +0 -18
data/doc/classes/IncludedInMemcached.html +0 -199
data/doc/classes/IncludedInMemcached.src/M000015.html +0 -18
data/doc/classes/IncludedInMemcached.src/M000016.html +0 -18
data/doc/classes/IncludedInMemcached.src/M000017.html +0 -18
data/doc/classes/LoopingServlet.html +0 -137
data/doc/classes/LoopingServlet.src/M000037.html +0 -23
data/doc/classes/NextUrlsInSQS.html +0 -204
data/doc/classes/NextUrlsInSQS.src/M000018.html +0 -19
data/doc/classes/NextUrlsInSQS.src/M000019.html +0 -22
data/doc/classes/NextUrlsInSQS.src/M000020.html +0 -19
data/doc/classes/QueryServlet.html +0 -137
data/doc/classes/QueryServlet.src/M000038.html +0 -19
data/doc/classes/RobotRules.html +0 -175
data/doc/classes/RobotRules.src/M000034.html +0 -19
data/doc/classes/RobotRules.src/M000035.html +0 -67
data/doc/classes/RobotRules.src/M000036.html +0 -24
data/doc/classes/Spider.html +0 -170
data/doc/classes/Spider.src/M000029.html +0 -21
data/doc/classes/SpiderInstance.html +0 -345
data/doc/classes/SpiderInstance.src/M000021.html +0 -18
data/doc/classes/SpiderInstance.src/M000022.html +0 -22
data/doc/classes/SpiderInstance.src/M000023.html +0 -22
data/doc/classes/SpiderInstance.src/M000024.html +0 -24
data/doc/classes/SpiderInstance.src/M000025.html +0 -18
data/doc/classes/SpiderInstance.src/M000026.html +0 -18
data/doc/classes/SpiderInstance.src/M000027.html +0 -18
data/doc/classes/SpiderInstance.src/M000028.html +0 -18
data/doc/created.rid +0 -1
data/doc/files/README.html +0 -223
data/doc/files/lib/spider/included_in_memcached_rb.html +0 -142
data/doc/files/lib/spider/next_urls_in_sqs_rb.html +0 -144
data/doc/files/lib/spider/robot_rules_rb.html +0 -114
data/doc/files/lib/spider/spider_instance_rb.html +0 -117
data/doc/files/lib/spider_rb.html +0 -254
data/doc/files/spec/spec_helper_rb.html +0 -196
data/doc/files/spec/spec_helper_rb.src/M000001.html +0 -20
data/doc/files/spec/spec_helper_rb.src/M000002.html +0 -26
data/doc/files/spec/spec_helper_rb.src/M000003.html +0 -24
data/doc/files/spec/spec_helper_rb.src/M000004.html +0 -18
data/doc/files/spec/spec_helper_rb.src/M000005.html +0 -23
data/doc/files/spec/spider/included_in_memcached_spec_rb.html +0 -142
data/doc/files/spec/spider/included_in_memcached_spec_rb.src/M000006.html +0 -19
data/doc/files/spec/spider/included_in_memcached_spec_rb.src/M000007.html +0 -18
data/doc/files/spec/spider/spider_instance_spec_rb.html +0 -210
data/doc/files/spec/spider/spider_instance_spec_rb.src/M000008.html +0 -21
data/doc/files/spec/spider/spider_instance_spec_rb.src/M000009.html +0 -19
data/doc/files/spec/spider/spider_instance_spec_rb.src/M000010.html +0 -19
data/doc/files/spec/spider/spider_instance_spec_rb.src/M000011.html +0 -27
data/doc/files/spec/spider/spider_instance_spec_rb.src/M000012.html +0 -26
data/doc/files/spec/spider/spider_instance_spec_rb.src/M000013.html +0 -27
data/doc/files/spec/spider_spec_rb.html +0 -127
data/doc/files/spec/spider_spec_rb.src/M000014.html +0 -23
data/doc/fr_class_index.html +0 -34
data/doc/fr_file_index.html +0 -35
data/doc/fr_method_index.html +0 -64
data/doc/index.html +0 -24
data/doc/rdoc-style.css +0 -208

data/doc/classes/BeStaticServerPages.src/M000030.html DELETED

@@ -1,19 +0,0 @@
-<?xml version="1.0" encoding="iso-8859-1"?>
-<!DOCTYPE html
-     PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
-     "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
-<html>
-<head>
-  <title>new (BeStaticServerPages)</title>
-  <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
-  <link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
-</head>
-<body class="standalone-code">
-  <pre><span class="ruby-comment cmt"># File spec/spec_helper.rb, line 14</span>
-  <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">initialize</span>
-    <span class="ruby-ivar">@pages</span> = [<span class="ruby-value str">'http://localhost:8888/'</span>, <span class="ruby-value str">'http://localhost:8888/foo'</span>]
-    <span class="ruby-ivar">@actual</span> = <span class="ruby-keyword kw">nil</span>
-  <span class="ruby-keyword kw">end</span></pre>
-</body>
-</html>

data/doc/classes/BeStaticServerPages.src/M000031.html DELETED

@@ -1,19 +0,0 @@
-<?xml version="1.0" encoding="iso-8859-1"?>
-<!DOCTYPE html
-     PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
-     "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
-<html>
-<head>
-  <title>matches? (BeStaticServerPages)</title>
-  <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
-  <link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
-</head>
-<body class="standalone-code">
-  <pre><span class="ruby-comment cmt"># File spec/spec_helper.rb, line 21</span>
-  <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">matches?</span>(<span class="ruby-identifier">actual</span>)
-    <span class="ruby-ivar">@actual</span> = <span class="ruby-identifier">actual</span>
-    <span class="ruby-identifier">actual</span> <span class="ruby-operator">==</span> <span class="ruby-ivar">@pages</span>
-  <span class="ruby-keyword kw">end</span></pre>
-</body>
-</html>

data/doc/classes/BeStaticServerPages.src/M000032.html DELETED

@@ -1,18 +0,0 @@
-<?xml version="1.0" encoding="iso-8859-1"?>
-<!DOCTYPE html
-     PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
-     "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
-<html>
-<head>
-  <title>failure_message (BeStaticServerPages)</title>
-  <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
-  <link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
-</head>
-<body class="standalone-code">
-  <pre><span class="ruby-comment cmt"># File spec/spec_helper.rb, line 26</span>
-  <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">failure_message</span>
-    <span class="ruby-node">&quot;expected #{@pages.inspect}, got #{@actual.inspect}&quot;</span>
-  <span class="ruby-keyword kw">end</span></pre>
-</body>
-</html>

data/doc/classes/BeStaticServerPages.src/M000033.html DELETED

@@ -1,18 +0,0 @@
-<?xml version="1.0" encoding="iso-8859-1"?>
-<!DOCTYPE html
-     PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
-     "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
-<html>
-<head>
-  <title>description (BeStaticServerPages)</title>
-  <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
-  <link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
-</head>
-<body class="standalone-code">
-  <pre><span class="ruby-comment cmt"># File spec/spec_helper.rb, line 30</span>
-  <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">description</span>
-    <span class="ruby-node">&quot;be the pages returned by the static server (#{@pages.inspect})&quot;</span>
-  <span class="ruby-keyword kw">end</span></pre>
-</body>
-</html>

data/doc/classes/IncludedInMemcached.html DELETED

@@ -1,199 +0,0 @@
-<?xml version="1.0" encoding="iso-8859-1"?>
-<!DOCTYPE html
-     PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
-     "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
-<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
-<head>
-  <title>Class: IncludedInMemcached</title>
-  <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
-  <meta http-equiv="Content-Script-Type" content="text/javascript" />
-  <link rel="stylesheet" href=".././rdoc-style.css" type="text/css" media="screen" />
-  <script type="text/javascript">
-  // <![CDATA[
-  function popupCode( url ) {
-    window.open(url, "Code", "resizable=yes,scrollbars=yes,toolbar=no,status=no,height=150,width=400")
-  }
-  function toggleCode( id ) {
-    if ( document.getElementById )
-      elem = document.getElementById( id );
-    else if ( document.all )
-      elem = eval( "document.all." + id );
-    else
-      return false;
-    elemStyle = elem.style;
-    if ( elemStyle.display != "block" ) {
-      elemStyle.display = "block"
-    } else {
-      elemStyle.display = "none"
-    }
-    return true;
-  }
-  // Make codeblocks hidden by default
-  document.writeln( "<style type=\"text/css\">div.method-source-code { display: none }</style>" )
-  // ]]>
-  </script>
-</head>
-<body>
-    <div id="classHeader">
-        <table class="header-table">
-        <tr class="top-aligned-row">
-          <td><strong>Class</strong></td>
-          <td class="class-name-in-header">IncludedInMemcached</td>
-        </tr>
-        <tr class="top-aligned-row">
-            <td><strong>In:</strong></td>
-            <td>
-                <a href="../files/lib/spider/included_in_memcached_rb.html">
-                lib/spider/included_in_memcached.rb
-                </a>
-        <br />
-            </td>
-        </tr>
-        <tr class="top-aligned-row">
-            <td><strong>Parent:</strong></td>
-            <td>
-                Object
-            </td>
-        </tr>
-        </table>
-    </div>
-  <!-- banner header -->
-  <div id="bodyContent">
-  <div id="contextContent">
-    <div id="description">
-      <p>
-A specialized class using memcached to track items stored. It supports
-three operations: <a href="IncludedInMemcached.html#M000015">new</a>,
-&lt;&lt;, and <a href="IncludedInMemcached.html#M000017">include?</a> .
-Together these can be used to add items to the memcache, then determine
-whether the item has been added.
-</p>
-<p>
-To use it with <a href="Spider.html">Spider</a> use the
-check_already_seen_with method:
-</p>
-<pre>
- Spider.start_at('http://example.com/') do |s|
-   s.check_already_seen_with IncludedInMemcached.new('localhost:11211')
- end
-</pre>
-    </div>
-   </div>
-    <div id="method-list">
-      <h3 class="section-bar">Methods</h3>
-      <div class="name-list">
-      <a href="#M000016">&lt;&lt;</a>&nbsp;&nbsp;
-      <a href="#M000017">include?</a>&nbsp;&nbsp;
-      <a href="#M000015">new</a>&nbsp;&nbsp;
-      </div>
-    </div>
-  </div>
-    <!-- if includes -->
-    <div id="section">
-    <!-- if method_list -->
-    <div id="methods">
-      <h3 class="section-bar">Public Class methods</h3>
-      <div id="method-M000015" class="method-detail">
-        <a name="M000015"></a>
-        <div class="method-heading">
-          <a href="IncludedInMemcached.src/M000015.html" target="Code" class="method-signature"
-            onclick="popupCode('IncludedInMemcached.src/M000015.html');return false;">
-          <span class="method-name">new</span><span class="method-args">(*a)</span>
-          </a>
-        </div>
-        <div class="method-description">
-          <p>
-Construct a <a href="IncludedInMemcached.html#M000015">new</a> <a
-href="IncludedInMemcached.html">IncludedInMemcached</a> instance. All
-arguments here are passed to MemCache (part of the memcache-client gem).
-</p>
-        </div>
-      </div>
-      <h3 class="section-bar">Public Instance methods</h3>
-      <div id="method-M000016" class="method-detail">
-        <a name="M000016"></a>
-        <div class="method-heading">
-          <a href="IncludedInMemcached.src/M000016.html" target="Code" class="method-signature"
-            onclick="popupCode('IncludedInMemcached.src/M000016.html');return false;">
-          <span class="method-name">&lt;&lt;</span><span class="method-args">(v)</span>
-          </a>
-        </div>
-        <div class="method-description">
-          <p>
-Add an item to the memcache.
-</p>
-        </div>
-      </div>
-      <div id="method-M000017" class="method-detail">
-        <a name="M000017"></a>
-        <div class="method-heading">
-          <a href="IncludedInMemcached.src/M000017.html" target="Code" class="method-signature"
-            onclick="popupCode('IncludedInMemcached.src/M000017.html');return false;">
-          <span class="method-name">include?</span><span class="method-args">(v)</span>
-          </a>
-        </div>
-        <div class="method-description">
-          <p>
-True if the item is in the memcache.
-</p>
-        </div>
-      </div>
-    </div>
-  </div>
-<div id="validator-badges">
-  <p><small><a href="http://validator.w3.org/check/referer">[Validate]</a></small></p>
-</div>
-</body>
-</html>

data/doc/classes/IncludedInMemcached.src/M000015.html DELETED

@@ -1,18 +0,0 @@
-<?xml version="1.0" encoding="iso-8859-1"?>
-<!DOCTYPE html
-     PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
-     "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
-<html>
-<head>
-  <title>new (IncludedInMemcached)</title>
-  <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
-  <link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
-</head>
-<body class="standalone-code">
-  <pre><span class="ruby-comment cmt"># File lib/spider/included_in_memcached.rb, line 40</span>
-  <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">initialize</span>(<span class="ruby-operator">*</span><span class="ruby-identifier">a</span>)
-    <span class="ruby-ivar">@c</span> = <span class="ruby-constant">MemCache</span>.<span class="ruby-identifier">new</span>(<span class="ruby-operator">*</span><span class="ruby-identifier">a</span>)
-  <span class="ruby-keyword kw">end</span></pre>
-</body>
-</html>

data/doc/classes/IncludedInMemcached.src/M000016.html DELETED

@@ -1,18 +0,0 @@
-<?xml version="1.0" encoding="iso-8859-1"?>
-<!DOCTYPE html
-     PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
-     "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
-<html>
-<head>
-  <title>&lt;&lt; (IncludedInMemcached)</title>
-  <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
-  <link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
-</head>
-<body class="standalone-code">
-  <pre><span class="ruby-comment cmt"># File lib/spider/included_in_memcached.rb, line 45</span>
-  <span class="ruby-keyword kw">def</span> <span class="ruby-operator">&lt;&lt;</span>(<span class="ruby-identifier">v</span>)
-    <span class="ruby-ivar">@c</span>.<span class="ruby-identifier">add</span>(<span class="ruby-identifier">v</span>.<span class="ruby-identifier">to_s</span>, <span class="ruby-identifier">v</span>)
-  <span class="ruby-keyword kw">end</span></pre>
-</body>
-</html>

data/doc/classes/IncludedInMemcached.src/M000017.html DELETED

@@ -1,18 +0,0 @@
-<?xml version="1.0" encoding="iso-8859-1"?>
-<!DOCTYPE html
-     PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
-     "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
-<html>
-<head>
-  <title>include? (IncludedInMemcached)</title>
-  <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
-  <link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
-</head>
-<body class="standalone-code">
-  <pre><span class="ruby-comment cmt"># File lib/spider/included_in_memcached.rb, line 50</span>
-  <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">include?</span>(<span class="ruby-identifier">v</span>)
-    <span class="ruby-ivar">@c</span>.<span class="ruby-identifier">get</span>(<span class="ruby-identifier">v</span>.<span class="ruby-identifier">to_s</span>) <span class="ruby-operator">==</span> <span class="ruby-identifier">v</span>
-  <span class="ruby-keyword kw">end</span></pre>
-</body>
-</html>

data/doc/classes/LoopingServlet.html DELETED

@@ -1,137 +0,0 @@
-<?xml version="1.0" encoding="iso-8859-1"?>
-<!DOCTYPE html
-     PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
-     "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
-<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
-<head>
-  <title>Class: LoopingServlet</title>
-  <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
-  <meta http-equiv="Content-Script-Type" content="text/javascript" />
-  <link rel="stylesheet" href=".././rdoc-style.css" type="text/css" media="screen" />
-  <script type="text/javascript">
-  // <![CDATA[
-  function popupCode( url ) {
-    window.open(url, "Code", "resizable=yes,scrollbars=yes,toolbar=no,status=no,height=150,width=400")
-  }
-  function toggleCode( id ) {
-    if ( document.getElementById )
-      elem = document.getElementById( id );
-    else if ( document.all )
-      elem = eval( "document.all." + id );
-    else
-      return false;
-    elemStyle = elem.style;
-    if ( elemStyle.display != "block" ) {
-      elemStyle.display = "block"
-    } else {
-      elemStyle.display = "none"
-    }
-    return true;
-  }
-  // Make codeblocks hidden by default
-  document.writeln( "<style type=\"text/css\">div.method-source-code { display: none }</style>" )
-  // ]]>
-  </script>
-</head>
-<body>
-    <div id="classHeader">
-        <table class="header-table">
-        <tr class="top-aligned-row">
-          <td><strong>Class</strong></td>
-          <td class="class-name-in-header">LoopingServlet</td>
-        </tr>
-        <tr class="top-aligned-row">
-            <td><strong>In:</strong></td>
-            <td>
-                <a href="../files/spec/spec_helper_rb.html">
-                spec/spec_helper.rb
-                </a>
-        <br />
-            </td>
-        </tr>
-        <tr class="top-aligned-row">
-            <td><strong>Parent:</strong></td>
-            <td>
-                WEBrick::HTTPServlet::AbstractServlet
-            </td>
-        </tr>
-        </table>
-    </div>
-  <!-- banner header -->
-  <div id="bodyContent">
-  <div id="contextContent">
-   </div>
-    <div id="method-list">
-      <h3 class="section-bar">Methods</h3>
-      <div class="name-list">
-      <a href="#M000037">do_GET</a>&nbsp;&nbsp;
-      </div>
-    </div>
-  </div>
-    <!-- if includes -->
-    <div id="section">
-    <!-- if method_list -->
-    <div id="methods">
-      <h3 class="section-bar">Public Instance methods</h3>
-      <div id="method-M000037" class="method-detail">
-        <a name="M000037"></a>
-        <div class="method-heading">
-          <a href="LoopingServlet.src/M000037.html" target="Code" class="method-signature"
-            onclick="popupCode('LoopingServlet.src/M000037.html');return false;">
-          <span class="method-name">do_GET</span><span class="method-args">(req, res)</span>
-          </a>
-        </div>
-        <div class="method-description">
-        </div>
-      </div>
-    </div>
-  </div>
-<div id="validator-badges">
-  <p><small><a href="http://validator.w3.org/check/referer">[Validate]</a></small></p>
-</div>
-</body>
-</html>