spider 0.3.0 → 0.4.0

data/spec/included_in_memcached_spec.rb

@@ -0,0 +1,44 @@
+require 'rubygems'
+require 'spec'
+
+def before_specing_memcached
+  require File.dirname(__FILE__)+'/../lib/included_in_memcached'
+  system('memcached -d -P /tmp/spider-memcached.pid')
+end
+
+def after_specing_memcached
+  system('kill -KILL `cat /tmp/spider-memcached.pid`')
+end
+
+Spec::Runner.configure { |c| c.mock_with :mocha }
+
+describe 'Object to halt cycles' do
+  before do
+    before_specing_memcached
+  end
+
+  it 'should understand <<' do
+    c = IncludedInMemcached.new('localhost:11211')
+    c.should respond_to(:<<)
+  end
+
+  it 'should understand included?' do
+    c = IncludedInMemcached.new('localhost:11211')
+    c.should respond_to(:include?)
+  end
+
+  it 'should produce false if the object is not included' do
+    c = IncludedInMemcached.new('localhost:11211')
+    c.include?('a').should be_false
+  end
+
+  it 'should produce true if the object is included' do
+    c = IncludedInMemcached.new('localhost:11211')
+    c << 'a'
+    c.include?('a').should be_true
+  end
+  
+  after do
+    after_specing_memcached
+  end
+end

data/spec/spider_instance_spec.rb

@@ -3,6 +3,7 @@ require 'spec'
 require 'webrick'
 require 'webrick/https'
 require File.dirname(__FILE__)+'/../lib/spider'
+require File.dirname(__FILE__)+'/../lib/included_in_memcached'
 
 Spec::Runner.configure { |c| c.mock_with :mocha }
 
@@ -13,6 +14,21 @@ class QueryServlet < WEBrick::HTTPServlet::AbstractServlet
   end
 end
 
+class LoopingServlet < WEBrick::HTTPServlet::AbstractServlet
+  def do_GET(req, res)
+    res['Content-type'] = 'text/html'
+    if req.path == '/foo'
+      res.body = <<-END
+      <a href="/">a</a>
+      END
+    else
+      res.body = <<-END
+      <a href="/foo">b</a>
+      END
+    end
+  end
+end
+
 def null_logger
   l = stub
   [:log, :fatal, :error, :warn , :info, :debug].each do |k|
@@ -23,6 +39,18 @@ def null_logger
 end
 
 describe 'SpiderInstance' do
+  it 'should prevent cycles with an IncludedInMemcached' do
+    system('memcached -d -P /tmp/spider-memcached.pid')
+    cacher = IncludedInMemcached.new('localhost:11211')
+    it_should_prevent_cycles_with(cacher)
+    system('kill -KILL `cat /tmp/spider-memcached.pid`')
+  end
+
+  it 'should prevent cycles with an Array' do
+    cacher = Array.new
+    it_should_prevent_cycles_with(cacher)
+  end
+
   it 'should call the "setup" callback before loading the Web page' do
     mock_successful_http
     @on_called = false
@@ -70,8 +98,6 @@ describe 'SpiderInstance' do
     si.start!
   end
 
-  it 'should allow for a proxy' # fill in more
-
   it 'should call the :every callback with the current URL, the response, and the prior URL' do
     mock_successful_http
     callback_arguments_on(:every)
@@ -146,8 +172,6 @@ describe 'SpiderInstance' do
     @page_called.should be_true
   end
 
-  it 'should maintain the entire graph within some external object (or memory, or memcached)'
-
   it 'should skip URLs when allowable_url? is false' do
     u = 'http://example.com/'
     u_p = URI.parse(u)
@@ -382,4 +406,22 @@ describe 'SpiderInstance' do
     end
     si.start!
   end
+
+  def it_should_prevent_cycles_with(cacher)
+    u = 'http://localhost:8888/'
+    u_p = URI.parse(u)
+    u2 = 'http://localhost:8888/foo'
+    u_p2 = URI.parse(u2)
+
+    server = WEBrick::HTTPServer.new(:Port => 8888, :Logger => null_logger,
+                                     :AccessLog => [])
+    server.mount('/', LoopingServlet)
+    Thread.new {server.start}
+
+    si = SpiderInstance.new(nil => [u])
+    si.check_already_seen_with cacher
+    si.start!
+
+    server.shutdown
+  end
 end

data/spider.gemspec

@@ -13,5 +13,5 @@ spec = Gem::Specification.new do |s|
 A Web spidering library: handles robots.txt, scraping, finding more
 links, and doing it all over again.
 EOF
-  s.version = '0.3.0'
+  s.version = '0.4.0'
 end

metadata

@@ -3,8 +3,8 @@ rubygems_version: 0.9.4
 specification_version: 1
 name: spider
 version: !ruby/object:Gem::Version 
-  version: 0.3.0
-date: 2007-11-01 00:00:00 -04:00
+  version: 0.4.0
+date: 2007-11-02 00:00:00 -04:00
 summary: A Web spidering library
 require_paths: 
 - lib
@@ -34,16 +34,13 @@ files:
 - doc/files
 - doc/files/lib
 - doc/files/lib/spider_rb.html
+- doc/files/lib/spider_instance_rb.html
+- doc/files/lib/included_in_memcached_rb.html
 - doc/files/README.html
 - doc/classes
+- doc/classes/IncludedInMemcached.html
 - doc/classes/SpiderInstance.html
 - doc/classes/Spider.html
-- doc/classes/Net.html
-- doc/classes/NilClass.html
-- doc/classes/Net
-- doc/classes/Net/HTTPRedirection.html
-- doc/classes/Net/HTTPSuccess.html
-- doc/classes/Net/HTTPResponse.html
 - doc/fr_file_index.html
 - doc/fr_class_index.html
 - doc/fr_method_index.html
@@ -51,6 +48,7 @@ files:
 - doc/created.rid
 - spec
 - spec/spider_spec.rb
+- spec/included_in_memcached_spec.rb
 - spec/spider_instance_spec.rb
 - README
 - spider.gemspec
@@ -58,6 +56,8 @@ files:
 - lib
 - lib/spider.rb
 - lib/robot_rules.rb
+- lib/spider_instance.rb
+- lib/included_in_memcached.rb
 - test_server
 - test_server/server1
 - test_server/server1/page1.html

data/doc/classes/Net/HTTPRedirection.html

@@ -1,144 +0,0 @@
-<?xml version="1.0" encoding="iso-8859-1"?>
-<!DOCTYPE html 
-     PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
-     "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
-
-<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
-<head>
-  <title>Class: Net::HTTPRedirection</title>
-  <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
-  <meta http-equiv="Content-Script-Type" content="text/javascript" />
-  <link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
-  <script type="text/javascript">
-  // <![CDATA[
-
-  function popupCode( url ) {
-    window.open(url, "Code", "resizable=yes,scrollbars=yes,toolbar=no,status=no,height=150,width=400")
-  }
-
-  function toggleCode( id ) {
-    if ( document.getElementById )
-      elem = document.getElementById( id );
-    else if ( document.all )
-      elem = eval( "document.all." + id );
-    else
-      return false;
-
-    elemStyle = elem.style;
-    
-    if ( elemStyle.display != "block" ) {
-      elemStyle.display = "block"
-    } else {
-      elemStyle.display = "none"
-    }
-
-    return true;
-  }
-  
-  // Make codeblocks hidden by default
-  document.writeln( "<style type=\"text/css\">div.method-source-code { display: none }</style>" )
-  
-  // ]]>
-  </script>
-
-</head>
-<body>
-
-
-
-    <div id="classHeader">
-        <table class="header-table">
-        <tr class="top-aligned-row">
-          <td><strong>Class</strong></td>
-          <td class="class-name-in-header">Net::HTTPRedirection</td>
-        </tr>
-        <tr class="top-aligned-row">
-            <td><strong>In:</strong></td>
-            <td>
-                <a href="../../files/lib/spider_rb.html">
-                lib/spider.rb
-                </a>
-        <br />
-            </td>
-        </tr>
-
-        <tr class="top-aligned-row">
-            <td><strong>Parent:</strong></td>
-            <td>
-                Object
-            </td>
-        </tr>
-        </table>
-    </div>
-  <!-- banner header -->
-
-  <div id="bodyContent">
-
-
-
-  <div id="contextContent">
-
-
-
-   </div>
-
-    <div id="method-list">
-      <h3 class="section-bar">Methods</h3>
-
-      <div class="name-list">
-      <a href="#M000008">redirect?</a>&nbsp;&nbsp;
-      </div>
-    </div>
-
-  </div>
-
-
-    <!-- if includes -->
-
-    <div id="section">
-
-
-
-
-
-      
-
-
-    <!-- if method_list -->
-    <div id="methods">
-      <h3 class="section-bar">Public Instance methods</h3>
-
-      <div id="method-M000008" class="method-detail">
-        <a name="M000008"></a>
-
-        <div class="method-heading">
-          <a href="#M000008" class="method-signature">
-          <span class="method-name">redirect?</span><span class="method-args">()</span>
-          </a>
-        </div>
-      
-        <div class="method-description">
-          <p><a class="source-toggle" href="#"
-            onclick="toggleCode('M000008-source');return false;">[Source]</a></p>
-          <div class="method-source-code" id="M000008-source">
-<pre>
-<span class="ruby-comment cmt"># File lib/spider.rb, line 41</span>
-    <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">redirect?</span>; <span class="ruby-keyword kw">true</span>; <span class="ruby-keyword kw">end</span>
-</pre>
-          </div>
-        </div>
-      </div>
-
-
-    </div>
-
-
-  </div>
-
-
-<div id="validator-badges">
-  <p><small><a href="http://validator.w3.org/check/referer">[Validate]</a></small></p>
-</div>
-
-</body>
-</html>

data/doc/classes/Net/HTTPResponse.html

@@ -1,166 +0,0 @@
-<?xml version="1.0" encoding="iso-8859-1"?>
-<!DOCTYPE html 
-     PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
-     "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
-
-<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
-<head>
-  <title>Class: Net::HTTPResponse</title>
-  <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
-  <meta http-equiv="Content-Script-Type" content="text/javascript" />
-  <link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
-  <script type="text/javascript">
-  // <![CDATA[
-
-  function popupCode( url ) {
-    window.open(url, "Code", "resizable=yes,scrollbars=yes,toolbar=no,status=no,height=150,width=400")
-  }
-
-  function toggleCode( id ) {
-    if ( document.getElementById )
-      elem = document.getElementById( id );
-    else if ( document.all )
-      elem = eval( "document.all." + id );
-    else
-      return false;
-
-    elemStyle = elem.style;
-    
-    if ( elemStyle.display != "block" ) {
-      elemStyle.display = "block"
-    } else {
-      elemStyle.display = "none"
-    }
-
-    return true;
-  }
-  
-  // Make codeblocks hidden by default
-  document.writeln( "<style type=\"text/css\">div.method-source-code { display: none }</style>" )
-  
-  // ]]>
-  </script>
-
-</head>
-<body>
-
-
-
-    <div id="classHeader">
-        <table class="header-table">
-        <tr class="top-aligned-row">
-          <td><strong>Class</strong></td>
-          <td class="class-name-in-header">Net::HTTPResponse</td>
-        </tr>
-        <tr class="top-aligned-row">
-            <td><strong>In:</strong></td>
-            <td>
-                <a href="../../files/lib/spider_rb.html">
-                lib/spider.rb
-                </a>
-        <br />
-            </td>
-        </tr>
-
-        <tr class="top-aligned-row">
-            <td><strong>Parent:</strong></td>
-            <td>
-                Object
-            </td>
-        </tr>
-        </table>
-    </div>
-  <!-- banner header -->
-
-  <div id="bodyContent">
-
-
-
-  <div id="contextContent">
-
-
-
-   </div>
-
-    <div id="method-list">
-      <h3 class="section-bar">Methods</h3>
-
-      <div class="name-list">
-      <a href="#M000011">redirect?</a>&nbsp;&nbsp;
-      <a href="#M000010">success?</a>&nbsp;&nbsp;
-      </div>
-    </div>
-
-  </div>
-
-
-    <!-- if includes -->
-
-    <div id="section">
-
-
-
-
-
-      
-
-
-    <!-- if method_list -->
-    <div id="methods">
-      <h3 class="section-bar">Public Instance methods</h3>
-
-      <div id="method-M000011" class="method-detail">
-        <a name="M000011"></a>
-
-        <div class="method-heading">
-          <a href="#M000011" class="method-signature">
-          <span class="method-name">redirect?</span><span class="method-args">()</span>
-          </a>
-        </div>
-      
-        <div class="method-description">
-          <p><a class="source-toggle" href="#"
-            onclick="toggleCode('M000011-source');return false;">[Source]</a></p>
-          <div class="method-source-code" id="M000011-source">
-<pre>
-<span class="ruby-comment cmt"># File lib/spider.rb, line 35</span>
-    <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">redirect?</span>; <span class="ruby-keyword kw">false</span>; <span class="ruby-keyword kw">end</span>
-</pre>
-          </div>
-        </div>
-      </div>
-
-      <div id="method-M000010" class="method-detail">
-        <a name="M000010"></a>
-
-        <div class="method-heading">
-          <a href="#M000010" class="method-signature">
-          <span class="method-name">success?</span><span class="method-args">()</span>
-          </a>
-        </div>
-      
-        <div class="method-description">
-          <p><a class="source-toggle" href="#"
-            onclick="toggleCode('M000010-source');return false;">[Source]</a></p>
-          <div class="method-source-code" id="M000010-source">
-<pre>
-<span class="ruby-comment cmt"># File lib/spider.rb, line 34</span>
-    <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">success?</span>; <span class="ruby-keyword kw">false</span>; <span class="ruby-keyword kw">end</span>
-</pre>
-          </div>
-        </div>
-      </div>
-
-
-    </div>
-
-
-  </div>
-
-
-<div id="validator-badges">
-  <p><small><a href="http://validator.w3.org/check/referer">[Validate]</a></small></p>
-</div>
-
-</body>
-</html>