spider 0.4.3 → 0.4.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (66) hide show
  1. data/CHANGES +6 -0
  2. data/README +3 -3
  3. data/doc/classes/BeStaticServerPages.html +197 -0
  4. data/doc/classes/BeStaticServerPages.src/M000030.html +19 -0
  5. data/doc/classes/BeStaticServerPages.src/M000031.html +19 -0
  6. data/doc/classes/BeStaticServerPages.src/M000032.html +18 -0
  7. data/doc/classes/BeStaticServerPages.src/M000033.html +18 -0
  8. data/doc/classes/IncludedInMemcached.html +18 -45
  9. data/doc/classes/IncludedInMemcached.src/M000015.html +18 -0
  10. data/doc/classes/IncludedInMemcached.src/M000016.html +18 -0
  11. data/doc/classes/IncludedInMemcached.src/M000017.html +18 -0
  12. data/doc/classes/LoopingServlet.html +137 -0
  13. data/doc/classes/LoopingServlet.src/M000037.html +23 -0
  14. data/doc/classes/NextUrlsInSQS.html +204 -0
  15. data/doc/classes/NextUrlsInSQS.src/M000018.html +19 -0
  16. data/doc/classes/NextUrlsInSQS.src/M000019.html +22 -0
  17. data/doc/classes/NextUrlsInSQS.src/M000020.html +19 -0
  18. data/doc/classes/QueryServlet.html +137 -0
  19. data/doc/classes/QueryServlet.src/M000038.html +19 -0
  20. data/doc/classes/RobotRules.html +175 -0
  21. data/doc/classes/RobotRules.src/M000034.html +19 -0
  22. data/doc/classes/RobotRules.src/M000035.html +67 -0
  23. data/doc/classes/RobotRules.src/M000036.html +24 -0
  24. data/doc/classes/Spider.html +5 -17
  25. data/doc/classes/Spider.src/M000029.html +21 -0
  26. data/doc/classes/SpiderInstance.html +72 -108
  27. data/doc/classes/SpiderInstance.src/M000021.html +18 -0
  28. data/doc/classes/SpiderInstance.src/M000022.html +22 -0
  29. data/doc/classes/SpiderInstance.src/M000023.html +22 -0
  30. data/doc/classes/SpiderInstance.src/M000024.html +24 -0
  31. data/doc/classes/SpiderInstance.src/M000025.html +18 -0
  32. data/doc/classes/SpiderInstance.src/M000026.html +18 -0
  33. data/doc/classes/SpiderInstance.src/M000027.html +18 -0
  34. data/doc/classes/SpiderInstance.src/M000028.html +18 -0
  35. data/doc/created.rid +1 -1
  36. data/doc/files/lib/spider/included_in_memcached_rb.html +29 -1
  37. data/doc/files/lib/spider/next_urls_in_sqs_rb.html +144 -0
  38. data/doc/files/lib/spider/robot_rules_rb.html +114 -0
  39. data/doc/files/lib/spider/spider_instance_rb.html +1 -2
  40. data/doc/files/lib/spider_rb.html +40 -9
  41. data/doc/files/spec/spec_helper_rb.html +196 -0
  42. data/doc/files/spec/spec_helper_rb.src/M000001.html +20 -0
  43. data/doc/files/spec/spec_helper_rb.src/M000002.html +26 -0
  44. data/doc/files/spec/spec_helper_rb.src/M000003.html +24 -0
  45. data/doc/files/spec/spec_helper_rb.src/M000004.html +18 -0
  46. data/doc/files/spec/spec_helper_rb.src/M000005.html +23 -0
  47. data/doc/files/spec/spider/included_in_memcached_spec_rb.html +142 -0
  48. data/doc/files/spec/spider/included_in_memcached_spec_rb.src/M000006.html +19 -0
  49. data/doc/files/spec/spider/included_in_memcached_spec_rb.src/M000007.html +18 -0
  50. data/doc/files/spec/spider/spider_instance_spec_rb.html +210 -0
  51. data/doc/files/spec/spider/spider_instance_spec_rb.src/M000008.html +21 -0
  52. data/doc/files/spec/spider/spider_instance_spec_rb.src/M000009.html +19 -0
  53. data/doc/files/spec/spider/spider_instance_spec_rb.src/M000010.html +19 -0
  54. data/doc/files/spec/spider/spider_instance_spec_rb.src/M000011.html +27 -0
  55. data/doc/files/spec/spider/spider_instance_spec_rb.src/M000012.html +26 -0
  56. data/doc/files/spec/spider/spider_instance_spec_rb.src/M000013.html +27 -0
  57. data/doc/files/spec/spider_spec_rb.html +127 -0
  58. data/doc/files/spec/spider_spec_rb.src/M000014.html +23 -0
  59. data/doc/fr_class_index.html +5 -0
  60. data/doc/fr_file_index.html +6 -1
  61. data/doc/fr_method_index.html +38 -11
  62. data/doc/index.html +1 -1
  63. data/lib/spider/spider_instance.rb +15 -7
  64. data/spider.gemspec +1 -1
  65. metadata +84 -22
  66. data/lib/test.rb +0 -27
data/doc/classes/SpiderInstance.src/M000026.html ADDED
@@ -0,0 +1,18 @@
1
+ <?xml version="1.0" encoding="iso-8859-1"?>
2
+ <!DOCTYPE html
3
+ PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
4
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
5
+
6
+ <html>
7
+ <head>
8
+ <title>teardown (SpiderInstance)</title>
9
+ <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
10
+ <link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
11
+ </head>
12
+ <body class="standalone-code">
13
+ <pre><span class="ruby-comment cmt"># File lib/spider/spider_instance.rb, line 164</span>
14
+ <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">teardown</span>(<span class="ruby-identifier">p</span> = <span class="ruby-keyword kw">nil</span>, <span class="ruby-operator">&amp;</span><span class="ruby-identifier">block</span>)
15
+ <span class="ruby-ivar">@teardown</span> = <span class="ruby-identifier">p</span> <span class="ruby-value">? </span><span class="ruby-identifier">p</span> <span class="ruby-operator">:</span> <span class="ruby-identifier">block</span>
16
+ <span class="ruby-keyword kw">end</span></pre>
17
+ </body>
18
+ </html>
data/doc/classes/SpiderInstance.src/M000027.html ADDED
@@ -0,0 +1,18 @@
1
+ <?xml version="1.0" encoding="iso-8859-1"?>
2
+ <!DOCTYPE html
3
+ PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
4
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
5
+
6
+ <html>
7
+ <head>
8
+ <title>headers (SpiderInstance)</title>
9
+ <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
10
+ <link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
11
+ </head>
12
+ <body class="standalone-code">
13
+ <pre><span class="ruby-comment cmt"># File lib/spider/spider_instance.rb, line 170</span>
14
+ <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">headers</span>
15
+ <span class="ruby-constant">HeaderSetter</span>.<span class="ruby-identifier">new</span>(<span class="ruby-keyword kw">self</span>)
16
+ <span class="ruby-keyword kw">end</span></pre>
17
+ </body>
18
+ </html>
data/doc/classes/SpiderInstance.src/M000028.html ADDED
@@ -0,0 +1,18 @@
1
+ <?xml version="1.0" encoding="iso-8859-1"?>
2
+ <!DOCTYPE html
3
+ PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
4
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
5
+
6
+ <html>
7
+ <head>
8
+ <title>clear_headers (SpiderInstance)</title>
9
+ <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
10
+ <link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
11
+ </head>
12
+ <body class="standalone-code">
13
+ <pre><span class="ruby-comment cmt"># File lib/spider/spider_instance.rb, line 182</span>
14
+ <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">clear_headers</span>
15
+ <span class="ruby-ivar">@headers</span> = {}
16
+ <span class="ruby-keyword kw">end</span></pre>
17
+ </body>
18
+ </html>
data/doc/created.rid CHANGED
@@ -1 +1 @@
1
- Sat, 10 Nov 2007 00:25:19 -0500
1
+ Thu, 21 May 2009 15:42:01 +0000
data/doc/files/lib/spider/included_in_memcached_rb.html CHANGED
@@ -56,7 +56,7 @@
56
56
  </tr>
57
57
  <tr class="top-aligned-row">
58
58
  <td><strong>Last Update:</strong></td>
59
- <td>Sat Nov 10 00:24:11 -0500 2007</td>
59
+ <td>Thu May 21 13:19:06 +0000 2009</td>
60
60
  </tr>
61
61
  </table>
62
62
  </div>
@@ -71,6 +71,34 @@
71
71
  <div id="description">
72
72
  <p>
73
73
  Use memcached to track cycles.
74
+ </p>
75
+ <p>
76
+ Copyright 2007 Mike Burns Redistribution and use in source and binary
77
+ forms, with or without modification, are permitted provided that the
78
+ following conditions are met:
79
+ </p>
80
+ <pre>
81
+ * Redistributions of source code must retain the above copyright
82
+ notice, this list of conditions and the following disclaimer.
83
+ * Redistributions in binary form must reproduce the above copyright
84
+ notice, this list of conditions and the following disclaimer in the
85
+ documentation and/or other materials provided with the distribution.
86
+ * Neither the name Mike Burns nor the
87
+ names of his contributors may be used to endorse or promote products
88
+ derived from this software without specific prior written permission.
89
+ </pre>
90
+ <p>
91
+ THIS SOFTWARE IS PROVIDED BY Mike Burns ``AS IS&#8217;&#8217; AND ANY
92
+ EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
93
+ WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
94
+ DISCLAIMED. IN NO EVENT SHALL Mike Burns BE LIABLE FOR ANY DIRECT,
95
+ INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
96
+ (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
97
+ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
98
+ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
99
+ LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
100
+ OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
101
+ DAMAGE.
74
102
  </p>
75
103
 
76
104
  </div>
data/doc/files/lib/spider/next_urls_in_sqs_rb.html ADDED
@@ -0,0 +1,144 @@
1
+ <?xml version="1.0" encoding="iso-8859-1"?>
2
+ <!DOCTYPE html
3
+ PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
4
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
5
+
6
+ <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
7
+ <head>
8
+ <title>File: next_urls_in_sqs.rb</title>
9
+ <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
10
+ <meta http-equiv="Content-Script-Type" content="text/javascript" />
11
+ <link rel="stylesheet" href="../../.././rdoc-style.css" type="text/css" media="screen" />
12
+ <script type="text/javascript">
13
+ // <![CDATA[
14
+
15
+ function popupCode( url ) {
16
+ window.open(url, "Code", "resizable=yes,scrollbars=yes,toolbar=no,status=no,height=150,width=400")
17
+ }
18
+
19
+ function toggleCode( id ) {
20
+ if ( document.getElementById )
21
+ elem = document.getElementById( id );
22
+ else if ( document.all )
23
+ elem = eval( "document.all." + id );
24
+ else
25
+ return false;
26
+
27
+ elemStyle = elem.style;
28
+
29
+ if ( elemStyle.display != "block" ) {
30
+ elemStyle.display = "block"
31
+ } else {
32
+ elemStyle.display = "none"
33
+ }
34
+
35
+ return true;
36
+ }
37
+
38
+ // Make codeblocks hidden by default
39
+ document.writeln( "<style type=\"text/css\">div.method-source-code { display: none }</style>" )
40
+
41
+ // ]]>
42
+ </script>
43
+
44
+ </head>
45
+ <body>
46
+
47
+
48
+
49
+ <div id="fileHeader">
50
+ <h1>next_urls_in_sqs.rb</h1>
51
+ <table class="header-table">
52
+ <tr class="top-aligned-row">
53
+ <td><strong>Path:</strong></td>
54
+ <td>lib/spider/next_urls_in_sqs.rb
55
+ </td>
56
+ </tr>
57
+ <tr class="top-aligned-row">
58
+ <td><strong>Last Update:</strong></td>
59
+ <td>Thu May 21 13:19:06 +0000 2009</td>
60
+ </tr>
61
+ </table>
62
+ </div>
63
+ <!-- banner header -->
64
+
65
+ <div id="bodyContent">
66
+
67
+
68
+
69
+ <div id="contextContent">
70
+
71
+ <div id="description">
72
+ <p>
73
+ Use AmazonSQS to track nodes to visit.
74
+ </p>
75
+ <p>
76
+ Copyright 2008 John Nagro Redistribution and use in source and binary
77
+ forms, with or without modification, are permitted provided that the
78
+ following conditions are met:
79
+ </p>
80
+ <pre>
81
+ * Redistributions of source code must retain the above copyright
82
+ notice, this list of conditions and the following disclaimer.
83
+ * Redistributions in binary form must reproduce the above copyright
84
+ notice, this list of conditions and the following disclaimer in the
85
+ documentation and/or other materials provided with the distribution.
86
+ * Neither the name Mike Burns nor the
87
+ names of his contributors may be used to endorse or promote products
88
+ derived from this software without specific prior written permission.
89
+ </pre>
90
+ <p>
91
+ THIS SOFTWARE IS PROVIDED BY Mike Burns ``AS IS&#8217;&#8217; AND ANY
92
+ EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
93
+ WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
94
+ DISCLAIMED. IN NO EVENT SHALL Mike Burns BE LIABLE FOR ANY DIRECT,
95
+ INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
96
+ (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
97
+ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
98
+ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
99
+ LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
100
+ OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
101
+ DAMAGE.
102
+ </p>
103
+
104
+ </div>
105
+
106
+ <div id="requires-list">
107
+ <h3 class="section-bar">Required files</h3>
108
+
109
+ <div class="name-list">
110
+ rubygems&nbsp;&nbsp;
111
+ right_aws&nbsp;&nbsp;
112
+ yaml&nbsp;&nbsp;
113
+ </div>
114
+ </div>
115
+
116
+ </div>
117
+
118
+
119
+ </div>
120
+
121
+
122
+ <!-- if includes -->
123
+
124
+ <div id="section">
125
+
126
+
127
+
128
+
129
+
130
+
131
+
132
+
133
+ <!-- if method_list -->
134
+
135
+
136
+ </div>
137
+
138
+
139
+ <div id="validator-badges">
140
+ <p><small><a href="http://validator.w3.org/check/referer">[Validate]</a></small></p>
141
+ </div>
142
+
143
+ </body>
144
+ </html>
data/doc/files/lib/spider/robot_rules_rb.html ADDED
@@ -0,0 +1,114 @@
1
+ <?xml version="1.0" encoding="iso-8859-1"?>
2
+ <!DOCTYPE html
3
+ PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
4
+ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
5
+
6
+ <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
7
+ <head>
8
+ <title>File: robot_rules.rb</title>
9
+ <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
10
+ <meta http-equiv="Content-Script-Type" content="text/javascript" />
11
+ <link rel="stylesheet" href="../../.././rdoc-style.css" type="text/css" media="screen" />
12
+ <script type="text/javascript">
13
+ // <![CDATA[
14
+
15
+ function popupCode( url ) {
16
+ window.open(url, "Code", "resizable=yes,scrollbars=yes,toolbar=no,status=no,height=150,width=400")
17
+ }
18
+
19
+ function toggleCode( id ) {
20
+ if ( document.getElementById )
21
+ elem = document.getElementById( id );
22
+ else if ( document.all )
23
+ elem = eval( "document.all." + id );
24
+ else
25
+ return false;
26
+
27
+ elemStyle = elem.style;
28
+
29
+ if ( elemStyle.display != "block" ) {
30
+ elemStyle.display = "block"
31
+ } else {
32
+ elemStyle.display = "none"
33
+ }
34
+
35
+ return true;
36
+ }
37
+
38
+ // Make codeblocks hidden by default
39
+ document.writeln( "<style type=\"text/css\">div.method-source-code { display: none }</style>" )
40
+
41
+ // ]]>
42
+ </script>
43
+
44
+ </head>
45
+ <body>
46
+
47
+
48
+
49
+ <div id="fileHeader">
50
+ <h1>robot_rules.rb</h1>
51
+ <table class="header-table">
52
+ <tr class="top-aligned-row">
53
+ <td><strong>Path:</strong></td>
54
+ <td>lib/spider/robot_rules.rb
55
+ </td>
56
+ </tr>
57
+ <tr class="top-aligned-row">
58
+ <td><strong>Last Update:</strong></td>
59
+ <td>Thu May 21 13:19:06 +0000 2009</td>
60
+ </tr>
61
+ </table>
62
+ </div>
63
+ <!-- banner header -->
64
+
65
+ <div id="bodyContent">
66
+
67
+
68
+
69
+ <div id="contextContent">
70
+
71
+ <div id="description">
72
+ <p>
73
+ Understand robots.txt.
74
+ </p>
75
+
76
+ </div>
77
+
78
+ <div id="requires-list">
79
+ <h3 class="section-bar">Required files</h3>
80
+
81
+ <div class="name-list">
82
+ uri&nbsp;&nbsp;
83
+ </div>
84
+ </div>
85
+
86
+ </div>
87
+
88
+
89
+ </div>
90
+
91
+
92
+ <!-- if includes -->
93
+
94
+ <div id="section">
95
+
96
+
97
+
98
+
99
+
100
+
101
+
102
+
103
+ <!-- if method_list -->
104
+
105
+
106
+ </div>
107
+
108
+
109
+ <div id="validator-badges">
110
+ <p><small><a href="http://validator.w3.org/check/referer">[Validate]</a></small></p>
111
+ </div>
112
+
113
+ </body>
114
+ </html>
data/doc/files/lib/spider/spider_instance_rb.html CHANGED
@@ -56,7 +56,7 @@
56
56
  </tr>
57
57
  <tr class="top-aligned-row">
58
58
  <td><strong>Last Update:</strong></td>
59
- <td>Sat Nov 10 00:25:04 -0500 2007</td>
59
+ <td>Thu May 21 15:38:44 +0000 2009</td>
60
60
  </tr>
61
61
  </table>
62
62
  </div>
@@ -79,7 +79,6 @@ Specialized spidering rules.
79
79
  <h3 class="section-bar">Required files</h3>
80
80
 
81
81
  <div class="name-list">
82
- robot_rules&nbsp;&nbsp;
83
82
  open-uri&nbsp;&nbsp;
84
83
  uri&nbsp;&nbsp;
85
84
  net/http&nbsp;&nbsp;
data/doc/files/lib/spider_rb.html CHANGED
@@ -56,7 +56,7 @@
56
56
  </tr>
57
57
  <tr class="top-aligned-row">
58
58
  <td><strong>Last Update:</strong></td>
59
- <td>Thu Nov 08 17:29:01 -0500 2007</td>
59
+ <td>Thu May 21 13:19:06 +0000 2009</td>
60
60
  </tr>
61
61
  </table>
62
62
  </div>
@@ -70,9 +70,12 @@
70
70
 
71
71
  <div id="description">
72
72
  <p>
73
- Copyright 2007 Mike Burns <a href="../../classes/Spider.html">Spider</a>, a
74
- Web spidering library for Ruby. It handles the robots.txt, scraping,
75
- collecting, and looping so that you can just handle the data.
73
+ Copyright 2007-2008 Mike Burns &amp; John Nagro
74
+ </p>
75
+ <p>
76
+ <a href="../../classes/Spider.html">Spider</a>, a Web spidering library for
77
+ Ruby. It handles the robots.txt, scraping, collecting, and looping so that
78
+ you can just handle the data.
76
79
  </p>
77
80
  <h2>Examples</h2>
78
81
  <h3>Crawl the Web, loading each page in turn, until you run out of memory</h3>
@@ -133,13 +136,12 @@ collecting, and looping so that you can just handle the data.
133
136
  <h3>Track cycles with a custom object</h3>
134
137
  <pre>
135
138
  require 'spider'
136
-
137
139
  class ExpireLinks &lt; Hash
138
140
  def &lt;&lt;(v)
139
- [v] = Time.now
141
+ self[v] = Time.now
140
142
  end
141
143
  def include?(v)
142
- [v] &amp;&amp; (Time.now + 86400) &lt;= [v]
144
+ self[v].kind_of?(Time) &amp;&amp; (self[v] + 86400) &gt;= Time.now
143
145
  end
144
146
  end
145
147
 
@@ -147,6 +149,31 @@ collecting, and looping so that you can just handle the data.
147
149
  s.check_already_seen_with ExpireLinks.new
148
150
  end
149
151
  </pre>
152
+ <h3>Store nodes to visit with Amazon SQS</h3>
153
+ <pre>
154
+ require 'spider'
155
+ require 'spider/next_urls_in_sqs'
156
+ Spider.start_at('http://mike-burns.com') do |s|
157
+ s.store_next_urls_with NextUrlsInSQS.new(AWS_ACCESS_KEY, AWS_SECRET_ACCESS_KEY)
158
+ end
159
+ </pre>
160
+ <h4>Store nodes to visit with a custom object</h4>
161
+ <pre>
162
+ require 'spider'
163
+ class MyArray &lt; Array
164
+ def pop
165
+ super
166
+ end
167
+
168
+ def push(a_msg)
169
+ super(a_msg)
170
+ end
171
+ end
172
+
173
+ Spider.start_at('http://mike-burns.com') do |s|
174
+ s.store_next_urls_with MyArray.new
175
+ end
176
+ </pre>
150
177
  <h3>Create a URL graph</h3>
151
178
  <pre>
152
179
  require 'spider'
@@ -178,11 +205,15 @@ collecting, and looping so that you can just handle the data.
178
205
  </pre>
179
206
  <h2>Author</h2>
180
207
  <p>
208
+ John Nagro john.nagro@gmail.com
209
+ </p>
210
+ <p>
181
211
  Mike Burns <a href="http://mike-burns.com">mike-burns.com</a>
182
- mike@mike-burns.com
212
+ mike@mike-burns.com (original author)
183
213
  </p>
184
214
  <p>
185
- Help from Matt Horan, John Nagro, and Henri Cook.
215
+ Many thanks to: Matt Horan Henri Cook Sander van der Vliet John Buckley
216
+ Brian Campbell
186
217
  </p>
187
218
  <p>
188
219
  With `robot_rules&#8217; from James Edward Gray II via <a