spider 0.4.4 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. checksums.yaml +7 -0
  2. data/AUTHORS +12 -0
  3. data/CHANGES +6 -0
  4. data/LICENSE +21 -0
  5. data/{README → README.md} +50 -43
  6. data/lib/spider.rb +12 -29
  7. data/lib/spider/included_in_memcached.rb +1 -24
  8. data/lib/spider/next_urls_in_sqs.rb +6 -29
  9. data/lib/spider/robot_rules.rb +61 -57
  10. data/lib/spider/spider_instance.rb +8 -31
  11. data/spider.gemspec +4 -2
  12. metadata +33 -124
  13. data/doc/classes/BeStaticServerPages.html +0 -197
  14. data/doc/classes/BeStaticServerPages.src/M000030.html +0 -19
  15. data/doc/classes/BeStaticServerPages.src/M000031.html +0 -19
  16. data/doc/classes/BeStaticServerPages.src/M000032.html +0 -18
  17. data/doc/classes/BeStaticServerPages.src/M000033.html +0 -18
  18. data/doc/classes/IncludedInMemcached.html +0 -199
  19. data/doc/classes/IncludedInMemcached.src/M000015.html +0 -18
  20. data/doc/classes/IncludedInMemcached.src/M000016.html +0 -18
  21. data/doc/classes/IncludedInMemcached.src/M000017.html +0 -18
  22. data/doc/classes/LoopingServlet.html +0 -137
  23. data/doc/classes/LoopingServlet.src/M000037.html +0 -23
  24. data/doc/classes/NextUrlsInSQS.html +0 -204
  25. data/doc/classes/NextUrlsInSQS.src/M000018.html +0 -19
  26. data/doc/classes/NextUrlsInSQS.src/M000019.html +0 -22
  27. data/doc/classes/NextUrlsInSQS.src/M000020.html +0 -19
  28. data/doc/classes/QueryServlet.html +0 -137
  29. data/doc/classes/QueryServlet.src/M000038.html +0 -19
  30. data/doc/classes/RobotRules.html +0 -175
  31. data/doc/classes/RobotRules.src/M000034.html +0 -19
  32. data/doc/classes/RobotRules.src/M000035.html +0 -67
  33. data/doc/classes/RobotRules.src/M000036.html +0 -24
  34. data/doc/classes/Spider.html +0 -170
  35. data/doc/classes/Spider.src/M000029.html +0 -21
  36. data/doc/classes/SpiderInstance.html +0 -345
  37. data/doc/classes/SpiderInstance.src/M000021.html +0 -18
  38. data/doc/classes/SpiderInstance.src/M000022.html +0 -22
  39. data/doc/classes/SpiderInstance.src/M000023.html +0 -22
  40. data/doc/classes/SpiderInstance.src/M000024.html +0 -24
  41. data/doc/classes/SpiderInstance.src/M000025.html +0 -18
  42. data/doc/classes/SpiderInstance.src/M000026.html +0 -18
  43. data/doc/classes/SpiderInstance.src/M000027.html +0 -18
  44. data/doc/classes/SpiderInstance.src/M000028.html +0 -18
  45. data/doc/created.rid +0 -1
  46. data/doc/files/README.html +0 -223
  47. data/doc/files/lib/spider/included_in_memcached_rb.html +0 -142
  48. data/doc/files/lib/spider/next_urls_in_sqs_rb.html +0 -144
  49. data/doc/files/lib/spider/robot_rules_rb.html +0 -114
  50. data/doc/files/lib/spider/spider_instance_rb.html +0 -117
  51. data/doc/files/lib/spider_rb.html +0 -254
  52. data/doc/files/spec/spec_helper_rb.html +0 -196
  53. data/doc/files/spec/spec_helper_rb.src/M000001.html +0 -20
  54. data/doc/files/spec/spec_helper_rb.src/M000002.html +0 -26
  55. data/doc/files/spec/spec_helper_rb.src/M000003.html +0 -24
  56. data/doc/files/spec/spec_helper_rb.src/M000004.html +0 -18
  57. data/doc/files/spec/spec_helper_rb.src/M000005.html +0 -23
  58. data/doc/files/spec/spider/included_in_memcached_spec_rb.html +0 -142
  59. data/doc/files/spec/spider/included_in_memcached_spec_rb.src/M000006.html +0 -19
  60. data/doc/files/spec/spider/included_in_memcached_spec_rb.src/M000007.html +0 -18
  61. data/doc/files/spec/spider/spider_instance_spec_rb.html +0 -210
  62. data/doc/files/spec/spider/spider_instance_spec_rb.src/M000008.html +0 -21
  63. data/doc/files/spec/spider/spider_instance_spec_rb.src/M000009.html +0 -19
  64. data/doc/files/spec/spider/spider_instance_spec_rb.src/M000010.html +0 -19
  65. data/doc/files/spec/spider/spider_instance_spec_rb.src/M000011.html +0 -27
  66. data/doc/files/spec/spider/spider_instance_spec_rb.src/M000012.html +0 -26
  67. data/doc/files/spec/spider/spider_instance_spec_rb.src/M000013.html +0 -27
  68. data/doc/files/spec/spider_spec_rb.html +0 -127
  69. data/doc/files/spec/spider_spec_rb.src/M000014.html +0 -23
  70. data/doc/fr_class_index.html +0 -34
  71. data/doc/fr_file_index.html +0 -35
  72. data/doc/fr_method_index.html +0 -64
  73. data/doc/index.html +0 -24
  74. data/doc/rdoc-style.css +0 -208
data/doc/classes/SpiderInstance.src/M000025.html DELETED
@@ -1,18 +0,0 @@
1
- <?xml version="1.0" encoding="iso-8859-1"?>
2
- <!DOCTYPE html
3
- PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
4
- "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
5
-
6
- <html>
7
- <head>
8
- <title>setup (SpiderInstance)</title>
9
- <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
10
- <link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
11
- </head>
12
- <body class="standalone-code">
13
- <pre><span class="ruby-comment cmt"># File lib/spider/spider_instance.rb, line 159</span>
14
- <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">setup</span>(<span class="ruby-identifier">p</span> = <span class="ruby-keyword kw">nil</span>, <span class="ruby-operator">&amp;</span><span class="ruby-identifier">block</span>)
15
- <span class="ruby-ivar">@setup</span> = <span class="ruby-identifier">p</span> <span class="ruby-value">? </span><span class="ruby-identifier">p</span> <span class="ruby-operator">:</span> <span class="ruby-identifier">block</span>
16
- <span class="ruby-keyword kw">end</span></pre>
17
- </body>
18
- </html>
data/doc/classes/SpiderInstance.src/M000026.html DELETED
@@ -1,18 +0,0 @@
1
- <?xml version="1.0" encoding="iso-8859-1"?>
2
- <!DOCTYPE html
3
- PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
4
- "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
5
-
6
- <html>
7
- <head>
8
- <title>teardown (SpiderInstance)</title>
9
- <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
10
- <link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
11
- </head>
12
- <body class="standalone-code">
13
- <pre><span class="ruby-comment cmt"># File lib/spider/spider_instance.rb, line 164</span>
14
- <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">teardown</span>(<span class="ruby-identifier">p</span> = <span class="ruby-keyword kw">nil</span>, <span class="ruby-operator">&amp;</span><span class="ruby-identifier">block</span>)
15
- <span class="ruby-ivar">@teardown</span> = <span class="ruby-identifier">p</span> <span class="ruby-value">? </span><span class="ruby-identifier">p</span> <span class="ruby-operator">:</span> <span class="ruby-identifier">block</span>
16
- <span class="ruby-keyword kw">end</span></pre>
17
- </body>
18
- </html>
data/doc/classes/SpiderInstance.src/M000027.html DELETED
@@ -1,18 +0,0 @@
1
- <?xml version="1.0" encoding="iso-8859-1"?>
2
- <!DOCTYPE html
3
- PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
4
- "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
5
-
6
- <html>
7
- <head>
8
- <title>headers (SpiderInstance)</title>
9
- <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
10
- <link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
11
- </head>
12
- <body class="standalone-code">
13
- <pre><span class="ruby-comment cmt"># File lib/spider/spider_instance.rb, line 170</span>
14
- <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">headers</span>
15
- <span class="ruby-constant">HeaderSetter</span>.<span class="ruby-identifier">new</span>(<span class="ruby-keyword kw">self</span>)
16
- <span class="ruby-keyword kw">end</span></pre>
17
- </body>
18
- </html>
data/doc/classes/SpiderInstance.src/M000028.html DELETED
@@ -1,18 +0,0 @@
1
- <?xml version="1.0" encoding="iso-8859-1"?>
2
- <!DOCTYPE html
3
- PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
4
- "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
5
-
6
- <html>
7
- <head>
8
- <title>clear_headers (SpiderInstance)</title>
9
- <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
10
- <link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
11
- </head>
12
- <body class="standalone-code">
13
- <pre><span class="ruby-comment cmt"># File lib/spider/spider_instance.rb, line 182</span>
14
- <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">clear_headers</span>
15
- <span class="ruby-ivar">@headers</span> = {}
16
- <span class="ruby-keyword kw">end</span></pre>
17
- </body>
18
- </html>
data/doc/created.rid DELETED
@@ -1 +0,0 @@
1
- Thu, 21 May 2009 15:42:01 +0000
data/doc/files/README.html DELETED
@@ -1,223 +0,0 @@
1
- <?xml version="1.0" encoding="iso-8859-1"?>
2
- <!DOCTYPE html
3
- PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
4
- "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
5
-
6
- <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
7
- <head>
8
- <title>File: README</title>
9
- <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
10
- <meta http-equiv="Content-Script-Type" content="text/javascript" />
11
- <link rel="stylesheet" href=".././rdoc-style.css" type="text/css" media="screen" />
12
- <script type="text/javascript">
13
- // <![CDATA[
14
-
15
- function popupCode( url ) {
16
- window.open(url, "Code", "resizable=yes,scrollbars=yes,toolbar=no,status=no,height=150,width=400")
17
- }
18
-
19
- function toggleCode( id ) {
20
- if ( document.getElementById )
21
- elem = document.getElementById( id );
22
- else if ( document.all )
23
- elem = eval( "document.all." + id );
24
- else
25
- return false;
26
-
27
- elemStyle = elem.style;
28
-
29
- if ( elemStyle.display != "block" ) {
30
- elemStyle.display = "block"
31
- } else {
32
- elemStyle.display = "none"
33
- }
34
-
35
- return true;
36
- }
37
-
38
- // Make codeblocks hidden by default
39
- document.writeln( "<style type=\"text/css\">div.method-source-code { display: none }</style>" )
40
-
41
- // ]]>
42
- </script>
43
-
44
- </head>
45
- <body>
46
-
47
-
48
-
49
- <div id="fileHeader">
50
- <h1>README</h1>
51
- <table class="header-table">
52
- <tr class="top-aligned-row">
53
- <td><strong>Path:</strong></td>
54
- <td>README
55
- </td>
56
- </tr>
57
- <tr class="top-aligned-row">
58
- <td><strong>Last Update:</strong></td>
59
- <td>Thu Nov 08 17:51:17 -0500 2007</td>
60
- </tr>
61
- </table>
62
- </div>
63
- <!-- banner header -->
64
-
65
- <div id="bodyContent">
66
-
67
-
68
-
69
- <div id="contextContent">
70
-
71
- <div id="description">
72
- <p>
73
- <a href="../classes/Spider.html">Spider</a>, a Web spidering library for
74
- Ruby. It handles the robots.txt, scraping, collecting, and looping so that
75
- you can just handle the data.
76
- </p>
77
- <h2>Examples</h2>
78
- <h3>Crawl the Web, loading each page in turn, until you run out of memory</h3>
79
- <pre>
80
- require 'spider'
81
- Spider.start_at('http://mike-burns.com/') {}
82
- </pre>
83
- <h3>To handle erroneous responses</h3>
84
- <pre>
85
- require 'spider'
86
- Spider.start_at('http://mike-burns.com/') do |s|
87
- s.on :failure do |a_url, resp, prior_url|
88
- puts &quot;URL failed: #{a_url}&quot;
89
- puts &quot; linked from #{prior_url}&quot;
90
- end
91
- end
92
- </pre>
93
- <h3>Or handle successful responses</h3>
94
- <pre>
95
- require 'spider'
96
- Spider.start_at('http://mike-burns.com/') do |s|
97
- s.on :success do |a_url, resp, prior_url|
98
- puts &quot;#{a_url}: #{resp.code}&quot;
99
- puts resp.body
100
- puts
101
- end
102
- end
103
- </pre>
104
- <h3>Limit to just one domain</h3>
105
- <pre>
106
- require 'spider'
107
- Spider.start_at('http://mike-burns.com/') do |s|
108
- s.add_url_check do |a_url|
109
- a_url =~ %r{^http://mike-burns.com.*}
110
- end
111
- end
112
- </pre>
113
- <h3>Pass headers to some requests</h3>
114
- <pre>
115
- require 'spider'
116
- Spider.start_at('http://mike-burns.com/') do |s|
117
- s.setup do |a_url|
118
- if a_url =~ %r{^http://.*wikipedia.*}
119
- headers['User-Agent'] = &quot;Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)&quot;
120
- end
121
- end
122
- end
123
- </pre>
124
- <h3>Use memcached to track cycles</h3>
125
- <pre>
126
- require 'spider'
127
- require 'spider/included_in_memcached'
128
- SERVERS = ['10.0.10.2:11211','10.0.10.3:11211','10.0.10.4:11211']
129
- Spider.start_at('http://mike-burns.com/') do |s|
130
- s.check_already_seen_with IncludedInMemcached.new(SERVERS)
131
- end
132
- </pre>
133
- <h3>Track cycles with a custom object</h3>
134
- <pre>
135
- require 'spider'
136
-
137
- class ExpireLinks &lt; Hash
138
- def &lt;&lt;(v)
139
- [v] = Time.now
140
- end
141
- def include?(v)
142
- [v] &amp;&amp; (Time.now + 86400) &lt;= [v]
143
- end
144
- end
145
-
146
- Spider.start_at('http://mike-burns.com/') do |s|
147
- s.check_already_seen_with ExpireLinks.new
148
- end
149
- </pre>
150
- <h3>Create a URL graph</h3>
151
- <pre>
152
- require 'spider'
153
- nodes = {}
154
- Spider.start_at('http://mike-burns.com/') do |s|
155
- s.add_url_check {|a_url| a_url =~ %r{^http://mike-burns.com.*} }
156
-
157
- s.on(:every) do |a_url, resp, prior_url|
158
- nodes[prior_url] ||= []
159
- nodes[prior_url] &lt;&lt; a_url
160
- end
161
- end
162
- </pre>
163
- <h3>Use a proxy</h3>
164
- <pre>
165
- require 'net/http_configuration'
166
- require 'spider'
167
- http_conf = Net::HTTP::Configuration.new(:proxy_host =&gt; '7proxies.org',
168
- :proxy_port =&gt; 8881)
169
- http_conf.apply do
170
- Spider.start_at('http://img.4chan.org/b/') do |s|
171
- s.on(:success) do |a_url, resp, prior_url|
172
- File.open(a_url.gsub('/',':'),'w') do |f|
173
- f.write(resp.body)
174
- end
175
- end
176
- end
177
- end
178
- </pre>
179
- <h2>Author</h2>
180
- <p>
181
- Mike Burns <a href="http://mike-burns.com">mike-burns.com</a>
182
- mike@mike-burns.com
183
- </p>
184
- <p>
185
- Help from Matt Horan, John Nagro, and Henri Cook.
186
- </p>
187
- <p>
188
- With `robot_rules&#8217; from James Edward Gray II via <a
189
- href="http://blade.nagaokaut.ac.jp/cgi-bin/scat.rb/ruby/ruby-talk/177589">blade.nagaokaut.ac.jp/cgi-bin/scat.rb/ruby/ruby-talk/177589</a>
190
- </p>
191
-
192
- </div>
193
-
194
-
195
- </div>
196
-
197
-
198
- </div>
199
-
200
-
201
- <!-- if includes -->
202
-
203
- <div id="section">
204
-
205
-
206
-
207
-
208
-
209
-
210
-
211
-
212
- <!-- if method_list -->
213
-
214
-
215
- </div>
216
-
217
-
218
- <div id="validator-badges">
219
- <p><small><a href="http://validator.w3.org/check/referer">[Validate]</a></small></p>
220
- </div>
221
-
222
- </body>
223
- </html>
data/doc/files/lib/spider/included_in_memcached_rb.html DELETED
@@ -1,142 +0,0 @@
1
- <?xml version="1.0" encoding="iso-8859-1"?>
2
- <!DOCTYPE html
3
- PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
4
- "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
5
-
6
- <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
7
- <head>
8
- <title>File: included_in_memcached.rb</title>
9
- <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
10
- <meta http-equiv="Content-Script-Type" content="text/javascript" />
11
- <link rel="stylesheet" href="../../.././rdoc-style.css" type="text/css" media="screen" />
12
- <script type="text/javascript">
13
- // <![CDATA[
14
-
15
- function popupCode( url ) {
16
- window.open(url, "Code", "resizable=yes,scrollbars=yes,toolbar=no,status=no,height=150,width=400")
17
- }
18
-
19
- function toggleCode( id ) {
20
- if ( document.getElementById )
21
- elem = document.getElementById( id );
22
- else if ( document.all )
23
- elem = eval( "document.all." + id );
24
- else
25
- return false;
26
-
27
- elemStyle = elem.style;
28
-
29
- if ( elemStyle.display != "block" ) {
30
- elemStyle.display = "block"
31
- } else {
32
- elemStyle.display = "none"
33
- }
34
-
35
- return true;
36
- }
37
-
38
- // Make codeblocks hidden by default
39
- document.writeln( "<style type=\"text/css\">div.method-source-code { display: none }</style>" )
40
-
41
- // ]]>
42
- </script>
43
-
44
- </head>
45
- <body>
46
-
47
-
48
-
49
- <div id="fileHeader">
50
- <h1>included_in_memcached.rb</h1>
51
- <table class="header-table">
52
- <tr class="top-aligned-row">
53
- <td><strong>Path:</strong></td>
54
- <td>lib/spider/included_in_memcached.rb
55
- </td>
56
- </tr>
57
- <tr class="top-aligned-row">
58
- <td><strong>Last Update:</strong></td>
59
- <td>Thu May 21 13:19:06 +0000 2009</td>
60
- </tr>
61
- </table>
62
- </div>
63
- <!-- banner header -->
64
-
65
- <div id="bodyContent">
66
-
67
-
68
-
69
- <div id="contextContent">
70
-
71
- <div id="description">
72
- <p>
73
- Use memcached to track cycles.
74
- </p>
75
- <p>
76
- Copyright 2007 Mike Burns Redistribution and use in source and binary
77
- forms, with or without modification, are permitted provided that the
78
- following conditions are met:
79
- </p>
80
- <pre>
81
- * Redistributions of source code must retain the above copyright
82
- notice, this list of conditions and the following disclaimer.
83
- * Redistributions in binary form must reproduce the above copyright
84
- notice, this list of conditions and the following disclaimer in the
85
- documentation and/or other materials provided with the distribution.
86
- * Neither the name Mike Burns nor the
87
- names of his contributors may be used to endorse or promote products
88
- derived from this software without specific prior written permission.
89
- </pre>
90
- <p>
91
- THIS SOFTWARE IS PROVIDED BY Mike Burns ``AS IS&#8217;&#8217; AND ANY
92
- EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
93
- WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
94
- DISCLAIMED. IN NO EVENT SHALL Mike Burns BE LIABLE FOR ANY DIRECT,
95
- INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
96
- (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
97
- SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
98
- CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
99
- LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
100
- OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
101
- DAMAGE.
102
- </p>
103
-
104
- </div>
105
-
106
- <div id="requires-list">
107
- <h3 class="section-bar">Required files</h3>
108
-
109
- <div class="name-list">
110
- memcache&nbsp;&nbsp;
111
- </div>
112
- </div>
113
-
114
- </div>
115
-
116
-
117
- </div>
118
-
119
-
120
- <!-- if includes -->
121
-
122
- <div id="section">
123
-
124
-
125
-
126
-
127
-
128
-
129
-
130
-
131
- <!-- if method_list -->
132
-
133
-
134
- </div>
135
-
136
-
137
- <div id="validator-badges">
138
- <p><small><a href="http://validator.w3.org/check/referer">[Validate]</a></small></p>
139
- </div>
140
-
141
- </body>
142
- </html>
data/doc/files/lib/spider/next_urls_in_sqs_rb.html DELETED
@@ -1,144 +0,0 @@
1
- <?xml version="1.0" encoding="iso-8859-1"?>
2
- <!DOCTYPE html
3
- PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
4
- "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
5
-
6
- <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
7
- <head>
8
- <title>File: next_urls_in_sqs.rb</title>
9
- <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
10
- <meta http-equiv="Content-Script-Type" content="text/javascript" />
11
- <link rel="stylesheet" href="../../.././rdoc-style.css" type="text/css" media="screen" />
12
- <script type="text/javascript">
13
- // <![CDATA[
14
-
15
- function popupCode( url ) {
16
- window.open(url, "Code", "resizable=yes,scrollbars=yes,toolbar=no,status=no,height=150,width=400")
17
- }
18
-
19
- function toggleCode( id ) {
20
- if ( document.getElementById )
21
- elem = document.getElementById( id );
22
- else if ( document.all )
23
- elem = eval( "document.all." + id );
24
- else
25
- return false;
26
-
27
- elemStyle = elem.style;
28
-
29
- if ( elemStyle.display != "block" ) {
30
- elemStyle.display = "block"
31
- } else {
32
- elemStyle.display = "none"
33
- }
34
-
35
- return true;
36
- }
37
-
38
- // Make codeblocks hidden by default
39
- document.writeln( "<style type=\"text/css\">div.method-source-code { display: none }</style>" )
40
-
41
- // ]]>
42
- </script>
43
-
44
- </head>
45
- <body>
46
-
47
-
48
-
49
- <div id="fileHeader">
50
- <h1>next_urls_in_sqs.rb</h1>
51
- <table class="header-table">
52
- <tr class="top-aligned-row">
53
- <td><strong>Path:</strong></td>
54
- <td>lib/spider/next_urls_in_sqs.rb
55
- </td>
56
- </tr>
57
- <tr class="top-aligned-row">
58
- <td><strong>Last Update:</strong></td>
59
- <td>Thu May 21 13:19:06 +0000 2009</td>
60
- </tr>
61
- </table>
62
- </div>
63
- <!-- banner header -->
64
-
65
- <div id="bodyContent">
66
-
67
-
68
-
69
- <div id="contextContent">
70
-
71
- <div id="description">
72
- <p>
73
- Use AmazonSQS to track nodes to visit.
74
- </p>
75
- <p>
76
- Copyright 2008 John Nagro Redistribution and use in source and binary
77
- forms, with or without modification, are permitted provided that the
78
- following conditions are met:
79
- </p>
80
- <pre>
81
- * Redistributions of source code must retain the above copyright
82
- notice, this list of conditions and the following disclaimer.
83
- * Redistributions in binary form must reproduce the above copyright
84
- notice, this list of conditions and the following disclaimer in the
85
- documentation and/or other materials provided with the distribution.
86
- * Neither the name Mike Burns nor the
87
- names of his contributors may be used to endorse or promote products
88
- derived from this software without specific prior written permission.
89
- </pre>
90
- <p>
91
- THIS SOFTWARE IS PROVIDED BY Mike Burns ``AS IS&#8217;&#8217; AND ANY
92
- EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
93
- WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
94
- DISCLAIMED. IN NO EVENT SHALL Mike Burns BE LIABLE FOR ANY DIRECT,
95
- INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
96
- (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
97
- SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
98
- CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
99
- LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
100
- OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH
101
- DAMAGE.
102
- </p>
103
-
104
- </div>
105
-
106
- <div id="requires-list">
107
- <h3 class="section-bar">Required files</h3>
108
-
109
- <div class="name-list">
110
- rubygems&nbsp;&nbsp;
111
- right_aws&nbsp;&nbsp;
112
- yaml&nbsp;&nbsp;
113
- </div>
114
- </div>
115
-
116
- </div>
117
-
118
-
119
- </div>
120
-
121
-
122
- <!-- if includes -->
123
-
124
- <div id="section">
125
-
126
-
127
-
128
-
129
-
130
-
131
-
132
-
133
- <!-- if method_list -->
134
-
135
-
136
- </div>
137
-
138
-
139
- <div id="validator-badges">
140
- <p><small><a href="http://validator.w3.org/check/referer">[Validate]</a></small></p>
141
- </div>
142
-
143
- </body>
144
- </html>