spider 0.4.4 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (74) hide show
  1. checksums.yaml +7 -0
  2. data/AUTHORS +12 -0
  3. data/CHANGES +6 -0
  4. data/LICENSE +21 -0
  5. data/{README → README.md} +50 -43
  6. data/lib/spider.rb +12 -29
  7. data/lib/spider/included_in_memcached.rb +1 -24
  8. data/lib/spider/next_urls_in_sqs.rb +6 -29
  9. data/lib/spider/robot_rules.rb +61 -57
  10. data/lib/spider/spider_instance.rb +8 -31
  11. data/spider.gemspec +4 -2
  12. metadata +33 -124
  13. data/doc/classes/BeStaticServerPages.html +0 -197
  14. data/doc/classes/BeStaticServerPages.src/M000030.html +0 -19
  15. data/doc/classes/BeStaticServerPages.src/M000031.html +0 -19
  16. data/doc/classes/BeStaticServerPages.src/M000032.html +0 -18
  17. data/doc/classes/BeStaticServerPages.src/M000033.html +0 -18
  18. data/doc/classes/IncludedInMemcached.html +0 -199
  19. data/doc/classes/IncludedInMemcached.src/M000015.html +0 -18
  20. data/doc/classes/IncludedInMemcached.src/M000016.html +0 -18
  21. data/doc/classes/IncludedInMemcached.src/M000017.html +0 -18
  22. data/doc/classes/LoopingServlet.html +0 -137
  23. data/doc/classes/LoopingServlet.src/M000037.html +0 -23
  24. data/doc/classes/NextUrlsInSQS.html +0 -204
  25. data/doc/classes/NextUrlsInSQS.src/M000018.html +0 -19
  26. data/doc/classes/NextUrlsInSQS.src/M000019.html +0 -22
  27. data/doc/classes/NextUrlsInSQS.src/M000020.html +0 -19
  28. data/doc/classes/QueryServlet.html +0 -137
  29. data/doc/classes/QueryServlet.src/M000038.html +0 -19
  30. data/doc/classes/RobotRules.html +0 -175
  31. data/doc/classes/RobotRules.src/M000034.html +0 -19
  32. data/doc/classes/RobotRules.src/M000035.html +0 -67
  33. data/doc/classes/RobotRules.src/M000036.html +0 -24
  34. data/doc/classes/Spider.html +0 -170
  35. data/doc/classes/Spider.src/M000029.html +0 -21
  36. data/doc/classes/SpiderInstance.html +0 -345
  37. data/doc/classes/SpiderInstance.src/M000021.html +0 -18
  38. data/doc/classes/SpiderInstance.src/M000022.html +0 -22
  39. data/doc/classes/SpiderInstance.src/M000023.html +0 -22
  40. data/doc/classes/SpiderInstance.src/M000024.html +0 -24
  41. data/doc/classes/SpiderInstance.src/M000025.html +0 -18
  42. data/doc/classes/SpiderInstance.src/M000026.html +0 -18
  43. data/doc/classes/SpiderInstance.src/M000027.html +0 -18
  44. data/doc/classes/SpiderInstance.src/M000028.html +0 -18
  45. data/doc/created.rid +0 -1
  46. data/doc/files/README.html +0 -223
  47. data/doc/files/lib/spider/included_in_memcached_rb.html +0 -142
  48. data/doc/files/lib/spider/next_urls_in_sqs_rb.html +0 -144
  49. data/doc/files/lib/spider/robot_rules_rb.html +0 -114
  50. data/doc/files/lib/spider/spider_instance_rb.html +0 -117
  51. data/doc/files/lib/spider_rb.html +0 -254
  52. data/doc/files/spec/spec_helper_rb.html +0 -196
  53. data/doc/files/spec/spec_helper_rb.src/M000001.html +0 -20
  54. data/doc/files/spec/spec_helper_rb.src/M000002.html +0 -26
  55. data/doc/files/spec/spec_helper_rb.src/M000003.html +0 -24
  56. data/doc/files/spec/spec_helper_rb.src/M000004.html +0 -18
  57. data/doc/files/spec/spec_helper_rb.src/M000005.html +0 -23
  58. data/doc/files/spec/spider/included_in_memcached_spec_rb.html +0 -142
  59. data/doc/files/spec/spider/included_in_memcached_spec_rb.src/M000006.html +0 -19
  60. data/doc/files/spec/spider/included_in_memcached_spec_rb.src/M000007.html +0 -18
  61. data/doc/files/spec/spider/spider_instance_spec_rb.html +0 -210
  62. data/doc/files/spec/spider/spider_instance_spec_rb.src/M000008.html +0 -21
  63. data/doc/files/spec/spider/spider_instance_spec_rb.src/M000009.html +0 -19
  64. data/doc/files/spec/spider/spider_instance_spec_rb.src/M000010.html +0 -19
  65. data/doc/files/spec/spider/spider_instance_spec_rb.src/M000011.html +0 -27
  66. data/doc/files/spec/spider/spider_instance_spec_rb.src/M000012.html +0 -26
  67. data/doc/files/spec/spider/spider_instance_spec_rb.src/M000013.html +0 -27
  68. data/doc/files/spec/spider_spec_rb.html +0 -127
  69. data/doc/files/spec/spider_spec_rb.src/M000014.html +0 -23
  70. data/doc/fr_class_index.html +0 -34
  71. data/doc/fr_file_index.html +0 -35
  72. data/doc/fr_method_index.html +0 -64
  73. data/doc/index.html +0 -24
  74. data/doc/rdoc-style.css +0 -208
data/doc/classes/RobotRules.html DELETED
@@ -1,175 +0,0 @@
1
- <?xml version="1.0" encoding="iso-8859-1"?>
2
- <!DOCTYPE html
3
- PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
4
- "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
5
-
6
- <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
7
- <head>
8
- <title>Class: RobotRules</title>
9
- <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
10
- <meta http-equiv="Content-Script-Type" content="text/javascript" />
11
- <link rel="stylesheet" href=".././rdoc-style.css" type="text/css" media="screen" />
12
- <script type="text/javascript">
13
- // <![CDATA[
14
-
15
- function popupCode( url ) {
16
- window.open(url, "Code", "resizable=yes,scrollbars=yes,toolbar=no,status=no,height=150,width=400")
17
- }
18
-
19
- function toggleCode( id ) {
20
- if ( document.getElementById )
21
- elem = document.getElementById( id );
22
- else if ( document.all )
23
- elem = eval( "document.all." + id );
24
- else
25
- return false;
26
-
27
- elemStyle = elem.style;
28
-
29
- if ( elemStyle.display != "block" ) {
30
- elemStyle.display = "block"
31
- } else {
32
- elemStyle.display = "none"
33
- }
34
-
35
- return true;
36
- }
37
-
38
- // Make codeblocks hidden by default
39
- document.writeln( "<style type=\"text/css\">div.method-source-code { display: none }</style>" )
40
-
41
- // ]]>
42
- </script>
43
-
44
- </head>
45
- <body>
46
-
47
-
48
-
49
- <div id="classHeader">
50
- <table class="header-table">
51
- <tr class="top-aligned-row">
52
- <td><strong>Class</strong></td>
53
- <td class="class-name-in-header">RobotRules</td>
54
- </tr>
55
- <tr class="top-aligned-row">
56
- <td><strong>In:</strong></td>
57
- <td>
58
- <a href="../files/lib/spider/robot_rules_rb.html">
59
- lib/spider/robot_rules.rb
60
- </a>
61
- <br />
62
- </td>
63
- </tr>
64
-
65
- <tr class="top-aligned-row">
66
- <td><strong>Parent:</strong></td>
67
- <td>
68
- Object
69
- </td>
70
- </tr>
71
- </table>
72
- </div>
73
- <!-- banner header -->
74
-
75
- <div id="bodyContent">
76
-
77
-
78
-
79
- <div id="contextContent">
80
-
81
- <div id="description">
82
- <p>
83
- Based on Perl&#8216;s WWW::RobotRules module, by Gisle Aas.
84
- </p>
85
-
86
- </div>
87
-
88
-
89
- </div>
90
-
91
- <div id="method-list">
92
- <h3 class="section-bar">Methods</h3>
93
-
94
- <div class="name-list">
95
- <a href="#M000036">allowed?</a>&nbsp;&nbsp;
96
- <a href="#M000034">new</a>&nbsp;&nbsp;
97
- <a href="#M000035">parse</a>&nbsp;&nbsp;
98
- </div>
99
- </div>
100
-
101
- </div>
102
-
103
-
104
- <!-- if includes -->
105
-
106
- <div id="section">
107
-
108
-
109
-
110
-
111
-
112
-
113
-
114
-
115
- <!-- if method_list -->
116
- <div id="methods">
117
- <h3 class="section-bar">Public Class methods</h3>
118
-
119
- <div id="method-M000034" class="method-detail">
120
- <a name="M000034"></a>
121
-
122
- <div class="method-heading">
123
- <a href="RobotRules.src/M000034.html" target="Code" class="method-signature"
124
- onclick="popupCode('RobotRules.src/M000034.html');return false;">
125
- <span class="method-name">new</span><span class="method-args">( user_agent )</span>
126
- </a>
127
- </div>
128
-
129
- <div class="method-description">
130
- </div>
131
- </div>
132
-
133
- <h3 class="section-bar">Public Instance methods</h3>
134
-
135
- <div id="method-M000036" class="method-detail">
136
- <a name="M000036"></a>
137
-
138
- <div class="method-heading">
139
- <a href="RobotRules.src/M000036.html" target="Code" class="method-signature"
140
- onclick="popupCode('RobotRules.src/M000036.html');return false;">
141
- <span class="method-name">allowed?</span><span class="method-args">( text_uri )</span>
142
- </a>
143
- </div>
144
-
145
- <div class="method-description">
146
- </div>
147
- </div>
148
-
149
- <div id="method-M000035" class="method-detail">
150
- <a name="M000035"></a>
151
-
152
- <div class="method-heading">
153
- <a href="RobotRules.src/M000035.html" target="Code" class="method-signature"
154
- onclick="popupCode('RobotRules.src/M000035.html');return false;">
155
- <span class="method-name">parse</span><span class="method-args">( text_uri, robots_data )</span>
156
- </a>
157
- </div>
158
-
159
- <div class="method-description">
160
- </div>
161
- </div>
162
-
163
-
164
- </div>
165
-
166
-
167
- </div>
168
-
169
-
170
- <div id="validator-badges">
171
- <p><small><a href="http://validator.w3.org/check/referer">[Validate]</a></small></p>
172
- </div>
173
-
174
- </body>
175
- </html>
data/doc/classes/RobotRules.src/M000034.html DELETED
@@ -1,19 +0,0 @@
1
- <?xml version="1.0" encoding="iso-8859-1"?>
2
- <!DOCTYPE html
3
- PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
4
- "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
5
-
6
- <html>
7
- <head>
8
- <title>new (RobotRules)</title>
9
- <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
10
- <link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
11
- </head>
12
- <body class="standalone-code">
13
- <pre><span class="ruby-comment cmt"># File lib/spider/robot_rules.rb, line 10</span>
14
- <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">initialize</span>( <span class="ruby-identifier">user_agent</span> )
15
- <span class="ruby-ivar">@user_agent</span> = <span class="ruby-identifier">user_agent</span>.<span class="ruby-identifier">scan</span>(<span class="ruby-regexp re">/\S+/</span>).<span class="ruby-identifier">first</span>.<span class="ruby-identifier">sub</span>(<span class="ruby-regexp re">%r{/.*}</span>, <span class="ruby-value str">&quot;&quot;</span>).<span class="ruby-identifier">downcase</span>
16
- <span class="ruby-ivar">@rules</span> = <span class="ruby-constant">Hash</span>.<span class="ruby-identifier">new</span> { <span class="ruby-operator">|</span><span class="ruby-identifier">rules</span>, <span class="ruby-identifier">rule</span><span class="ruby-operator">|</span> <span class="ruby-identifier">rules</span>[<span class="ruby-identifier">rule</span>] = <span class="ruby-constant">Array</span>.<span class="ruby-identifier">new</span> }
17
- <span class="ruby-keyword kw">end</span></pre>
18
- </body>
19
- </html>
data/doc/classes/RobotRules.src/M000035.html DELETED
@@ -1,67 +0,0 @@
1
- <?xml version="1.0" encoding="iso-8859-1"?>
2
- <!DOCTYPE html
3
- PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
4
- "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
5
-
6
- <html>
7
- <head>
8
- <title>parse (RobotRules)</title>
9
- <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
10
- <link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
11
- </head>
12
- <body class="standalone-code">
13
- <pre><span class="ruby-comment cmt"># File lib/spider/robot_rules.rb, line 15</span>
14
- <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">parse</span>( <span class="ruby-identifier">text_uri</span>, <span class="ruby-identifier">robots_data</span> )
15
- <span class="ruby-identifier">uri</span> = <span class="ruby-constant">URI</span>.<span class="ruby-identifier">parse</span>(<span class="ruby-identifier">text_uri</span>)
16
- <span class="ruby-identifier">location</span> = <span class="ruby-node">&quot;#{uri.host}:#{uri.port}&quot;</span>
17
- <span class="ruby-ivar">@rules</span>.<span class="ruby-identifier">delete</span>(<span class="ruby-identifier">location</span>)
18
-
19
- <span class="ruby-identifier">rules</span> = <span class="ruby-identifier">robots_data</span>.<span class="ruby-identifier">split</span>(<span class="ruby-regexp re">/[\015\012]+/</span>).<span class="ruby-identifier">map</span> <span class="ruby-keyword kw">do</span> <span class="ruby-operator">|</span><span class="ruby-identifier">rule</span><span class="ruby-operator">|</span>
20
- <span class="ruby-identifier">rule</span>.<span class="ruby-identifier">sub</span>(<span class="ruby-regexp re">/\s*#.*$/</span>, <span class="ruby-value str">&quot;&quot;</span>)
21
- <span class="ruby-keyword kw">end</span>
22
- <span class="ruby-identifier">anon_rules</span> = <span class="ruby-constant">Array</span>.<span class="ruby-identifier">new</span>
23
- <span class="ruby-identifier">my_rules</span> = <span class="ruby-constant">Array</span>.<span class="ruby-identifier">new</span>
24
- <span class="ruby-identifier">current</span> = <span class="ruby-identifier">anon_rules</span>
25
- <span class="ruby-identifier">rules</span>.<span class="ruby-identifier">each</span> <span class="ruby-keyword kw">do</span> <span class="ruby-operator">|</span><span class="ruby-identifier">rule</span><span class="ruby-operator">|</span>
26
- <span class="ruby-keyword kw">case</span> <span class="ruby-identifier">rule</span>
27
- <span class="ruby-keyword kw">when</span> <span class="ruby-regexp re">/^\s*User-Agent\s*:\s*(.+?)\s*$/i</span>
28
- <span class="ruby-keyword kw">break</span> <span class="ruby-keyword kw">unless</span> <span class="ruby-identifier">my_rules</span>.<span class="ruby-identifier">empty?</span>
29
-
30
- <span class="ruby-identifier">current</span> = <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">$1</span> <span class="ruby-operator">==</span> <span class="ruby-value str">&quot;*&quot;</span>
31
- <span class="ruby-identifier">anon_rules</span>
32
- <span class="ruby-keyword kw">elsif</span> <span class="ruby-identifier">$1</span>.<span class="ruby-identifier">downcase</span>.<span class="ruby-identifier">index</span>(<span class="ruby-ivar">@user_agent</span>)
33
- <span class="ruby-identifier">my_rules</span>
34
- <span class="ruby-keyword kw">else</span>
35
- <span class="ruby-keyword kw">nil</span>
36
- <span class="ruby-keyword kw">end</span>
37
- <span class="ruby-keyword kw">when</span> <span class="ruby-regexp re">/^\s*Disallow\s*:\s*(.*?)\s*$/i</span>
38
- <span class="ruby-keyword kw">next</span> <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">current</span>.<span class="ruby-identifier">nil?</span>
39
-
40
- <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">$1</span>.<span class="ruby-identifier">empty?</span>
41
- <span class="ruby-identifier">current</span> <span class="ruby-operator">&lt;&lt;</span> <span class="ruby-keyword kw">nil</span>
42
- <span class="ruby-keyword kw">else</span>
43
- <span class="ruby-identifier">disallow</span> = <span class="ruby-constant">URI</span>.<span class="ruby-identifier">parse</span>(<span class="ruby-identifier">$1</span>)
44
-
45
- <span class="ruby-keyword kw">next</span> <span class="ruby-keyword kw">unless</span> <span class="ruby-identifier">disallow</span>.<span class="ruby-identifier">scheme</span>.<span class="ruby-identifier">nil?</span> <span class="ruby-keyword kw">or</span> <span class="ruby-identifier">disallow</span>.<span class="ruby-identifier">scheme</span> <span class="ruby-operator">==</span>
46
- <span class="ruby-identifier">uri</span>.<span class="ruby-identifier">scheme</span>
47
- <span class="ruby-keyword kw">next</span> <span class="ruby-keyword kw">unless</span> <span class="ruby-identifier">disallow</span>.<span class="ruby-identifier">port</span>.<span class="ruby-identifier">nil?</span> <span class="ruby-keyword kw">or</span> <span class="ruby-identifier">disallow</span>.<span class="ruby-identifier">port</span> <span class="ruby-operator">==</span> <span class="ruby-identifier">uri</span>.<span class="ruby-identifier">port</span>
48
- <span class="ruby-keyword kw">next</span> <span class="ruby-keyword kw">unless</span> <span class="ruby-identifier">disallow</span>.<span class="ruby-identifier">host</span>.<span class="ruby-identifier">nil?</span> <span class="ruby-keyword kw">or</span>
49
- <span class="ruby-identifier">disallow</span>.<span class="ruby-identifier">host</span>.<span class="ruby-identifier">downcase</span> <span class="ruby-operator">==</span> <span class="ruby-identifier">uri</span>.<span class="ruby-identifier">host</span>.<span class="ruby-identifier">downcase</span>
50
-
51
- <span class="ruby-identifier">disallow</span> = <span class="ruby-identifier">disallow</span>.<span class="ruby-identifier">path</span>
52
- <span class="ruby-identifier">disallow</span> = <span class="ruby-value str">&quot;/&quot;</span> <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">disallow</span>.<span class="ruby-identifier">empty?</span>
53
- <span class="ruby-identifier">disallow</span> = <span class="ruby-node">&quot;/#{disallow}&quot;</span> <span class="ruby-keyword kw">unless</span> <span class="ruby-identifier">disallow</span>[<span class="ruby-value">0</span>] <span class="ruby-operator">==</span> <span class="ruby-value">?/</span>
54
-
55
- <span class="ruby-identifier">current</span> <span class="ruby-operator">&lt;&lt;</span> <span class="ruby-identifier">disallow</span>
56
- <span class="ruby-keyword kw">end</span>
57
- <span class="ruby-keyword kw">end</span>
58
- <span class="ruby-keyword kw">end</span>
59
-
60
- <span class="ruby-ivar">@rules</span>[<span class="ruby-identifier">location</span>] = <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">my_rules</span>.<span class="ruby-identifier">empty?</span>
61
- <span class="ruby-identifier">anon_rules</span>.<span class="ruby-identifier">compact</span>
62
- <span class="ruby-keyword kw">else</span>
63
- <span class="ruby-identifier">my_rules</span>.<span class="ruby-identifier">compact</span>
64
- <span class="ruby-keyword kw">end</span>
65
- <span class="ruby-keyword kw">end</span></pre>
66
- </body>
67
- </html>
data/doc/classes/RobotRules.src/M000036.html DELETED
@@ -1,24 +0,0 @@
1
- <?xml version="1.0" encoding="iso-8859-1"?>
2
- <!DOCTYPE html
3
- PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
4
- "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
5
-
6
- <html>
7
- <head>
8
- <title>allowed? (RobotRules)</title>
9
- <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
10
- <link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
11
- </head>
12
- <body class="standalone-code">
13
- <pre><span class="ruby-comment cmt"># File lib/spider/robot_rules.rb, line 68</span>
14
- <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">allowed?</span>( <span class="ruby-identifier">text_uri</span> )
15
- <span class="ruby-identifier">uri</span> = <span class="ruby-constant">URI</span>.<span class="ruby-identifier">parse</span>(<span class="ruby-identifier">text_uri</span>)
16
- <span class="ruby-identifier">location</span> = <span class="ruby-node">&quot;#{uri.host}:#{uri.port}&quot;</span>
17
- <span class="ruby-identifier">path</span> = <span class="ruby-identifier">uri</span>.<span class="ruby-identifier">path</span>
18
-
19
- <span class="ruby-keyword kw">return</span> <span class="ruby-keyword kw">true</span> <span class="ruby-keyword kw">unless</span> <span class="ruby-node">%w{http https}</span>.<span class="ruby-identifier">include?</span>(<span class="ruby-identifier">uri</span>.<span class="ruby-identifier">scheme</span>)
20
-
21
- <span class="ruby-keyword kw">not</span> <span class="ruby-ivar">@rules</span>[<span class="ruby-identifier">location</span>].<span class="ruby-identifier">any?</span> { <span class="ruby-operator">|</span><span class="ruby-identifier">rule</span><span class="ruby-operator">|</span> <span class="ruby-identifier">path</span>.<span class="ruby-identifier">index</span>(<span class="ruby-identifier">rule</span>) <span class="ruby-operator">==</span> <span class="ruby-value">0</span> }
22
- <span class="ruby-keyword kw">end</span></pre>
23
- </body>
24
- </html>
data/doc/classes/Spider.html DELETED
@@ -1,170 +0,0 @@
1
- <?xml version="1.0" encoding="iso-8859-1"?>
2
- <!DOCTYPE html
3
- PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
4
- "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
5
-
6
- <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
7
- <head>
8
- <title>Class: Spider</title>
9
- <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
10
- <meta http-equiv="Content-Script-Type" content="text/javascript" />
11
- <link rel="stylesheet" href=".././rdoc-style.css" type="text/css" media="screen" />
12
- <script type="text/javascript">
13
- // <![CDATA[
14
-
15
- function popupCode( url ) {
16
- window.open(url, "Code", "resizable=yes,scrollbars=yes,toolbar=no,status=no,height=150,width=400")
17
- }
18
-
19
- function toggleCode( id ) {
20
- if ( document.getElementById )
21
- elem = document.getElementById( id );
22
- else if ( document.all )
23
- elem = eval( "document.all." + id );
24
- else
25
- return false;
26
-
27
- elemStyle = elem.style;
28
-
29
- if ( elemStyle.display != "block" ) {
30
- elemStyle.display = "block"
31
- } else {
32
- elemStyle.display = "none"
33
- }
34
-
35
- return true;
36
- }
37
-
38
- // Make codeblocks hidden by default
39
- document.writeln( "<style type=\"text/css\">div.method-source-code { display: none }</style>" )
40
-
41
- // ]]>
42
- </script>
43
-
44
- </head>
45
- <body>
46
-
47
-
48
-
49
- <div id="classHeader">
50
- <table class="header-table">
51
- <tr class="top-aligned-row">
52
- <td><strong>Class</strong></td>
53
- <td class="class-name-in-header">Spider</td>
54
- </tr>
55
- <tr class="top-aligned-row">
56
- <td><strong>In:</strong></td>
57
- <td>
58
- <a href="../files/lib/spider_rb.html">
59
- lib/spider.rb
60
- </a>
61
- <br />
62
- </td>
63
- </tr>
64
-
65
- <tr class="top-aligned-row">
66
- <td><strong>Parent:</strong></td>
67
- <td>
68
- Object
69
- </td>
70
- </tr>
71
- </table>
72
- </div>
73
- <!-- banner header -->
74
-
75
- <div id="bodyContent">
76
-
77
-
78
-
79
- <div id="contextContent">
80
-
81
- <div id="description">
82
- <p>
83
- A spidering library for Ruby. Handles robots.txt, scraping, finding more
84
- links, and doing it all over again.
85
- </p>
86
-
87
- </div>
88
-
89
-
90
- </div>
91
-
92
- <div id="method-list">
93
- <h3 class="section-bar">Methods</h3>
94
-
95
- <div class="name-list">
96
- <a href="#M000029">start_at</a>&nbsp;&nbsp;
97
- </div>
98
- </div>
99
-
100
- </div>
101
-
102
-
103
- <!-- if includes -->
104
-
105
- <div id="section">
106
-
107
-
108
-
109
-
110
-
111
-
112
-
113
-
114
- <!-- if method_list -->
115
- <div id="methods">
116
- <h3 class="section-bar">Public Class methods</h3>
117
-
118
- <div id="method-M000029" class="method-detail">
119
- <a name="M000029"></a>
120
-
121
- <div class="method-heading">
122
- <a href="Spider.src/M000029.html" target="Code" class="method-signature"
123
- onclick="popupCode('Spider.src/M000029.html');return false;">
124
- <span class="method-name">start_at</span><span class="method-args">(a_url, &amp;block)</span>
125
- </a>
126
- </div>
127
-
128
- <div class="method-description">
129
- <p>
130
- Runs the spider starting at the given URL. Also takes a block that is given
131
- the <a href="SpiderInstance.html">SpiderInstance</a>. Use the block to
132
- define the rules and handlers for the discovered Web pages. See <a
133
- href="SpiderInstance.html">SpiderInstance</a> for the possible rules and
134
- handlers.
135
- </p>
136
- <pre>
137
- Spider.start_at('http://mike-burns.com/') do |s|
138
- s.add_url_check do |a_url|
139
- a_url =~ %r{^http://mike-burns.com.*}
140
- end
141
-
142
- s.on 404 do |a_url, resp, prior_url|
143
- puts &quot;URL not found: #{a_url}&quot;
144
- end
145
-
146
- s.on :success do |a_url, resp, prior_url|
147
- puts &quot;body: #{resp.body}&quot;
148
- end
149
-
150
- s.on :every do |a_url, resp, prior_url|
151
- puts &quot;URL returned anything: #{a_url} with this code #{resp.code}&quot;
152
- end
153
- end
154
- </pre>
155
- </div>
156
- </div>
157
-
158
-
159
- </div>
160
-
161
-
162
- </div>
163
-
164
-
165
- <div id="validator-badges">
166
- <p><small><a href="http://validator.w3.org/check/referer">[Validate]</a></small></p>
167
- </div>
168
-
169
- </body>
170
- </html>