spider 0.4.4 → 0.5.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (74) hide show
  1. checksums.yaml +7 -0
  2. data/AUTHORS +12 -0
  3. data/CHANGES +6 -0
  4. data/LICENSE +21 -0
  5. data/{README → README.md} +50 -43
  6. data/lib/spider.rb +12 -29
  7. data/lib/spider/included_in_memcached.rb +1 -24
  8. data/lib/spider/next_urls_in_sqs.rb +6 -29
  9. data/lib/spider/robot_rules.rb +61 -57
  10. data/lib/spider/spider_instance.rb +8 -31
  11. data/spider.gemspec +4 -2
  12. metadata +33 -124
  13. data/doc/classes/BeStaticServerPages.html +0 -197
  14. data/doc/classes/BeStaticServerPages.src/M000030.html +0 -19
  15. data/doc/classes/BeStaticServerPages.src/M000031.html +0 -19
  16. data/doc/classes/BeStaticServerPages.src/M000032.html +0 -18
  17. data/doc/classes/BeStaticServerPages.src/M000033.html +0 -18
  18. data/doc/classes/IncludedInMemcached.html +0 -199
  19. data/doc/classes/IncludedInMemcached.src/M000015.html +0 -18
  20. data/doc/classes/IncludedInMemcached.src/M000016.html +0 -18
  21. data/doc/classes/IncludedInMemcached.src/M000017.html +0 -18
  22. data/doc/classes/LoopingServlet.html +0 -137
  23. data/doc/classes/LoopingServlet.src/M000037.html +0 -23
  24. data/doc/classes/NextUrlsInSQS.html +0 -204
  25. data/doc/classes/NextUrlsInSQS.src/M000018.html +0 -19
  26. data/doc/classes/NextUrlsInSQS.src/M000019.html +0 -22
  27. data/doc/classes/NextUrlsInSQS.src/M000020.html +0 -19
  28. data/doc/classes/QueryServlet.html +0 -137
  29. data/doc/classes/QueryServlet.src/M000038.html +0 -19
  30. data/doc/classes/RobotRules.html +0 -175
  31. data/doc/classes/RobotRules.src/M000034.html +0 -19
  32. data/doc/classes/RobotRules.src/M000035.html +0 -67
  33. data/doc/classes/RobotRules.src/M000036.html +0 -24
  34. data/doc/classes/Spider.html +0 -170
  35. data/doc/classes/Spider.src/M000029.html +0 -21
  36. data/doc/classes/SpiderInstance.html +0 -345
  37. data/doc/classes/SpiderInstance.src/M000021.html +0 -18
  38. data/doc/classes/SpiderInstance.src/M000022.html +0 -22
  39. data/doc/classes/SpiderInstance.src/M000023.html +0 -22
  40. data/doc/classes/SpiderInstance.src/M000024.html +0 -24
  41. data/doc/classes/SpiderInstance.src/M000025.html +0 -18
  42. data/doc/classes/SpiderInstance.src/M000026.html +0 -18
  43. data/doc/classes/SpiderInstance.src/M000027.html +0 -18
  44. data/doc/classes/SpiderInstance.src/M000028.html +0 -18
  45. data/doc/created.rid +0 -1
  46. data/doc/files/README.html +0 -223
  47. data/doc/files/lib/spider/included_in_memcached_rb.html +0 -142
  48. data/doc/files/lib/spider/next_urls_in_sqs_rb.html +0 -144
  49. data/doc/files/lib/spider/robot_rules_rb.html +0 -114
  50. data/doc/files/lib/spider/spider_instance_rb.html +0 -117
  51. data/doc/files/lib/spider_rb.html +0 -254
  52. data/doc/files/spec/spec_helper_rb.html +0 -196
  53. data/doc/files/spec/spec_helper_rb.src/M000001.html +0 -20
  54. data/doc/files/spec/spec_helper_rb.src/M000002.html +0 -26
  55. data/doc/files/spec/spec_helper_rb.src/M000003.html +0 -24
  56. data/doc/files/spec/spec_helper_rb.src/M000004.html +0 -18
  57. data/doc/files/spec/spec_helper_rb.src/M000005.html +0 -23
  58. data/doc/files/spec/spider/included_in_memcached_spec_rb.html +0 -142
  59. data/doc/files/spec/spider/included_in_memcached_spec_rb.src/M000006.html +0 -19
  60. data/doc/files/spec/spider/included_in_memcached_spec_rb.src/M000007.html +0 -18
  61. data/doc/files/spec/spider/spider_instance_spec_rb.html +0 -210
  62. data/doc/files/spec/spider/spider_instance_spec_rb.src/M000008.html +0 -21
  63. data/doc/files/spec/spider/spider_instance_spec_rb.src/M000009.html +0 -19
  64. data/doc/files/spec/spider/spider_instance_spec_rb.src/M000010.html +0 -19
  65. data/doc/files/spec/spider/spider_instance_spec_rb.src/M000011.html +0 -27
  66. data/doc/files/spec/spider/spider_instance_spec_rb.src/M000012.html +0 -26
  67. data/doc/files/spec/spider/spider_instance_spec_rb.src/M000013.html +0 -27
  68. data/doc/files/spec/spider_spec_rb.html +0 -127
  69. data/doc/files/spec/spider_spec_rb.src/M000014.html +0 -23
  70. data/doc/fr_class_index.html +0 -34
  71. data/doc/fr_file_index.html +0 -35
  72. data/doc/fr_method_index.html +0 -64
  73. data/doc/index.html +0 -24
  74. data/doc/rdoc-style.css +0 -208
data/doc/classes/RobotRules.html DELETED
@@ -1,175 +0,0 @@
1
- <?xml version="1.0" encoding="iso-8859-1"?>
2
- <!DOCTYPE html
3
- PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
4
- "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
5
-
6
- <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
7
- <head>
8
- <title>Class: RobotRules</title>
9
- <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
10
- <meta http-equiv="Content-Script-Type" content="text/javascript" />
11
- <link rel="stylesheet" href=".././rdoc-style.css" type="text/css" media="screen" />
12
- <script type="text/javascript">
13
- // <![CDATA[
14
-
15
- function popupCode( url ) {
16
- window.open(url, "Code", "resizable=yes,scrollbars=yes,toolbar=no,status=no,height=150,width=400")
17
- }
18
-
19
- function toggleCode( id ) {
20
- if ( document.getElementById )
21
- elem = document.getElementById( id );
22
- else if ( document.all )
23
- elem = eval( "document.all." + id );
24
- else
25
- return false;
26
-
27
- elemStyle = elem.style;
28
-
29
- if ( elemStyle.display != "block" ) {
30
- elemStyle.display = "block"
31
- } else {
32
- elemStyle.display = "none"
33
- }
34
-
35
- return true;
36
- }
37
-
38
- // Make codeblocks hidden by default
39
- document.writeln( "<style type=\"text/css\">div.method-source-code { display: none }</style>" )
40
-
41
- // ]]>
42
- </script>
43
-
44
- </head>
45
- <body>
46
-
47
-
48
-
49
- <div id="classHeader">
50
- <table class="header-table">
51
- <tr class="top-aligned-row">
52
- <td><strong>Class</strong></td>
53
- <td class="class-name-in-header">RobotRules</td>
54
- </tr>
55
- <tr class="top-aligned-row">
56
- <td><strong>In:</strong></td>
57
- <td>
58
- <a href="../files/lib/spider/robot_rules_rb.html">
59
- lib/spider/robot_rules.rb
60
- </a>
61
- <br />
62
- </td>
63
- </tr>
64
-
65
- <tr class="top-aligned-row">
66
- <td><strong>Parent:</strong></td>
67
- <td>
68
- Object
69
- </td>
70
- </tr>
71
- </table>
72
- </div>
73
- <!-- banner header -->
74
-
75
- <div id="bodyContent">
76
-
77
-
78
-
79
- <div id="contextContent">
80
-
81
- <div id="description">
82
- <p>
83
- Based on Perl&#8216;s WWW::RobotRules module, by Gisle Aas.
84
- </p>
85
-
86
- </div>
87
-
88
-
89
- </div>
90
-
91
- <div id="method-list">
92
- <h3 class="section-bar">Methods</h3>
93
-
94
- <div class="name-list">
95
- <a href="#M000036">allowed?</a>&nbsp;&nbsp;
96
- <a href="#M000034">new</a>&nbsp;&nbsp;
97
- <a href="#M000035">parse</a>&nbsp;&nbsp;
98
- </div>
99
- </div>
100
-
101
- </div>
102
-
103
-
104
- <!-- if includes -->
105
-
106
- <div id="section">
107
-
108
-
109
-
110
-
111
-
112
-
113
-
114
-
115
- <!-- if method_list -->
116
- <div id="methods">
117
- <h3 class="section-bar">Public Class methods</h3>
118
-
119
- <div id="method-M000034" class="method-detail">
120
- <a name="M000034"></a>
121
-
122
- <div class="method-heading">
123
- <a href="RobotRules.src/M000034.html" target="Code" class="method-signature"
124
- onclick="popupCode('RobotRules.src/M000034.html');return false;">
125
- <span class="method-name">new</span><span class="method-args">( user_agent )</span>
126
- </a>
127
- </div>
128
-
129
- <div class="method-description">
130
- </div>
131
- </div>
132
-
133
- <h3 class="section-bar">Public Instance methods</h3>
134
-
135
- <div id="method-M000036" class="method-detail">
136
- <a name="M000036"></a>
137
-
138
- <div class="method-heading">
139
- <a href="RobotRules.src/M000036.html" target="Code" class="method-signature"
140
- onclick="popupCode('RobotRules.src/M000036.html');return false;">
141
- <span class="method-name">allowed?</span><span class="method-args">( text_uri )</span>
142
- </a>
143
- </div>
144
-
145
- <div class="method-description">
146
- </div>
147
- </div>
148
-
149
- <div id="method-M000035" class="method-detail">
150
- <a name="M000035"></a>
151
-
152
- <div class="method-heading">
153
- <a href="RobotRules.src/M000035.html" target="Code" class="method-signature"
154
- onclick="popupCode('RobotRules.src/M000035.html');return false;">
155
- <span class="method-name">parse</span><span class="method-args">( text_uri, robots_data )</span>
156
- </a>
157
- </div>
158
-
159
- <div class="method-description">
160
- </div>
161
- </div>
162
-
163
-
164
- </div>
165
-
166
-
167
- </div>
168
-
169
-
170
- <div id="validator-badges">
171
- <p><small><a href="http://validator.w3.org/check/referer">[Validate]</a></small></p>
172
- </div>
173
-
174
- </body>
175
- </html>
data/doc/classes/RobotRules.src/M000034.html DELETED
@@ -1,19 +0,0 @@
1
- <?xml version="1.0" encoding="iso-8859-1"?>
2
- <!DOCTYPE html
3
- PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
4
- "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
5
-
6
- <html>
7
- <head>
8
- <title>new (RobotRules)</title>
9
- <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
10
- <link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
11
- </head>
12
- <body class="standalone-code">
13
- <pre><span class="ruby-comment cmt"># File lib/spider/robot_rules.rb, line 10</span>
14
- <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">initialize</span>( <span class="ruby-identifier">user_agent</span> )
15
- <span class="ruby-ivar">@user_agent</span> = <span class="ruby-identifier">user_agent</span>.<span class="ruby-identifier">scan</span>(<span class="ruby-regexp re">/\S+/</span>).<span class="ruby-identifier">first</span>.<span class="ruby-identifier">sub</span>(<span class="ruby-regexp re">%r{/.*}</span>, <span class="ruby-value str">&quot;&quot;</span>).<span class="ruby-identifier">downcase</span>
16
- <span class="ruby-ivar">@rules</span> = <span class="ruby-constant">Hash</span>.<span class="ruby-identifier">new</span> { <span class="ruby-operator">|</span><span class="ruby-identifier">rules</span>, <span class="ruby-identifier">rule</span><span class="ruby-operator">|</span> <span class="ruby-identifier">rules</span>[<span class="ruby-identifier">rule</span>] = <span class="ruby-constant">Array</span>.<span class="ruby-identifier">new</span> }
17
- <span class="ruby-keyword kw">end</span></pre>
18
- </body>
19
- </html>
data/doc/classes/RobotRules.src/M000035.html DELETED
@@ -1,67 +0,0 @@
1
- <?xml version="1.0" encoding="iso-8859-1"?>
2
- <!DOCTYPE html
3
- PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
4
- "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
5
-
6
- <html>
7
- <head>
8
- <title>parse (RobotRules)</title>
9
- <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
10
- <link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
11
- </head>
12
- <body class="standalone-code">
13
- <pre><span class="ruby-comment cmt"># File lib/spider/robot_rules.rb, line 15</span>
14
- <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">parse</span>( <span class="ruby-identifier">text_uri</span>, <span class="ruby-identifier">robots_data</span> )
15
- <span class="ruby-identifier">uri</span> = <span class="ruby-constant">URI</span>.<span class="ruby-identifier">parse</span>(<span class="ruby-identifier">text_uri</span>)
16
- <span class="ruby-identifier">location</span> = <span class="ruby-node">&quot;#{uri.host}:#{uri.port}&quot;</span>
17
- <span class="ruby-ivar">@rules</span>.<span class="ruby-identifier">delete</span>(<span class="ruby-identifier">location</span>)
18
-
19
- <span class="ruby-identifier">rules</span> = <span class="ruby-identifier">robots_data</span>.<span class="ruby-identifier">split</span>(<span class="ruby-regexp re">/[\015\012]+/</span>).<span class="ruby-identifier">map</span> <span class="ruby-keyword kw">do</span> <span class="ruby-operator">|</span><span class="ruby-identifier">rule</span><span class="ruby-operator">|</span>
20
- <span class="ruby-identifier">rule</span>.<span class="ruby-identifier">sub</span>(<span class="ruby-regexp re">/\s*#.*$/</span>, <span class="ruby-value str">&quot;&quot;</span>)
21
- <span class="ruby-keyword kw">end</span>
22
- <span class="ruby-identifier">anon_rules</span> = <span class="ruby-constant">Array</span>.<span class="ruby-identifier">new</span>
23
- <span class="ruby-identifier">my_rules</span> = <span class="ruby-constant">Array</span>.<span class="ruby-identifier">new</span>
24
- <span class="ruby-identifier">current</span> = <span class="ruby-identifier">anon_rules</span>
25
- <span class="ruby-identifier">rules</span>.<span class="ruby-identifier">each</span> <span class="ruby-keyword kw">do</span> <span class="ruby-operator">|</span><span class="ruby-identifier">rule</span><span class="ruby-operator">|</span>
26
- <span class="ruby-keyword kw">case</span> <span class="ruby-identifier">rule</span>
27
- <span class="ruby-keyword kw">when</span> <span class="ruby-regexp re">/^\s*User-Agent\s*:\s*(.+?)\s*$/i</span>
28
- <span class="ruby-keyword kw">break</span> <span class="ruby-keyword kw">unless</span> <span class="ruby-identifier">my_rules</span>.<span class="ruby-identifier">empty?</span>
29
-
30
- <span class="ruby-identifier">current</span> = <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">$1</span> <span class="ruby-operator">==</span> <span class="ruby-value str">&quot;*&quot;</span>
31
- <span class="ruby-identifier">anon_rules</span>
32
- <span class="ruby-keyword kw">elsif</span> <span class="ruby-identifier">$1</span>.<span class="ruby-identifier">downcase</span>.<span class="ruby-identifier">index</span>(<span class="ruby-ivar">@user_agent</span>)
33
- <span class="ruby-identifier">my_rules</span>
34
- <span class="ruby-keyword kw">else</span>
35
- <span class="ruby-keyword kw">nil</span>
36
- <span class="ruby-keyword kw">end</span>
37
- <span class="ruby-keyword kw">when</span> <span class="ruby-regexp re">/^\s*Disallow\s*:\s*(.*?)\s*$/i</span>
38
- <span class="ruby-keyword kw">next</span> <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">current</span>.<span class="ruby-identifier">nil?</span>
39
-
40
- <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">$1</span>.<span class="ruby-identifier">empty?</span>
41
- <span class="ruby-identifier">current</span> <span class="ruby-operator">&lt;&lt;</span> <span class="ruby-keyword kw">nil</span>
42
- <span class="ruby-keyword kw">else</span>
43
- <span class="ruby-identifier">disallow</span> = <span class="ruby-constant">URI</span>.<span class="ruby-identifier">parse</span>(<span class="ruby-identifier">$1</span>)
44
-
45
- <span class="ruby-keyword kw">next</span> <span class="ruby-keyword kw">unless</span> <span class="ruby-identifier">disallow</span>.<span class="ruby-identifier">scheme</span>.<span class="ruby-identifier">nil?</span> <span class="ruby-keyword kw">or</span> <span class="ruby-identifier">disallow</span>.<span class="ruby-identifier">scheme</span> <span class="ruby-operator">==</span>
46
- <span class="ruby-identifier">uri</span>.<span class="ruby-identifier">scheme</span>
47
- <span class="ruby-keyword kw">next</span> <span class="ruby-keyword kw">unless</span> <span class="ruby-identifier">disallow</span>.<span class="ruby-identifier">port</span>.<span class="ruby-identifier">nil?</span> <span class="ruby-keyword kw">or</span> <span class="ruby-identifier">disallow</span>.<span class="ruby-identifier">port</span> <span class="ruby-operator">==</span> <span class="ruby-identifier">uri</span>.<span class="ruby-identifier">port</span>
48
- <span class="ruby-keyword kw">next</span> <span class="ruby-keyword kw">unless</span> <span class="ruby-identifier">disallow</span>.<span class="ruby-identifier">host</span>.<span class="ruby-identifier">nil?</span> <span class="ruby-keyword kw">or</span>
49
- <span class="ruby-identifier">disallow</span>.<span class="ruby-identifier">host</span>.<span class="ruby-identifier">downcase</span> <span class="ruby-operator">==</span> <span class="ruby-identifier">uri</span>.<span class="ruby-identifier">host</span>.<span class="ruby-identifier">downcase</span>
50
-
51
- <span class="ruby-identifier">disallow</span> = <span class="ruby-identifier">disallow</span>.<span class="ruby-identifier">path</span>
52
- <span class="ruby-identifier">disallow</span> = <span class="ruby-value str">&quot;/&quot;</span> <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">disallow</span>.<span class="ruby-identifier">empty?</span>
53
- <span class="ruby-identifier">disallow</span> = <span class="ruby-node">&quot;/#{disallow}&quot;</span> <span class="ruby-keyword kw">unless</span> <span class="ruby-identifier">disallow</span>[<span class="ruby-value">0</span>] <span class="ruby-operator">==</span> <span class="ruby-value">?/</span>
54
-
55
- <span class="ruby-identifier">current</span> <span class="ruby-operator">&lt;&lt;</span> <span class="ruby-identifier">disallow</span>
56
- <span class="ruby-keyword kw">end</span>
57
- <span class="ruby-keyword kw">end</span>
58
- <span class="ruby-keyword kw">end</span>
59
-
60
- <span class="ruby-ivar">@rules</span>[<span class="ruby-identifier">location</span>] = <span class="ruby-keyword kw">if</span> <span class="ruby-identifier">my_rules</span>.<span class="ruby-identifier">empty?</span>
61
- <span class="ruby-identifier">anon_rules</span>.<span class="ruby-identifier">compact</span>
62
- <span class="ruby-keyword kw">else</span>
63
- <span class="ruby-identifier">my_rules</span>.<span class="ruby-identifier">compact</span>
64
- <span class="ruby-keyword kw">end</span>
65
- <span class="ruby-keyword kw">end</span></pre>
66
- </body>
67
- </html>
data/doc/classes/RobotRules.src/M000036.html DELETED
@@ -1,24 +0,0 @@
1
- <?xml version="1.0" encoding="iso-8859-1"?>
2
- <!DOCTYPE html
3
- PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
4
- "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
5
-
6
- <html>
7
- <head>
8
- <title>allowed? (RobotRules)</title>
9
- <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
10
- <link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
11
- </head>
12
- <body class="standalone-code">
13
- <pre><span class="ruby-comment cmt"># File lib/spider/robot_rules.rb, line 68</span>
14
- <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">allowed?</span>( <span class="ruby-identifier">text_uri</span> )
15
- <span class="ruby-identifier">uri</span> = <span class="ruby-constant">URI</span>.<span class="ruby-identifier">parse</span>(<span class="ruby-identifier">text_uri</span>)
16
- <span class="ruby-identifier">location</span> = <span class="ruby-node">&quot;#{uri.host}:#{uri.port}&quot;</span>
17
- <span class="ruby-identifier">path</span> = <span class="ruby-identifier">uri</span>.<span class="ruby-identifier">path</span>
18
-
19
- <span class="ruby-keyword kw">return</span> <span class="ruby-keyword kw">true</span> <span class="ruby-keyword kw">unless</span> <span class="ruby-node">%w{http https}</span>.<span class="ruby-identifier">include?</span>(<span class="ruby-identifier">uri</span>.<span class="ruby-identifier">scheme</span>)
20
-
21
- <span class="ruby-keyword kw">not</span> <span class="ruby-ivar">@rules</span>[<span class="ruby-identifier">location</span>].<span class="ruby-identifier">any?</span> { <span class="ruby-operator">|</span><span class="ruby-identifier">rule</span><span class="ruby-operator">|</span> <span class="ruby-identifier">path</span>.<span class="ruby-identifier">index</span>(<span class="ruby-identifier">rule</span>) <span class="ruby-operator">==</span> <span class="ruby-value">0</span> }
22
- <span class="ruby-keyword kw">end</span></pre>
23
- </body>
24
- </html>
data/doc/classes/Spider.html DELETED
@@ -1,170 +0,0 @@
1
- <?xml version="1.0" encoding="iso-8859-1"?>
2
- <!DOCTYPE html
3
- PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
4
- "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
5
-
6
- <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
7
- <head>
8
- <title>Class: Spider</title>
9
- <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
10
- <meta http-equiv="Content-Script-Type" content="text/javascript" />
11
- <link rel="stylesheet" href=".././rdoc-style.css" type="text/css" media="screen" />
12
- <script type="text/javascript">
13
- // <![CDATA[
14
-
15
- function popupCode( url ) {
16
- window.open(url, "Code", "resizable=yes,scrollbars=yes,toolbar=no,status=no,height=150,width=400")
17
- }
18
-
19
- function toggleCode( id ) {
20
- if ( document.getElementById )
21
- elem = document.getElementById( id );
22
- else if ( document.all )
23
- elem = eval( "document.all." + id );
24
- else
25
- return false;
26
-
27
- elemStyle = elem.style;
28
-
29
- if ( elemStyle.display != "block" ) {
30
- elemStyle.display = "block"
31
- } else {
32
- elemStyle.display = "none"
33
- }
34
-
35
- return true;
36
- }
37
-
38
- // Make codeblocks hidden by default
39
- document.writeln( "<style type=\"text/css\">div.method-source-code { display: none }</style>" )
40
-
41
- // ]]>
42
- </script>
43
-
44
- </head>
45
- <body>
46
-
47
-
48
-
49
- <div id="classHeader">
50
- <table class="header-table">
51
- <tr class="top-aligned-row">
52
- <td><strong>Class</strong></td>
53
- <td class="class-name-in-header">Spider</td>
54
- </tr>
55
- <tr class="top-aligned-row">
56
- <td><strong>In:</strong></td>
57
- <td>
58
- <a href="../files/lib/spider_rb.html">
59
- lib/spider.rb
60
- </a>
61
- <br />
62
- </td>
63
- </tr>
64
-
65
- <tr class="top-aligned-row">
66
- <td><strong>Parent:</strong></td>
67
- <td>
68
- Object
69
- </td>
70
- </tr>
71
- </table>
72
- </div>
73
- <!-- banner header -->
74
-
75
- <div id="bodyContent">
76
-
77
-
78
-
79
- <div id="contextContent">
80
-
81
- <div id="description">
82
- <p>
83
- A spidering library for Ruby. Handles robots.txt, scraping, finding more
84
- links, and doing it all over again.
85
- </p>
86
-
87
- </div>
88
-
89
-
90
- </div>
91
-
92
- <div id="method-list">
93
- <h3 class="section-bar">Methods</h3>
94
-
95
- <div class="name-list">
96
- <a href="#M000029">start_at</a>&nbsp;&nbsp;
97
- </div>
98
- </div>
99
-
100
- </div>
101
-
102
-
103
- <!-- if includes -->
104
-
105
- <div id="section">
106
-
107
-
108
-
109
-
110
-
111
-
112
-
113
-
114
- <!-- if method_list -->
115
- <div id="methods">
116
- <h3 class="section-bar">Public Class methods</h3>
117
-
118
- <div id="method-M000029" class="method-detail">
119
- <a name="M000029"></a>
120
-
121
- <div class="method-heading">
122
- <a href="Spider.src/M000029.html" target="Code" class="method-signature"
123
- onclick="popupCode('Spider.src/M000029.html');return false;">
124
- <span class="method-name">start_at</span><span class="method-args">(a_url, &amp;block)</span>
125
- </a>
126
- </div>
127
-
128
- <div class="method-description">
129
- <p>
130
- Runs the spider starting at the given URL. Also takes a block that is given
131
- the <a href="SpiderInstance.html">SpiderInstance</a>. Use the block to
132
- define the rules and handlers for the discovered Web pages. See <a
133
- href="SpiderInstance.html">SpiderInstance</a> for the possible rules and
134
- handlers.
135
- </p>
136
- <pre>
137
- Spider.start_at('http://mike-burns.com/') do |s|
138
- s.add_url_check do |a_url|
139
- a_url =~ %r{^http://mike-burns.com.*}
140
- end
141
-
142
- s.on 404 do |a_url, resp, prior_url|
143
- puts &quot;URL not found: #{a_url}&quot;
144
- end
145
-
146
- s.on :success do |a_url, resp, prior_url|
147
- puts &quot;body: #{resp.body}&quot;
148
- end
149
-
150
- s.on :every do |a_url, resp, prior_url|
151
- puts &quot;URL returned anything: #{a_url} with this code #{resp.code}&quot;
152
- end
153
- end
154
- </pre>
155
- </div>
156
- </div>
157
-
158
-
159
- </div>
160
-
161
-
162
- </div>
163
-
164
-
165
- <div id="validator-badges">
166
- <p><small><a href="http://validator.w3.org/check/referer">[Validate]</a></small></p>
167
- </div>
168
-
169
- </body>
170
- </html>