spider 0.4.4 → 0.5.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/AUTHORS +17 -0
- data/CHANGES +16 -0
- data/LICENSE +21 -0
- data/{README → README.md} +73 -44
- data/lib/spider.rb +12 -29
- data/lib/spider/included_in_file.rb +32 -0
- data/lib/spider/included_in_memcached.rb +1 -24
- data/lib/spider/included_in_redis.rb +31 -0
- data/lib/spider/next_urls_in_sqs.rb +6 -29
- data/lib/spider/robot_rules.rb +61 -57
- data/lib/spider/spider_instance.rb +16 -35
- data/spec/spider/included_in_redis_spec.rb +43 -0
- data/spider.gemspec +5 -3
- metadata +38 -125
- data/doc/classes/BeStaticServerPages.html +0 -197
- data/doc/classes/BeStaticServerPages.src/M000030.html +0 -19
- data/doc/classes/BeStaticServerPages.src/M000031.html +0 -19
- data/doc/classes/BeStaticServerPages.src/M000032.html +0 -18
- data/doc/classes/BeStaticServerPages.src/M000033.html +0 -18
- data/doc/classes/IncludedInMemcached.html +0 -199
- data/doc/classes/IncludedInMemcached.src/M000015.html +0 -18
- data/doc/classes/IncludedInMemcached.src/M000016.html +0 -18
- data/doc/classes/IncludedInMemcached.src/M000017.html +0 -18
- data/doc/classes/LoopingServlet.html +0 -137
- data/doc/classes/LoopingServlet.src/M000037.html +0 -23
- data/doc/classes/NextUrlsInSQS.html +0 -204
- data/doc/classes/NextUrlsInSQS.src/M000018.html +0 -19
- data/doc/classes/NextUrlsInSQS.src/M000019.html +0 -22
- data/doc/classes/NextUrlsInSQS.src/M000020.html +0 -19
- data/doc/classes/QueryServlet.html +0 -137
- data/doc/classes/QueryServlet.src/M000038.html +0 -19
- data/doc/classes/RobotRules.html +0 -175
- data/doc/classes/RobotRules.src/M000034.html +0 -19
- data/doc/classes/RobotRules.src/M000035.html +0 -67
- data/doc/classes/RobotRules.src/M000036.html +0 -24
- data/doc/classes/Spider.html +0 -170
- data/doc/classes/Spider.src/M000029.html +0 -21
- data/doc/classes/SpiderInstance.html +0 -345
- data/doc/classes/SpiderInstance.src/M000021.html +0 -18
- data/doc/classes/SpiderInstance.src/M000022.html +0 -22
- data/doc/classes/SpiderInstance.src/M000023.html +0 -22
- data/doc/classes/SpiderInstance.src/M000024.html +0 -24
- data/doc/classes/SpiderInstance.src/M000025.html +0 -18
- data/doc/classes/SpiderInstance.src/M000026.html +0 -18
- data/doc/classes/SpiderInstance.src/M000027.html +0 -18
- data/doc/classes/SpiderInstance.src/M000028.html +0 -18
- data/doc/created.rid +0 -1
- data/doc/files/README.html +0 -223
- data/doc/files/lib/spider/included_in_memcached_rb.html +0 -142
- data/doc/files/lib/spider/next_urls_in_sqs_rb.html +0 -144
- data/doc/files/lib/spider/robot_rules_rb.html +0 -114
- data/doc/files/lib/spider/spider_instance_rb.html +0 -117
- data/doc/files/lib/spider_rb.html +0 -254
- data/doc/files/spec/spec_helper_rb.html +0 -196
- data/doc/files/spec/spec_helper_rb.src/M000001.html +0 -20
- data/doc/files/spec/spec_helper_rb.src/M000002.html +0 -26
- data/doc/files/spec/spec_helper_rb.src/M000003.html +0 -24
- data/doc/files/spec/spec_helper_rb.src/M000004.html +0 -18
- data/doc/files/spec/spec_helper_rb.src/M000005.html +0 -23
- data/doc/files/spec/spider/included_in_memcached_spec_rb.html +0 -142
- data/doc/files/spec/spider/included_in_memcached_spec_rb.src/M000006.html +0 -19
- data/doc/files/spec/spider/included_in_memcached_spec_rb.src/M000007.html +0 -18
- data/doc/files/spec/spider/spider_instance_spec_rb.html +0 -210
- data/doc/files/spec/spider/spider_instance_spec_rb.src/M000008.html +0 -21
- data/doc/files/spec/spider/spider_instance_spec_rb.src/M000009.html +0 -19
- data/doc/files/spec/spider/spider_instance_spec_rb.src/M000010.html +0 -19
- data/doc/files/spec/spider/spider_instance_spec_rb.src/M000011.html +0 -27
- data/doc/files/spec/spider/spider_instance_spec_rb.src/M000012.html +0 -26
- data/doc/files/spec/spider/spider_instance_spec_rb.src/M000013.html +0 -27
- data/doc/files/spec/spider_spec_rb.html +0 -127
- data/doc/files/spec/spider_spec_rb.src/M000014.html +0 -23
- data/doc/fr_class_index.html +0 -34
- data/doc/fr_file_index.html +0 -35
- data/doc/fr_method_index.html +0 -64
- data/doc/index.html +0 -24
- data/doc/rdoc-style.css +0 -208
@@ -1,19 +0,0 @@
|
|
1
|
-
<?xml version="1.0" encoding="iso-8859-1"?>
|
2
|
-
<!DOCTYPE html
|
3
|
-
PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
|
4
|
-
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
|
5
|
-
|
6
|
-
<html>
|
7
|
-
<head>
|
8
|
-
<title>before_specing_memcached (spec/spider/included_in_memcached_spec.rb)</title>
|
9
|
-
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
|
10
|
-
<link rel="stylesheet" href="../../../.././rdoc-style.css" type="text/css" media="screen" />
|
11
|
-
</head>
|
12
|
-
<body class="standalone-code">
|
13
|
-
<pre><span class="ruby-comment cmt"># File spec/spider/included_in_memcached_spec.rb, line 3</span>
|
14
|
-
<span class="ruby-keyword kw">def</span> <span class="ruby-identifier">before_specing_memcached</span>
|
15
|
-
<span class="ruby-identifier">local_require</span> <span class="ruby-value str">'spider/included_in_memcached'</span>
|
16
|
-
<span class="ruby-identifier">system</span>(<span class="ruby-value str">'memcached -d -P /tmp/spider-memcached.pid'</span>)
|
17
|
-
<span class="ruby-keyword kw">end</span></pre>
|
18
|
-
</body>
|
19
|
-
</html>
|
@@ -1,18 +0,0 @@
|
|
1
|
-
<?xml version="1.0" encoding="iso-8859-1"?>
|
2
|
-
<!DOCTYPE html
|
3
|
-
PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
|
4
|
-
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
|
5
|
-
|
6
|
-
<html>
|
7
|
-
<head>
|
8
|
-
<title>after_specing_memcached (spec/spider/included_in_memcached_spec.rb)</title>
|
9
|
-
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
|
10
|
-
<link rel="stylesheet" href="../../../.././rdoc-style.css" type="text/css" media="screen" />
|
11
|
-
</head>
|
12
|
-
<body class="standalone-code">
|
13
|
-
<pre><span class="ruby-comment cmt"># File spec/spider/included_in_memcached_spec.rb, line 8</span>
|
14
|
-
<span class="ruby-keyword kw">def</span> <span class="ruby-identifier">after_specing_memcached</span>
|
15
|
-
<span class="ruby-identifier">system</span>(<span class="ruby-value str">'kill -KILL `cat /tmp/spider-memcached.pid`'</span>)
|
16
|
-
<span class="ruby-keyword kw">end</span></pre>
|
17
|
-
</body>
|
18
|
-
</html>
|
@@ -1,210 +0,0 @@
|
|
1
|
-
<?xml version="1.0" encoding="iso-8859-1"?>
|
2
|
-
<!DOCTYPE html
|
3
|
-
PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
|
4
|
-
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
|
5
|
-
|
6
|
-
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
|
7
|
-
<head>
|
8
|
-
<title>File: spider_instance_spec.rb</title>
|
9
|
-
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
|
10
|
-
<meta http-equiv="Content-Script-Type" content="text/javascript" />
|
11
|
-
<link rel="stylesheet" href="../../.././rdoc-style.css" type="text/css" media="screen" />
|
12
|
-
<script type="text/javascript">
|
13
|
-
// <![CDATA[
|
14
|
-
|
15
|
-
function popupCode( url ) {
|
16
|
-
window.open(url, "Code", "resizable=yes,scrollbars=yes,toolbar=no,status=no,height=150,width=400")
|
17
|
-
}
|
18
|
-
|
19
|
-
function toggleCode( id ) {
|
20
|
-
if ( document.getElementById )
|
21
|
-
elem = document.getElementById( id );
|
22
|
-
else if ( document.all )
|
23
|
-
elem = eval( "document.all." + id );
|
24
|
-
else
|
25
|
-
return false;
|
26
|
-
|
27
|
-
elemStyle = elem.style;
|
28
|
-
|
29
|
-
if ( elemStyle.display != "block" ) {
|
30
|
-
elemStyle.display = "block"
|
31
|
-
} else {
|
32
|
-
elemStyle.display = "none"
|
33
|
-
}
|
34
|
-
|
35
|
-
return true;
|
36
|
-
}
|
37
|
-
|
38
|
-
// Make codeblocks hidden by default
|
39
|
-
document.writeln( "<style type=\"text/css\">div.method-source-code { display: none }</style>" )
|
40
|
-
|
41
|
-
// ]]>
|
42
|
-
</script>
|
43
|
-
|
44
|
-
</head>
|
45
|
-
<body>
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
<div id="fileHeader">
|
50
|
-
<h1>spider_instance_spec.rb</h1>
|
51
|
-
<table class="header-table">
|
52
|
-
<tr class="top-aligned-row">
|
53
|
-
<td><strong>Path:</strong></td>
|
54
|
-
<td>spec/spider/spider_instance_spec.rb
|
55
|
-
</td>
|
56
|
-
</tr>
|
57
|
-
<tr class="top-aligned-row">
|
58
|
-
<td><strong>Last Update:</strong></td>
|
59
|
-
<td>Thu May 21 13:19:06 +0000 2009</td>
|
60
|
-
</tr>
|
61
|
-
</table>
|
62
|
-
</div>
|
63
|
-
<!-- banner header -->
|
64
|
-
|
65
|
-
<div id="bodyContent">
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
<div id="contextContent">
|
70
|
-
|
71
|
-
|
72
|
-
<div id="requires-list">
|
73
|
-
<h3 class="section-bar">Required files</h3>
|
74
|
-
|
75
|
-
<div class="name-list">
|
76
|
-
webrick
|
77
|
-
webrick/https
|
78
|
-
</div>
|
79
|
-
</div>
|
80
|
-
|
81
|
-
</div>
|
82
|
-
|
83
|
-
<div id="method-list">
|
84
|
-
<h3 class="section-bar">Methods</h3>
|
85
|
-
|
86
|
-
<div class="name-list">
|
87
|
-
<a href="#M000012">callback_arguments_on</a>
|
88
|
-
<a href="#M000013">it_should_prevent_cycles_with</a>
|
89
|
-
<a href="#M000010">mock_failed_http</a>
|
90
|
-
<a href="#M000008">mock_http</a>
|
91
|
-
<a href="#M000011">mock_redirect_http</a>
|
92
|
-
<a href="#M000009">mock_successful_http</a>
|
93
|
-
</div>
|
94
|
-
</div>
|
95
|
-
|
96
|
-
</div>
|
97
|
-
|
98
|
-
|
99
|
-
<!-- if includes -->
|
100
|
-
|
101
|
-
<div id="section">
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
<!-- if method_list -->
|
111
|
-
<div id="methods">
|
112
|
-
<h3 class="section-bar">Public Instance methods</h3>
|
113
|
-
|
114
|
-
<div id="method-M000012" class="method-detail">
|
115
|
-
<a name="M000012"></a>
|
116
|
-
|
117
|
-
<div class="method-heading">
|
118
|
-
<a href="spider_instance_spec_rb.src/M000012.html" target="Code" class="method-signature"
|
119
|
-
onclick="popupCode('spider_instance_spec_rb.src/M000012.html');return false;">
|
120
|
-
<span class="method-name">callback_arguments_on</span><span class="method-args">(code)</span>
|
121
|
-
</a>
|
122
|
-
</div>
|
123
|
-
|
124
|
-
<div class="method-description">
|
125
|
-
</div>
|
126
|
-
</div>
|
127
|
-
|
128
|
-
<div id="method-M000013" class="method-detail">
|
129
|
-
<a name="M000013"></a>
|
130
|
-
|
131
|
-
<div class="method-heading">
|
132
|
-
<a href="spider_instance_spec_rb.src/M000013.html" target="Code" class="method-signature"
|
133
|
-
onclick="popupCode('spider_instance_spec_rb.src/M000013.html');return false;">
|
134
|
-
<span class="method-name">it_should_prevent_cycles_with</span><span class="method-args">(cacher)</span>
|
135
|
-
</a>
|
136
|
-
</div>
|
137
|
-
|
138
|
-
<div class="method-description">
|
139
|
-
</div>
|
140
|
-
</div>
|
141
|
-
|
142
|
-
<div id="method-M000010" class="method-detail">
|
143
|
-
<a name="M000010"></a>
|
144
|
-
|
145
|
-
<div class="method-heading">
|
146
|
-
<a href="spider_instance_spec_rb.src/M000010.html" target="Code" class="method-signature"
|
147
|
-
onclick="popupCode('spider_instance_spec_rb.src/M000010.html');return false;">
|
148
|
-
<span class="method-name">mock_failed_http</span><span class="method-args">()</span>
|
149
|
-
</a>
|
150
|
-
</div>
|
151
|
-
|
152
|
-
<div class="method-description">
|
153
|
-
</div>
|
154
|
-
</div>
|
155
|
-
|
156
|
-
<div id="method-M000008" class="method-detail">
|
157
|
-
<a name="M000008"></a>
|
158
|
-
|
159
|
-
<div class="method-heading">
|
160
|
-
<a href="spider_instance_spec_rb.src/M000008.html" target="Code" class="method-signature"
|
161
|
-
onclick="popupCode('spider_instance_spec_rb.src/M000008.html');return false;">
|
162
|
-
<span class="method-name">mock_http</span><span class="method-args">(http_req)</span>
|
163
|
-
</a>
|
164
|
-
</div>
|
165
|
-
|
166
|
-
<div class="method-description">
|
167
|
-
</div>
|
168
|
-
</div>
|
169
|
-
|
170
|
-
<div id="method-M000011" class="method-detail">
|
171
|
-
<a name="M000011"></a>
|
172
|
-
|
173
|
-
<div class="method-heading">
|
174
|
-
<a href="spider_instance_spec_rb.src/M000011.html" target="Code" class="method-signature"
|
175
|
-
onclick="popupCode('spider_instance_spec_rb.src/M000011.html');return false;">
|
176
|
-
<span class="method-name">mock_redirect_http</span><span class="method-args">()</span>
|
177
|
-
</a>
|
178
|
-
</div>
|
179
|
-
|
180
|
-
<div class="method-description">
|
181
|
-
</div>
|
182
|
-
</div>
|
183
|
-
|
184
|
-
<div id="method-M000009" class="method-detail">
|
185
|
-
<a name="M000009"></a>
|
186
|
-
|
187
|
-
<div class="method-heading">
|
188
|
-
<a href="spider_instance_spec_rb.src/M000009.html" target="Code" class="method-signature"
|
189
|
-
onclick="popupCode('spider_instance_spec_rb.src/M000009.html');return false;">
|
190
|
-
<span class="method-name">mock_successful_http</span><span class="method-args">()</span>
|
191
|
-
</a>
|
192
|
-
</div>
|
193
|
-
|
194
|
-
<div class="method-description">
|
195
|
-
</div>
|
196
|
-
</div>
|
197
|
-
|
198
|
-
|
199
|
-
</div>
|
200
|
-
|
201
|
-
|
202
|
-
</div>
|
203
|
-
|
204
|
-
|
205
|
-
<div id="validator-badges">
|
206
|
-
<p><small><a href="http://validator.w3.org/check/referer">[Validate]</a></small></p>
|
207
|
-
</div>
|
208
|
-
|
209
|
-
</body>
|
210
|
-
</html>
|
@@ -1,21 +0,0 @@
|
|
1
|
-
<?xml version="1.0" encoding="iso-8859-1"?>
|
2
|
-
<!DOCTYPE html
|
3
|
-
PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
|
4
|
-
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
|
5
|
-
|
6
|
-
<html>
|
7
|
-
<head>
|
8
|
-
<title>mock_http (spec/spider/spider_instance_spec.rb)</title>
|
9
|
-
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
|
10
|
-
<link rel="stylesheet" href="../../../.././rdoc-style.css" type="text/css" media="screen" />
|
11
|
-
</head>
|
12
|
-
<body class="standalone-code">
|
13
|
-
<pre><span class="ruby-comment cmt"># File spec/spider/spider_instance_spec.rb, line 351</span>
|
14
|
-
<span class="ruby-keyword kw">def</span> <span class="ruby-identifier">mock_http</span>(<span class="ruby-identifier">http_req</span>)
|
15
|
-
<span class="ruby-identifier">http_obj</span> = <span class="ruby-identifier">mock</span>(<span class="ruby-identifier">:use_ssl=</span> =<span class="ruby-operator">></span> <span class="ruby-keyword kw">true</span>)
|
16
|
-
<span class="ruby-identifier">http_obj</span>.<span class="ruby-identifier">expects</span>(<span class="ruby-identifier">:start</span>).
|
17
|
-
<span class="ruby-identifier">yields</span>(<span class="ruby-identifier">mock</span>(<span class="ruby-identifier">:request</span> =<span class="ruby-operator">></span> <span class="ruby-identifier">http_req</span>)).<span class="ruby-identifier">returns</span>(<span class="ruby-identifier">http_req</span>)
|
18
|
-
<span class="ruby-constant">Net</span><span class="ruby-operator">::</span><span class="ruby-constant">HTTP</span>.<span class="ruby-identifier">expects</span>(<span class="ruby-identifier">:new</span>).<span class="ruby-identifier">returns</span>(<span class="ruby-identifier">http_obj</span>)
|
19
|
-
<span class="ruby-keyword kw">end</span></pre>
|
20
|
-
</body>
|
21
|
-
</html>
|
@@ -1,19 +0,0 @@
|
|
1
|
-
<?xml version="1.0" encoding="iso-8859-1"?>
|
2
|
-
<!DOCTYPE html
|
3
|
-
PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
|
4
|
-
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
|
5
|
-
|
6
|
-
<html>
|
7
|
-
<head>
|
8
|
-
<title>mock_successful_http (spec/spider/spider_instance_spec.rb)</title>
|
9
|
-
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
|
10
|
-
<link rel="stylesheet" href="../../../.././rdoc-style.css" type="text/css" media="screen" />
|
11
|
-
</head>
|
12
|
-
<body class="standalone-code">
|
13
|
-
<pre><span class="ruby-comment cmt"># File spec/spider/spider_instance_spec.rb, line 358</span>
|
14
|
-
<span class="ruby-keyword kw">def</span> <span class="ruby-identifier">mock_successful_http</span>
|
15
|
-
<span class="ruby-identifier">http_req</span> = <span class="ruby-identifier">stub</span>(<span class="ruby-identifier">:redirect?</span> =<span class="ruby-operator">></span> <span class="ruby-keyword kw">false</span>, <span class="ruby-identifier">:success?</span> =<span class="ruby-operator">></span> <span class="ruby-keyword kw">true</span>, <span class="ruby-identifier">:code</span> =<span class="ruby-operator">></span> <span class="ruby-value">200</span>, <span class="ruby-identifier">:body</span> =<span class="ruby-operator">></span> <span class="ruby-value str">'body'</span>)
|
16
|
-
<span class="ruby-identifier">mock_http</span>(<span class="ruby-identifier">http_req</span>)
|
17
|
-
<span class="ruby-keyword kw">end</span></pre>
|
18
|
-
</body>
|
19
|
-
</html>
|
@@ -1,19 +0,0 @@
|
|
1
|
-
<?xml version="1.0" encoding="iso-8859-1"?>
|
2
|
-
<!DOCTYPE html
|
3
|
-
PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
|
4
|
-
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
|
5
|
-
|
6
|
-
<html>
|
7
|
-
<head>
|
8
|
-
<title>mock_failed_http (spec/spider/spider_instance_spec.rb)</title>
|
9
|
-
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
|
10
|
-
<link rel="stylesheet" href="../../../.././rdoc-style.css" type="text/css" media="screen" />
|
11
|
-
</head>
|
12
|
-
<body class="standalone-code">
|
13
|
-
<pre><span class="ruby-comment cmt"># File spec/spider/spider_instance_spec.rb, line 363</span>
|
14
|
-
<span class="ruby-keyword kw">def</span> <span class="ruby-identifier">mock_failed_http</span>
|
15
|
-
<span class="ruby-identifier">http_req</span> = <span class="ruby-identifier">stub</span>(<span class="ruby-identifier">:redirect?</span> =<span class="ruby-operator">></span> <span class="ruby-keyword kw">false</span>, <span class="ruby-identifier">:success?</span> =<span class="ruby-operator">></span> <span class="ruby-keyword kw">false</span>, <span class="ruby-identifier">:code</span> =<span class="ruby-operator">></span> <span class="ruby-value">404</span>)
|
16
|
-
<span class="ruby-identifier">mock_http</span>(<span class="ruby-identifier">http_req</span>)
|
17
|
-
<span class="ruby-keyword kw">end</span></pre>
|
18
|
-
</body>
|
19
|
-
</html>
|
@@ -1,27 +0,0 @@
|
|
1
|
-
<?xml version="1.0" encoding="iso-8859-1"?>
|
2
|
-
<!DOCTYPE html
|
3
|
-
PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
|
4
|
-
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
|
5
|
-
|
6
|
-
<html>
|
7
|
-
<head>
|
8
|
-
<title>mock_redirect_http (spec/spider/spider_instance_spec.rb)</title>
|
9
|
-
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
|
10
|
-
<link rel="stylesheet" href="../../../.././rdoc-style.css" type="text/css" media="screen" />
|
11
|
-
</head>
|
12
|
-
<body class="standalone-code">
|
13
|
-
<pre><span class="ruby-comment cmt"># File spec/spider/spider_instance_spec.rb, line 368</span>
|
14
|
-
<span class="ruby-keyword kw">def</span> <span class="ruby-identifier">mock_redirect_http</span>
|
15
|
-
<span class="ruby-identifier">http_req</span> = <span class="ruby-identifier">stub</span>(<span class="ruby-identifier">:redirect?</span> =<span class="ruby-operator">></span> <span class="ruby-keyword kw">true</span>, <span class="ruby-identifier">:success?</span> =<span class="ruby-operator">></span> <span class="ruby-keyword kw">false</span>, <span class="ruby-identifier">:code</span> =<span class="ruby-operator">></span> <span class="ruby-value">404</span>)
|
16
|
-
<span class="ruby-identifier">http_req</span>.<span class="ruby-identifier">expects</span>(<span class="ruby-identifier">:[]</span>).<span class="ruby-identifier">with</span>(<span class="ruby-value str">'Location'</span>).<span class="ruby-identifier">returns</span>(<span class="ruby-value str">'http://example.com/'</span>)
|
17
|
-
<span class="ruby-identifier">http_req2</span> = <span class="ruby-identifier">stub</span>(<span class="ruby-identifier">:redirect?</span> =<span class="ruby-operator">></span> <span class="ruby-keyword kw">false</span>, <span class="ruby-identifier">:success?</span> =<span class="ruby-operator">></span> <span class="ruby-keyword kw">true</span>, <span class="ruby-identifier">:code</span> =<span class="ruby-operator">></span> <span class="ruby-value">200</span>)
|
18
|
-
<span class="ruby-identifier">http_obj</span> = <span class="ruby-identifier">mock</span>(<span class="ruby-identifier">:use_ssl=</span> =<span class="ruby-operator">></span> <span class="ruby-keyword kw">true</span>)
|
19
|
-
<span class="ruby-identifier">http_obj</span>.<span class="ruby-identifier">expects</span>(<span class="ruby-identifier">:start</span>).
|
20
|
-
<span class="ruby-identifier">yields</span>(<span class="ruby-identifier">mock</span>(<span class="ruby-identifier">:request</span> =<span class="ruby-operator">></span> <span class="ruby-identifier">http_req</span>)).<span class="ruby-identifier">returns</span>(<span class="ruby-identifier">http_req</span>)
|
21
|
-
<span class="ruby-identifier">http_obj2</span> = <span class="ruby-identifier">mock</span>(<span class="ruby-identifier">:use_ssl=</span> =<span class="ruby-operator">></span> <span class="ruby-keyword kw">true</span>)
|
22
|
-
<span class="ruby-identifier">http_obj2</span>.<span class="ruby-identifier">expects</span>(<span class="ruby-identifier">:start</span>).
|
23
|
-
<span class="ruby-identifier">yields</span>(<span class="ruby-identifier">mock</span>(<span class="ruby-identifier">:request</span> =<span class="ruby-operator">></span> <span class="ruby-identifier">http_req2</span>)).<span class="ruby-identifier">returns</span>(<span class="ruby-identifier">http_req2</span>)
|
24
|
-
<span class="ruby-constant">Net</span><span class="ruby-operator">::</span><span class="ruby-constant">HTTP</span>.<span class="ruby-identifier">expects</span>(<span class="ruby-identifier">:new</span>).<span class="ruby-identifier">times</span>(<span class="ruby-value">2</span>).<span class="ruby-identifier">returns</span>(<span class="ruby-identifier">http_obj</span>).<span class="ruby-identifier">then</span>.<span class="ruby-identifier">returns</span>(<span class="ruby-identifier">http_obj2</span>)
|
25
|
-
<span class="ruby-keyword kw">end</span></pre>
|
26
|
-
</body>
|
27
|
-
</html>
|
@@ -1,26 +0,0 @@
|
|
1
|
-
<?xml version="1.0" encoding="iso-8859-1"?>
|
2
|
-
<!DOCTYPE html
|
3
|
-
PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
|
4
|
-
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
|
5
|
-
|
6
|
-
<html>
|
7
|
-
<head>
|
8
|
-
<title>callback_arguments_on (spec/spider/spider_instance_spec.rb)</title>
|
9
|
-
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
|
10
|
-
<link rel="stylesheet" href="../../../.././rdoc-style.css" type="text/css" media="screen" />
|
11
|
-
</head>
|
12
|
-
<body class="standalone-code">
|
13
|
-
<pre><span class="ruby-comment cmt"># File spec/spider/spider_instance_spec.rb, line 381</span>
|
14
|
-
<span class="ruby-keyword kw">def</span> <span class="ruby-identifier">callback_arguments_on</span>(<span class="ruby-identifier">code</span>)
|
15
|
-
<span class="ruby-identifier">si</span> = <span class="ruby-constant">SpiderInstance</span>.<span class="ruby-identifier">new</span>(<span class="ruby-value str">'http://foo.com/'</span> =<span class="ruby-operator">></span> [<span class="ruby-value str">'http://example.com/'</span>])
|
16
|
-
<span class="ruby-identifier">si</span>.<span class="ruby-identifier">stubs</span>(<span class="ruby-identifier">:allowed?</span>).<span class="ruby-identifier">returns</span>(<span class="ruby-keyword kw">true</span>)
|
17
|
-
<span class="ruby-identifier">si</span>.<span class="ruby-identifier">stubs</span>(<span class="ruby-identifier">:generate_next_urls</span>).<span class="ruby-identifier">returns</span>([])
|
18
|
-
<span class="ruby-identifier">si</span>.<span class="ruby-identifier">on</span>(<span class="ruby-identifier">code</span>) <span class="ruby-keyword kw">do</span> <span class="ruby-operator">|</span><span class="ruby-identifier">a_url</span>, <span class="ruby-identifier">resp</span>, <span class="ruby-identifier">prior_url</span><span class="ruby-operator">|</span>
|
19
|
-
<span class="ruby-identifier">a_url</span>.<span class="ruby-identifier">should</span> <span class="ruby-operator">==</span> <span class="ruby-value str">'http://example.com/'</span>
|
20
|
-
<span class="ruby-identifier">resp</span>.<span class="ruby-identifier">should_not</span> <span class="ruby-identifier">be_nil</span>
|
21
|
-
<span class="ruby-identifier">prior_url</span>.<span class="ruby-identifier">should</span> <span class="ruby-operator">==</span> <span class="ruby-value str">'http://foo.com/'</span>
|
22
|
-
<span class="ruby-keyword kw">end</span>
|
23
|
-
<span class="ruby-identifier">si</span>.<span class="ruby-identifier">start!</span>
|
24
|
-
<span class="ruby-keyword kw">end</span></pre>
|
25
|
-
</body>
|
26
|
-
</html>
|
@@ -1,27 +0,0 @@
|
|
1
|
-
<?xml version="1.0" encoding="iso-8859-1"?>
|
2
|
-
<!DOCTYPE html
|
3
|
-
PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
|
4
|
-
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
|
5
|
-
|
6
|
-
<html>
|
7
|
-
<head>
|
8
|
-
<title>it_should_prevent_cycles_with (spec/spider/spider_instance_spec.rb)</title>
|
9
|
-
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
|
10
|
-
<link rel="stylesheet" href="../../../.././rdoc-style.css" type="text/css" media="screen" />
|
11
|
-
</head>
|
12
|
-
<body class="standalone-code">
|
13
|
-
<pre><span class="ruby-comment cmt"># File spec/spider/spider_instance_spec.rb, line 393</span>
|
14
|
-
<span class="ruby-keyword kw">def</span> <span class="ruby-identifier">it_should_prevent_cycles_with</span>(<span class="ruby-identifier">cacher</span>)
|
15
|
-
<span class="ruby-identifier">u</span> = <span class="ruby-value str">'http://localhost:8888/'</span>
|
16
|
-
<span class="ruby-identifier">u_p</span> = <span class="ruby-constant">URI</span>.<span class="ruby-identifier">parse</span>(<span class="ruby-identifier">u</span>)
|
17
|
-
<span class="ruby-identifier">u2</span> = <span class="ruby-value str">'http://localhost:8888/foo'</span>
|
18
|
-
<span class="ruby-identifier">u_p2</span> = <span class="ruby-constant">URI</span>.<span class="ruby-identifier">parse</span>(<span class="ruby-identifier">u2</span>)
|
19
|
-
|
20
|
-
<span class="ruby-identifier">with_web_server</span>(<span class="ruby-constant">LoopingServlet</span>) <span class="ruby-keyword kw">do</span>
|
21
|
-
<span class="ruby-identifier">si</span> = <span class="ruby-constant">SpiderInstance</span>.<span class="ruby-identifier">new</span>(<span class="ruby-keyword kw">nil</span> =<span class="ruby-operator">></span> [<span class="ruby-identifier">u</span>])
|
22
|
-
<span class="ruby-identifier">si</span>.<span class="ruby-identifier">check_already_seen_with</span> <span class="ruby-identifier">cacher</span>
|
23
|
-
<span class="ruby-identifier">si</span>.<span class="ruby-identifier">start!</span>
|
24
|
-
<span class="ruby-keyword kw">end</span>
|
25
|
-
<span class="ruby-keyword kw">end</span></pre>
|
26
|
-
</body>
|
27
|
-
</html>
|
@@ -1,127 +0,0 @@
|
|
1
|
-
<?xml version="1.0" encoding="iso-8859-1"?>
|
2
|
-
<!DOCTYPE html
|
3
|
-
PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
|
4
|
-
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
|
5
|
-
|
6
|
-
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
|
7
|
-
<head>
|
8
|
-
<title>File: spider_spec.rb</title>
|
9
|
-
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
|
10
|
-
<meta http-equiv="Content-Script-Type" content="text/javascript" />
|
11
|
-
<link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
|
12
|
-
<script type="text/javascript">
|
13
|
-
// <![CDATA[
|
14
|
-
|
15
|
-
function popupCode( url ) {
|
16
|
-
window.open(url, "Code", "resizable=yes,scrollbars=yes,toolbar=no,status=no,height=150,width=400")
|
17
|
-
}
|
18
|
-
|
19
|
-
function toggleCode( id ) {
|
20
|
-
if ( document.getElementById )
|
21
|
-
elem = document.getElementById( id );
|
22
|
-
else if ( document.all )
|
23
|
-
elem = eval( "document.all." + id );
|
24
|
-
else
|
25
|
-
return false;
|
26
|
-
|
27
|
-
elemStyle = elem.style;
|
28
|
-
|
29
|
-
if ( elemStyle.display != "block" ) {
|
30
|
-
elemStyle.display = "block"
|
31
|
-
} else {
|
32
|
-
elemStyle.display = "none"
|
33
|
-
}
|
34
|
-
|
35
|
-
return true;
|
36
|
-
}
|
37
|
-
|
38
|
-
// Make codeblocks hidden by default
|
39
|
-
document.writeln( "<style type=\"text/css\">div.method-source-code { display: none }</style>" )
|
40
|
-
|
41
|
-
// ]]>
|
42
|
-
</script>
|
43
|
-
|
44
|
-
</head>
|
45
|
-
<body>
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
<div id="fileHeader">
|
50
|
-
<h1>spider_spec.rb</h1>
|
51
|
-
<table class="header-table">
|
52
|
-
<tr class="top-aligned-row">
|
53
|
-
<td><strong>Path:</strong></td>
|
54
|
-
<td>spec/spider_spec.rb
|
55
|
-
</td>
|
56
|
-
</tr>
|
57
|
-
<tr class="top-aligned-row">
|
58
|
-
<td><strong>Last Update:</strong></td>
|
59
|
-
<td>Thu May 21 13:19:06 +0000 2009</td>
|
60
|
-
</tr>
|
61
|
-
</table>
|
62
|
-
</div>
|
63
|
-
<!-- banner header -->
|
64
|
-
|
65
|
-
<div id="bodyContent">
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
<div id="contextContent">
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
</div>
|
74
|
-
|
75
|
-
<div id="method-list">
|
76
|
-
<h3 class="section-bar">Methods</h3>
|
77
|
-
|
78
|
-
<div class="name-list">
|
79
|
-
<a href="#M000014">find_pages_with_static_server</a>
|
80
|
-
</div>
|
81
|
-
</div>
|
82
|
-
|
83
|
-
</div>
|
84
|
-
|
85
|
-
|
86
|
-
<!-- if includes -->
|
87
|
-
|
88
|
-
<div id="section">
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
<!-- if method_list -->
|
98
|
-
<div id="methods">
|
99
|
-
<h3 class="section-bar">Public Instance methods</h3>
|
100
|
-
|
101
|
-
<div id="method-M000014" class="method-detail">
|
102
|
-
<a name="M000014"></a>
|
103
|
-
|
104
|
-
<div class="method-heading">
|
105
|
-
<a href="spider_spec_rb.src/M000014.html" target="Code" class="method-signature"
|
106
|
-
onclick="popupCode('spider_spec_rb.src/M000014.html');return false;">
|
107
|
-
<span class="method-name">find_pages_with_static_server</span><span class="method-args">(&block)</span>
|
108
|
-
</a>
|
109
|
-
</div>
|
110
|
-
|
111
|
-
<div class="method-description">
|
112
|
-
</div>
|
113
|
-
</div>
|
114
|
-
|
115
|
-
|
116
|
-
</div>
|
117
|
-
|
118
|
-
|
119
|
-
</div>
|
120
|
-
|
121
|
-
|
122
|
-
<div id="validator-badges">
|
123
|
-
<p><small><a href="http://validator.w3.org/check/referer">[Validate]</a></small></p>
|
124
|
-
</div>
|
125
|
-
|
126
|
-
</body>
|
127
|
-
</html>
|