spider 0.4.4 → 0.5.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/AUTHORS +12 -0
- data/CHANGES +6 -0
- data/LICENSE +21 -0
- data/{README → README.md} +50 -43
- data/lib/spider.rb +12 -29
- data/lib/spider/included_in_memcached.rb +1 -24
- data/lib/spider/next_urls_in_sqs.rb +6 -29
- data/lib/spider/robot_rules.rb +61 -57
- data/lib/spider/spider_instance.rb +8 -31
- data/spider.gemspec +4 -2
- metadata +33 -124
- data/doc/classes/BeStaticServerPages.html +0 -197
- data/doc/classes/BeStaticServerPages.src/M000030.html +0 -19
- data/doc/classes/BeStaticServerPages.src/M000031.html +0 -19
- data/doc/classes/BeStaticServerPages.src/M000032.html +0 -18
- data/doc/classes/BeStaticServerPages.src/M000033.html +0 -18
- data/doc/classes/IncludedInMemcached.html +0 -199
- data/doc/classes/IncludedInMemcached.src/M000015.html +0 -18
- data/doc/classes/IncludedInMemcached.src/M000016.html +0 -18
- data/doc/classes/IncludedInMemcached.src/M000017.html +0 -18
- data/doc/classes/LoopingServlet.html +0 -137
- data/doc/classes/LoopingServlet.src/M000037.html +0 -23
- data/doc/classes/NextUrlsInSQS.html +0 -204
- data/doc/classes/NextUrlsInSQS.src/M000018.html +0 -19
- data/doc/classes/NextUrlsInSQS.src/M000019.html +0 -22
- data/doc/classes/NextUrlsInSQS.src/M000020.html +0 -19
- data/doc/classes/QueryServlet.html +0 -137
- data/doc/classes/QueryServlet.src/M000038.html +0 -19
- data/doc/classes/RobotRules.html +0 -175
- data/doc/classes/RobotRules.src/M000034.html +0 -19
- data/doc/classes/RobotRules.src/M000035.html +0 -67
- data/doc/classes/RobotRules.src/M000036.html +0 -24
- data/doc/classes/Spider.html +0 -170
- data/doc/classes/Spider.src/M000029.html +0 -21
- data/doc/classes/SpiderInstance.html +0 -345
- data/doc/classes/SpiderInstance.src/M000021.html +0 -18
- data/doc/classes/SpiderInstance.src/M000022.html +0 -22
- data/doc/classes/SpiderInstance.src/M000023.html +0 -22
- data/doc/classes/SpiderInstance.src/M000024.html +0 -24
- data/doc/classes/SpiderInstance.src/M000025.html +0 -18
- data/doc/classes/SpiderInstance.src/M000026.html +0 -18
- data/doc/classes/SpiderInstance.src/M000027.html +0 -18
- data/doc/classes/SpiderInstance.src/M000028.html +0 -18
- data/doc/created.rid +0 -1
- data/doc/files/README.html +0 -223
- data/doc/files/lib/spider/included_in_memcached_rb.html +0 -142
- data/doc/files/lib/spider/next_urls_in_sqs_rb.html +0 -144
- data/doc/files/lib/spider/robot_rules_rb.html +0 -114
- data/doc/files/lib/spider/spider_instance_rb.html +0 -117
- data/doc/files/lib/spider_rb.html +0 -254
- data/doc/files/spec/spec_helper_rb.html +0 -196
- data/doc/files/spec/spec_helper_rb.src/M000001.html +0 -20
- data/doc/files/spec/spec_helper_rb.src/M000002.html +0 -26
- data/doc/files/spec/spec_helper_rb.src/M000003.html +0 -24
- data/doc/files/spec/spec_helper_rb.src/M000004.html +0 -18
- data/doc/files/spec/spec_helper_rb.src/M000005.html +0 -23
- data/doc/files/spec/spider/included_in_memcached_spec_rb.html +0 -142
- data/doc/files/spec/spider/included_in_memcached_spec_rb.src/M000006.html +0 -19
- data/doc/files/spec/spider/included_in_memcached_spec_rb.src/M000007.html +0 -18
- data/doc/files/spec/spider/spider_instance_spec_rb.html +0 -210
- data/doc/files/spec/spider/spider_instance_spec_rb.src/M000008.html +0 -21
- data/doc/files/spec/spider/spider_instance_spec_rb.src/M000009.html +0 -19
- data/doc/files/spec/spider/spider_instance_spec_rb.src/M000010.html +0 -19
- data/doc/files/spec/spider/spider_instance_spec_rb.src/M000011.html +0 -27
- data/doc/files/spec/spider/spider_instance_spec_rb.src/M000012.html +0 -26
- data/doc/files/spec/spider/spider_instance_spec_rb.src/M000013.html +0 -27
- data/doc/files/spec/spider_spec_rb.html +0 -127
- data/doc/files/spec/spider_spec_rb.src/M000014.html +0 -23
- data/doc/fr_class_index.html +0 -34
- data/doc/fr_file_index.html +0 -35
- data/doc/fr_method_index.html +0 -64
- data/doc/index.html +0 -24
- data/doc/rdoc-style.css +0 -208
@@ -1,21 +0,0 @@
|
|
1
|
-
<?xml version="1.0" encoding="iso-8859-1"?>
|
2
|
-
<!DOCTYPE html
|
3
|
-
PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
|
4
|
-
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
|
5
|
-
|
6
|
-
<html>
|
7
|
-
<head>
|
8
|
-
<title>start_at (Spider)</title>
|
9
|
-
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
|
10
|
-
<link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
|
11
|
-
</head>
|
12
|
-
<body class="standalone-code">
|
13
|
-
<pre><span class="ruby-comment cmt"># File lib/spider.rb, line 54</span>
|
14
|
-
<span class="ruby-keyword kw">def</span> <span class="ruby-keyword kw">self</span>.<span class="ruby-identifier">start_at</span>(<span class="ruby-identifier">a_url</span>, <span class="ruby-operator">&</span><span class="ruby-identifier">block</span>)
|
15
|
-
<span class="ruby-identifier">rules</span> = <span class="ruby-constant">RobotRules</span>.<span class="ruby-identifier">new</span>(<span class="ruby-value str">'Ruby Spider 1.0'</span>)
|
16
|
-
<span class="ruby-identifier">a_spider</span> = <span class="ruby-constant">SpiderInstance</span>.<span class="ruby-identifier">new</span>({<span class="ruby-keyword kw">nil</span> =<span class="ruby-operator">></span> <span class="ruby-identifier">a_url</span>}, [], <span class="ruby-identifier">rules</span>, [])
|
17
|
-
<span class="ruby-identifier">block</span>.<span class="ruby-identifier">call</span>(<span class="ruby-identifier">a_spider</span>)
|
18
|
-
<span class="ruby-identifier">a_spider</span>.<span class="ruby-identifier">start!</span>
|
19
|
-
<span class="ruby-keyword kw">end</span></pre>
|
20
|
-
</body>
|
21
|
-
</html>
|
@@ -1,345 +0,0 @@
|
|
1
|
-
<?xml version="1.0" encoding="iso-8859-1"?>
|
2
|
-
<!DOCTYPE html
|
3
|
-
PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
|
4
|
-
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
|
5
|
-
|
6
|
-
<html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
|
7
|
-
<head>
|
8
|
-
<title>Class: SpiderInstance</title>
|
9
|
-
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
|
10
|
-
<meta http-equiv="Content-Script-Type" content="text/javascript" />
|
11
|
-
<link rel="stylesheet" href=".././rdoc-style.css" type="text/css" media="screen" />
|
12
|
-
<script type="text/javascript">
|
13
|
-
// <![CDATA[
|
14
|
-
|
15
|
-
function popupCode( url ) {
|
16
|
-
window.open(url, "Code", "resizable=yes,scrollbars=yes,toolbar=no,status=no,height=150,width=400")
|
17
|
-
}
|
18
|
-
|
19
|
-
function toggleCode( id ) {
|
20
|
-
if ( document.getElementById )
|
21
|
-
elem = document.getElementById( id );
|
22
|
-
else if ( document.all )
|
23
|
-
elem = eval( "document.all." + id );
|
24
|
-
else
|
25
|
-
return false;
|
26
|
-
|
27
|
-
elemStyle = elem.style;
|
28
|
-
|
29
|
-
if ( elemStyle.display != "block" ) {
|
30
|
-
elemStyle.display = "block"
|
31
|
-
} else {
|
32
|
-
elemStyle.display = "none"
|
33
|
-
}
|
34
|
-
|
35
|
-
return true;
|
36
|
-
}
|
37
|
-
|
38
|
-
// Make codeblocks hidden by default
|
39
|
-
document.writeln( "<style type=\"text/css\">div.method-source-code { display: none }</style>" )
|
40
|
-
|
41
|
-
// ]]>
|
42
|
-
</script>
|
43
|
-
|
44
|
-
</head>
|
45
|
-
<body>
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
<div id="classHeader">
|
50
|
-
<table class="header-table">
|
51
|
-
<tr class="top-aligned-row">
|
52
|
-
<td><strong>Class</strong></td>
|
53
|
-
<td class="class-name-in-header">SpiderInstance</td>
|
54
|
-
</tr>
|
55
|
-
<tr class="top-aligned-row">
|
56
|
-
<td><strong>In:</strong></td>
|
57
|
-
<td>
|
58
|
-
<a href="../files/lib/spider/spider_instance_rb.html">
|
59
|
-
lib/spider/spider_instance.rb
|
60
|
-
</a>
|
61
|
-
<br />
|
62
|
-
</td>
|
63
|
-
</tr>
|
64
|
-
|
65
|
-
<tr class="top-aligned-row">
|
66
|
-
<td><strong>Parent:</strong></td>
|
67
|
-
<td>
|
68
|
-
Object
|
69
|
-
</td>
|
70
|
-
</tr>
|
71
|
-
</table>
|
72
|
-
</div>
|
73
|
-
<!-- banner header -->
|
74
|
-
|
75
|
-
<div id="bodyContent">
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
<div id="contextContent">
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
</div>
|
84
|
-
|
85
|
-
<div id="method-list">
|
86
|
-
<h3 class="section-bar">Methods</h3>
|
87
|
-
|
88
|
-
<div class="name-list">
|
89
|
-
<a href="#M000021">add_url_check</a>
|
90
|
-
<a href="#M000022">check_already_seen_with</a>
|
91
|
-
<a href="#M000028">clear_headers</a>
|
92
|
-
<a href="#M000027">headers</a>
|
93
|
-
<a href="#M000024">on</a>
|
94
|
-
<a href="#M000025">setup</a>
|
95
|
-
<a href="#M000023">store_next_urls_with</a>
|
96
|
-
<a href="#M000026">teardown</a>
|
97
|
-
</div>
|
98
|
-
</div>
|
99
|
-
|
100
|
-
</div>
|
101
|
-
|
102
|
-
|
103
|
-
<!-- if includes -->
|
104
|
-
|
105
|
-
<div id="section">
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
114
|
-
<!-- if method_list -->
|
115
|
-
<div id="methods">
|
116
|
-
<h3 class="section-bar">Public Instance methods</h3>
|
117
|
-
|
118
|
-
<div id="method-M000021" class="method-detail">
|
119
|
-
<a name="M000021"></a>
|
120
|
-
|
121
|
-
<div class="method-heading">
|
122
|
-
<a href="SpiderInstance.src/M000021.html" target="Code" class="method-signature"
|
123
|
-
onclick="popupCode('SpiderInstance.src/M000021.html');return false;">
|
124
|
-
<span class="method-name">add_url_check</span><span class="method-args">(&block)</span>
|
125
|
-
</a>
|
126
|
-
</div>
|
127
|
-
|
128
|
-
<div class="method-description">
|
129
|
-
<p>
|
130
|
-
Add a predicate that determines whether to continue down this URL‘s
|
131
|
-
path. All predicates must be true in order for a URL to proceed.
|
132
|
-
</p>
|
133
|
-
<p>
|
134
|
-
Takes a block that takes a string and produces a boolean. For example, this
|
135
|
-
will ensure that the URL starts with ‘<a
|
136
|
-
href="http://mike-burns.com">mike-burns.com</a>’:
|
137
|
-
</p>
|
138
|
-
<pre>
|
139
|
-
add_url_check { |a_url| a_url =~ %r{^http://mike-burns.com.*}
|
140
|
-
</pre>
|
141
|
-
</div>
|
142
|
-
</div>
|
143
|
-
|
144
|
-
<div id="method-M000022" class="method-detail">
|
145
|
-
<a name="M000022"></a>
|
146
|
-
|
147
|
-
<div class="method-heading">
|
148
|
-
<a href="SpiderInstance.src/M000022.html" target="Code" class="method-signature"
|
149
|
-
onclick="popupCode('SpiderInstance.src/M000022.html');return false;">
|
150
|
-
<span class="method-name">check_already_seen_with</span><span class="method-args">(cacher)</span>
|
151
|
-
</a>
|
152
|
-
</div>
|
153
|
-
|
154
|
-
<div class="method-description">
|
155
|
-
<p>
|
156
|
-
The Web is a graph; to avoid cycles we store the nodes (URLs) already
|
157
|
-
visited. The Web is a really, really, really big graph; as such, this list
|
158
|
-
of visited nodes grows really, really, really big.
|
159
|
-
</p>
|
160
|
-
<p>
|
161
|
-
Change the object used to store these seen nodes with this. The default
|
162
|
-
object is an instance of Array. Available with <a
|
163
|
-
href="Spider.html">Spider</a> is a wrapper of memcached.
|
164
|
-
</p>
|
165
|
-
<p>
|
166
|
-
You can implement a custom class for this; any object passed to <a
|
167
|
-
href="SpiderInstance.html#M000022">check_already_seen_with</a> must
|
168
|
-
understand just << and included? .
|
169
|
-
</p>
|
170
|
-
<pre>
|
171
|
-
# default
|
172
|
-
check_already_seen_with Array.new
|
173
|
-
|
174
|
-
# memcached
|
175
|
-
require 'spider/included_in_memcached'
|
176
|
-
check_already_seen_with IncludedInMemcached.new('localhost:11211')
|
177
|
-
</pre>
|
178
|
-
</div>
|
179
|
-
</div>
|
180
|
-
|
181
|
-
<div id="method-M000028" class="method-detail">
|
182
|
-
<a name="M000028"></a>
|
183
|
-
|
184
|
-
<div class="method-heading">
|
185
|
-
<a href="SpiderInstance.src/M000028.html" target="Code" class="method-signature"
|
186
|
-
onclick="popupCode('SpiderInstance.src/M000028.html');return false;">
|
187
|
-
<span class="method-name">clear_headers</span><span class="method-args">()</span>
|
188
|
-
</a>
|
189
|
-
</div>
|
190
|
-
|
191
|
-
<div class="method-description">
|
192
|
-
<p>
|
193
|
-
Reset the <a href="SpiderInstance.html#M000027">headers</a> hash.
|
194
|
-
</p>
|
195
|
-
</div>
|
196
|
-
</div>
|
197
|
-
|
198
|
-
<div id="method-M000027" class="method-detail">
|
199
|
-
<a name="M000027"></a>
|
200
|
-
|
201
|
-
<div class="method-heading">
|
202
|
-
<a href="SpiderInstance.src/M000027.html" target="Code" class="method-signature"
|
203
|
-
onclick="popupCode('SpiderInstance.src/M000027.html');return false;">
|
204
|
-
<span class="method-name">headers</span><span class="method-args">()</span>
|
205
|
-
</a>
|
206
|
-
</div>
|
207
|
-
|
208
|
-
<div class="method-description">
|
209
|
-
<p>
|
210
|
-
Use like a hash:
|
211
|
-
</p>
|
212
|
-
<pre>
|
213
|
-
headers['Cookies'] = 'user_id=1;password=btrross3'
|
214
|
-
</pre>
|
215
|
-
</div>
|
216
|
-
</div>
|
217
|
-
|
218
|
-
<div id="method-M000024" class="method-detail">
|
219
|
-
<a name="M000024"></a>
|
220
|
-
|
221
|
-
<div class="method-heading">
|
222
|
-
<a href="SpiderInstance.src/M000024.html" target="Code" class="method-signature"
|
223
|
-
onclick="popupCode('SpiderInstance.src/M000024.html');return false;">
|
224
|
-
<span class="method-name">on</span><span class="method-args">(code, p = nil, &block)</span>
|
225
|
-
</a>
|
226
|
-
</div>
|
227
|
-
|
228
|
-
<div class="method-description">
|
229
|
-
<p>
|
230
|
-
Add a response handler. A response handler‘s trigger can be :every,
|
231
|
-
:success, :failure, or any HTTP status code. The handler itself can be
|
232
|
-
either a Proc or a block.
|
233
|
-
</p>
|
234
|
-
<p>
|
235
|
-
The arguments to the block are: the URL as a string, an instance of
|
236
|
-
Net::HTTPResponse, and the prior URL as a string.
|
237
|
-
</p>
|
238
|
-
<p>
|
239
|
-
For example:
|
240
|
-
</p>
|
241
|
-
<pre>
|
242
|
-
on 404 do |a_url, resp, prior_url|
|
243
|
-
puts "URL not found: #{a_url}"
|
244
|
-
end
|
245
|
-
|
246
|
-
on :success do |a_url, resp, prior_url|
|
247
|
-
puts a_url
|
248
|
-
puts resp.body
|
249
|
-
end
|
250
|
-
|
251
|
-
on :every do |a_url, resp, prior_url|
|
252
|
-
puts "Given this code: #{resp.code}"
|
253
|
-
end
|
254
|
-
</pre>
|
255
|
-
</div>
|
256
|
-
</div>
|
257
|
-
|
258
|
-
<div id="method-M000025" class="method-detail">
|
259
|
-
<a name="M000025"></a>
|
260
|
-
|
261
|
-
<div class="method-heading">
|
262
|
-
<a href="SpiderInstance.src/M000025.html" target="Code" class="method-signature"
|
263
|
-
onclick="popupCode('SpiderInstance.src/M000025.html');return false;">
|
264
|
-
<span class="method-name">setup</span><span class="method-args">(p = nil, &block)</span>
|
265
|
-
</a>
|
266
|
-
</div>
|
267
|
-
|
268
|
-
<div class="method-description">
|
269
|
-
<p>
|
270
|
-
Run before the HTTP request. Given the URL as a string.
|
271
|
-
</p>
|
272
|
-
<pre>
|
273
|
-
setup do |a_url|
|
274
|
-
headers['Cookies'] = 'user_id=1;admin=true'
|
275
|
-
end
|
276
|
-
</pre>
|
277
|
-
</div>
|
278
|
-
</div>
|
279
|
-
|
280
|
-
<div id="method-M000023" class="method-detail">
|
281
|
-
<a name="M000023"></a>
|
282
|
-
|
283
|
-
<div class="method-heading">
|
284
|
-
<a href="SpiderInstance.src/M000023.html" target="Code" class="method-signature"
|
285
|
-
onclick="popupCode('SpiderInstance.src/M000023.html');return false;">
|
286
|
-
<span class="method-name">store_next_urls_with</span><span class="method-args">(a_store)</span>
|
287
|
-
</a>
|
288
|
-
</div>
|
289
|
-
|
290
|
-
<div class="method-description">
|
291
|
-
<p>
|
292
|
-
The Web is a really, really, really big graph; as such, this list of nodes
|
293
|
-
to visit grows really, really, really big.
|
294
|
-
</p>
|
295
|
-
<p>
|
296
|
-
Change the object used to store nodes we have yet to walk. The default
|
297
|
-
object is an instance of Array. Available with <a
|
298
|
-
href="Spider.html">Spider</a> is a wrapper of AmazonSQS.
|
299
|
-
</p>
|
300
|
-
<p>
|
301
|
-
You can implement a custom class for this; any object passed to <a
|
302
|
-
href="SpiderInstance.html#M000022">check_already_seen_with</a> must
|
303
|
-
understand just push and pop .
|
304
|
-
</p>
|
305
|
-
<pre>
|
306
|
-
# default
|
307
|
-
store_next_urls_with Array.new
|
308
|
-
|
309
|
-
# AmazonSQS
|
310
|
-
require 'spider/next_urls_in_sqs'
|
311
|
-
store_next_urls_with NextUrlsInSQS.new(AWS_ACCESS_KEY, AWS_SECRET_ACCESS_KEY, queue_name)
|
312
|
-
</pre>
|
313
|
-
</div>
|
314
|
-
</div>
|
315
|
-
|
316
|
-
<div id="method-M000026" class="method-detail">
|
317
|
-
<a name="M000026"></a>
|
318
|
-
|
319
|
-
<div class="method-heading">
|
320
|
-
<a href="SpiderInstance.src/M000026.html" target="Code" class="method-signature"
|
321
|
-
onclick="popupCode('SpiderInstance.src/M000026.html');return false;">
|
322
|
-
<span class="method-name">teardown</span><span class="method-args">(p = nil, &block)</span>
|
323
|
-
</a>
|
324
|
-
</div>
|
325
|
-
|
326
|
-
<div class="method-description">
|
327
|
-
<p>
|
328
|
-
Run last, once for each page. Given the URL as a string.
|
329
|
-
</p>
|
330
|
-
</div>
|
331
|
-
</div>
|
332
|
-
|
333
|
-
|
334
|
-
</div>
|
335
|
-
|
336
|
-
|
337
|
-
</div>
|
338
|
-
|
339
|
-
|
340
|
-
<div id="validator-badges">
|
341
|
-
<p><small><a href="http://validator.w3.org/check/referer">[Validate]</a></small></p>
|
342
|
-
</div>
|
343
|
-
|
344
|
-
</body>
|
345
|
-
</html>
|
@@ -1,18 +0,0 @@
|
|
1
|
-
<?xml version="1.0" encoding="iso-8859-1"?>
|
2
|
-
<!DOCTYPE html
|
3
|
-
PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
|
4
|
-
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
|
5
|
-
|
6
|
-
<html>
|
7
|
-
<head>
|
8
|
-
<title>add_url_check (SpiderInstance)</title>
|
9
|
-
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
|
10
|
-
<link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
|
11
|
-
</head>
|
12
|
-
<body class="standalone-code">
|
13
|
-
<pre><span class="ruby-comment cmt"># File lib/spider/spider_instance.rb, line 70</span>
|
14
|
-
<span class="ruby-keyword kw">def</span> <span class="ruby-identifier">add_url_check</span>(<span class="ruby-operator">&</span><span class="ruby-identifier">block</span>)
|
15
|
-
<span class="ruby-ivar">@url_checks</span> <span class="ruby-operator"><<</span> <span class="ruby-identifier">block</span>
|
16
|
-
<span class="ruby-keyword kw">end</span></pre>
|
17
|
-
</body>
|
18
|
-
</html>
|
@@ -1,22 +0,0 @@
|
|
1
|
-
<?xml version="1.0" encoding="iso-8859-1"?>
|
2
|
-
<!DOCTYPE html
|
3
|
-
PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
|
4
|
-
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
|
5
|
-
|
6
|
-
<html>
|
7
|
-
<head>
|
8
|
-
<title>check_already_seen_with (SpiderInstance)</title>
|
9
|
-
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
|
10
|
-
<link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
|
11
|
-
</head>
|
12
|
-
<body class="standalone-code">
|
13
|
-
<pre><span class="ruby-comment cmt"># File lib/spider/spider_instance.rb, line 91</span>
|
14
|
-
<span class="ruby-keyword kw">def</span> <span class="ruby-identifier">check_already_seen_with</span>(<span class="ruby-identifier">cacher</span>)
|
15
|
-
<span class="ruby-keyword kw">if</span> <span class="ruby-identifier">cacher</span>.<span class="ruby-identifier">respond_to?</span>(<span class="ruby-identifier">:<<</span>) <span class="ruby-operator">&&</span> <span class="ruby-identifier">cacher</span>.<span class="ruby-identifier">respond_to?</span>(<span class="ruby-identifier">:include?</span>)
|
16
|
-
<span class="ruby-ivar">@seen</span> = <span class="ruby-identifier">cacher</span>
|
17
|
-
<span class="ruby-keyword kw">else</span>
|
18
|
-
<span class="ruby-identifier">raise</span> <span class="ruby-constant">ArgumentError</span>, <span class="ruby-value str">'expected something that responds to << and included?'</span>
|
19
|
-
<span class="ruby-keyword kw">end</span>
|
20
|
-
<span class="ruby-keyword kw">end</span></pre>
|
21
|
-
</body>
|
22
|
-
</html>
|
@@ -1,22 +0,0 @@
|
|
1
|
-
<?xml version="1.0" encoding="iso-8859-1"?>
|
2
|
-
<!DOCTYPE html
|
3
|
-
PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
|
4
|
-
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
|
5
|
-
|
6
|
-
<html>
|
7
|
-
<head>
|
8
|
-
<title>store_next_urls_with (SpiderInstance)</title>
|
9
|
-
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
|
10
|
-
<link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
|
11
|
-
</head>
|
12
|
-
<body class="standalone-code">
|
13
|
-
<pre><span class="ruby-comment cmt"># File lib/spider/spider_instance.rb, line 115</span>
|
14
|
-
<span class="ruby-keyword kw">def</span> <span class="ruby-identifier">store_next_urls_with</span>(<span class="ruby-identifier">a_store</span>)
|
15
|
-
<span class="ruby-identifier">tmp_next_urls</span> = <span class="ruby-ivar">@next_urls</span>
|
16
|
-
<span class="ruby-ivar">@next_urls</span> = <span class="ruby-identifier">a_store</span>
|
17
|
-
<span class="ruby-identifier">tmp_next_urls</span>.<span class="ruby-identifier">each</span> <span class="ruby-keyword kw">do</span> <span class="ruby-operator">|</span><span class="ruby-identifier">a_url_hash</span><span class="ruby-operator">|</span>
|
18
|
-
<span class="ruby-ivar">@next_urls</span>.<span class="ruby-identifier">push</span> <span class="ruby-identifier">a_url_hash</span>
|
19
|
-
<span class="ruby-keyword kw">end</span>
|
20
|
-
<span class="ruby-keyword kw">end</span></pre>
|
21
|
-
</body>
|
22
|
-
</html>
|
@@ -1,24 +0,0 @@
|
|
1
|
-
<?xml version="1.0" encoding="iso-8859-1"?>
|
2
|
-
<!DOCTYPE html
|
3
|
-
PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
|
4
|
-
"http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
|
5
|
-
|
6
|
-
<html>
|
7
|
-
<head>
|
8
|
-
<title>on (SpiderInstance)</title>
|
9
|
-
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
|
10
|
-
<link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
|
11
|
-
</head>
|
12
|
-
<body class="standalone-code">
|
13
|
-
<pre><span class="ruby-comment cmt"># File lib/spider/spider_instance.rb, line 145</span>
|
14
|
-
<span class="ruby-keyword kw">def</span> <span class="ruby-identifier">on</span>(<span class="ruby-identifier">code</span>, <span class="ruby-identifier">p</span> = <span class="ruby-keyword kw">nil</span>, <span class="ruby-operator">&</span><span class="ruby-identifier">block</span>)
|
15
|
-
<span class="ruby-identifier">f</span> = <span class="ruby-identifier">p</span> <span class="ruby-value">? </span><span class="ruby-identifier">p</span> <span class="ruby-operator">:</span> <span class="ruby-identifier">block</span>
|
16
|
-
<span class="ruby-keyword kw">case</span> <span class="ruby-identifier">code</span>
|
17
|
-
<span class="ruby-keyword kw">when</span> <span class="ruby-constant">Fixnum</span>
|
18
|
-
<span class="ruby-ivar">@callbacks</span>[<span class="ruby-identifier">code</span>] = <span class="ruby-identifier">f</span>
|
19
|
-
<span class="ruby-keyword kw">else</span>
|
20
|
-
<span class="ruby-ivar">@callbacks</span>[<span class="ruby-identifier">code</span>.<span class="ruby-identifier">to_sym</span>] = <span class="ruby-identifier">f</span>
|
21
|
-
<span class="ruby-keyword kw">end</span>
|
22
|
-
<span class="ruby-keyword kw">end</span></pre>
|
23
|
-
</body>
|
24
|
-
</html>
|