spider 0.4.4 → 0.5.4

Sign up to get free protection for your applications and to get access to all the features.
Files changed (77) hide show
  1. checksums.yaml +7 -0
  2. data/AUTHORS +17 -0
  3. data/CHANGES +16 -0
  4. data/LICENSE +21 -0
  5. data/{README → README.md} +73 -44
  6. data/lib/spider.rb +12 -29
  7. data/lib/spider/included_in_file.rb +32 -0
  8. data/lib/spider/included_in_memcached.rb +1 -24
  9. data/lib/spider/included_in_redis.rb +31 -0
  10. data/lib/spider/next_urls_in_sqs.rb +6 -29
  11. data/lib/spider/robot_rules.rb +61 -57
  12. data/lib/spider/spider_instance.rb +16 -35
  13. data/spec/spider/included_in_redis_spec.rb +43 -0
  14. data/spider.gemspec +5 -3
  15. metadata +38 -125
  16. data/doc/classes/BeStaticServerPages.html +0 -197
  17. data/doc/classes/BeStaticServerPages.src/M000030.html +0 -19
  18. data/doc/classes/BeStaticServerPages.src/M000031.html +0 -19
  19. data/doc/classes/BeStaticServerPages.src/M000032.html +0 -18
  20. data/doc/classes/BeStaticServerPages.src/M000033.html +0 -18
  21. data/doc/classes/IncludedInMemcached.html +0 -199
  22. data/doc/classes/IncludedInMemcached.src/M000015.html +0 -18
  23. data/doc/classes/IncludedInMemcached.src/M000016.html +0 -18
  24. data/doc/classes/IncludedInMemcached.src/M000017.html +0 -18
  25. data/doc/classes/LoopingServlet.html +0 -137
  26. data/doc/classes/LoopingServlet.src/M000037.html +0 -23
  27. data/doc/classes/NextUrlsInSQS.html +0 -204
  28. data/doc/classes/NextUrlsInSQS.src/M000018.html +0 -19
  29. data/doc/classes/NextUrlsInSQS.src/M000019.html +0 -22
  30. data/doc/classes/NextUrlsInSQS.src/M000020.html +0 -19
  31. data/doc/classes/QueryServlet.html +0 -137
  32. data/doc/classes/QueryServlet.src/M000038.html +0 -19
  33. data/doc/classes/RobotRules.html +0 -175
  34. data/doc/classes/RobotRules.src/M000034.html +0 -19
  35. data/doc/classes/RobotRules.src/M000035.html +0 -67
  36. data/doc/classes/RobotRules.src/M000036.html +0 -24
  37. data/doc/classes/Spider.html +0 -170
  38. data/doc/classes/Spider.src/M000029.html +0 -21
  39. data/doc/classes/SpiderInstance.html +0 -345
  40. data/doc/classes/SpiderInstance.src/M000021.html +0 -18
  41. data/doc/classes/SpiderInstance.src/M000022.html +0 -22
  42. data/doc/classes/SpiderInstance.src/M000023.html +0 -22
  43. data/doc/classes/SpiderInstance.src/M000024.html +0 -24
  44. data/doc/classes/SpiderInstance.src/M000025.html +0 -18
  45. data/doc/classes/SpiderInstance.src/M000026.html +0 -18
  46. data/doc/classes/SpiderInstance.src/M000027.html +0 -18
  47. data/doc/classes/SpiderInstance.src/M000028.html +0 -18
  48. data/doc/created.rid +0 -1
  49. data/doc/files/README.html +0 -223
  50. data/doc/files/lib/spider/included_in_memcached_rb.html +0 -142
  51. data/doc/files/lib/spider/next_urls_in_sqs_rb.html +0 -144
  52. data/doc/files/lib/spider/robot_rules_rb.html +0 -114
  53. data/doc/files/lib/spider/spider_instance_rb.html +0 -117
  54. data/doc/files/lib/spider_rb.html +0 -254
  55. data/doc/files/spec/spec_helper_rb.html +0 -196
  56. data/doc/files/spec/spec_helper_rb.src/M000001.html +0 -20
  57. data/doc/files/spec/spec_helper_rb.src/M000002.html +0 -26
  58. data/doc/files/spec/spec_helper_rb.src/M000003.html +0 -24
  59. data/doc/files/spec/spec_helper_rb.src/M000004.html +0 -18
  60. data/doc/files/spec/spec_helper_rb.src/M000005.html +0 -23
  61. data/doc/files/spec/spider/included_in_memcached_spec_rb.html +0 -142
  62. data/doc/files/spec/spider/included_in_memcached_spec_rb.src/M000006.html +0 -19
  63. data/doc/files/spec/spider/included_in_memcached_spec_rb.src/M000007.html +0 -18
  64. data/doc/files/spec/spider/spider_instance_spec_rb.html +0 -210
  65. data/doc/files/spec/spider/spider_instance_spec_rb.src/M000008.html +0 -21
  66. data/doc/files/spec/spider/spider_instance_spec_rb.src/M000009.html +0 -19
  67. data/doc/files/spec/spider/spider_instance_spec_rb.src/M000010.html +0 -19
  68. data/doc/files/spec/spider/spider_instance_spec_rb.src/M000011.html +0 -27
  69. data/doc/files/spec/spider/spider_instance_spec_rb.src/M000012.html +0 -26
  70. data/doc/files/spec/spider/spider_instance_spec_rb.src/M000013.html +0 -27
  71. data/doc/files/spec/spider_spec_rb.html +0 -127
  72. data/doc/files/spec/spider_spec_rb.src/M000014.html +0 -23
  73. data/doc/fr_class_index.html +0 -34
  74. data/doc/fr_file_index.html +0 -35
  75. data/doc/fr_method_index.html +0 -64
  76. data/doc/index.html +0 -24
  77. data/doc/rdoc-style.css +0 -208
data/doc/classes/BeStaticServerPages.html DELETED
@@ -1,197 +0,0 @@
1
- <?xml version="1.0" encoding="iso-8859-1"?>
2
- <!DOCTYPE html
3
- PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
4
- "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
5
-
6
- <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
7
- <head>
8
- <title>Class: BeStaticServerPages</title>
9
- <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
10
- <meta http-equiv="Content-Script-Type" content="text/javascript" />
11
- <link rel="stylesheet" href=".././rdoc-style.css" type="text/css" media="screen" />
12
- <script type="text/javascript">
13
- // <![CDATA[
14
-
15
- function popupCode( url ) {
16
- window.open(url, "Code", "resizable=yes,scrollbars=yes,toolbar=no,status=no,height=150,width=400")
17
- }
18
-
19
- function toggleCode( id ) {
20
- if ( document.getElementById )
21
- elem = document.getElementById( id );
22
- else if ( document.all )
23
- elem = eval( "document.all." + id );
24
- else
25
- return false;
26
-
27
- elemStyle = elem.style;
28
-
29
- if ( elemStyle.display != "block" ) {
30
- elemStyle.display = "block"
31
- } else {
32
- elemStyle.display = "none"
33
- }
34
-
35
- return true;
36
- }
37
-
38
- // Make codeblocks hidden by default
39
- document.writeln( "<style type=\"text/css\">div.method-source-code { display: none }</style>" )
40
-
41
- // ]]>
42
- </script>
43
-
44
- </head>
45
- <body>
46
-
47
-
48
-
49
- <div id="classHeader">
50
- <table class="header-table">
51
- <tr class="top-aligned-row">
52
- <td><strong>Class</strong></td>
53
- <td class="class-name-in-header">BeStaticServerPages</td>
54
- </tr>
55
- <tr class="top-aligned-row">
56
- <td><strong>In:</strong></td>
57
- <td>
58
- <a href="../files/spec/spec_helper_rb.html">
59
- spec/spec_helper.rb
60
- </a>
61
- <br />
62
- </td>
63
- </tr>
64
-
65
- <tr class="top-aligned-row">
66
- <td><strong>Parent:</strong></td>
67
- <td>
68
- Object
69
- </td>
70
- </tr>
71
- </table>
72
- </div>
73
- <!-- banner header -->
74
-
75
- <div id="bodyContent">
76
-
77
-
78
-
79
- <div id="contextContent">
80
-
81
-
82
-
83
- </div>
84
-
85
- <div id="method-list">
86
- <h3 class="section-bar">Methods</h3>
87
-
88
- <div class="name-list">
89
- <a href="#M000033">description</a>&nbsp;&nbsp;
90
- <a href="#M000032">failure_message</a>&nbsp;&nbsp;
91
- <a href="#M000031">matches?</a>&nbsp;&nbsp;
92
- <a href="#M000030">new</a>&nbsp;&nbsp;
93
- </div>
94
- </div>
95
-
96
- </div>
97
-
98
-
99
- <!-- if includes -->
100
-
101
- <div id="section">
102
-
103
-
104
-
105
-
106
-
107
- <div id="attribute-list">
108
- <h3 class="section-bar">Attributes</h3>
109
-
110
- <div class="name-list">
111
- <table>
112
- <tr class="top-aligned-row context-row">
113
- <td class="context-item-name">actual</td>
114
- <td class="context-item-value">&nbsp;[RW]&nbsp;</td>
115
- <td class="context-item-desc"></td>
116
- </tr>
117
- </table>
118
- </div>
119
- </div>
120
-
121
-
122
-
123
- <!-- if method_list -->
124
- <div id="methods">
125
- <h3 class="section-bar">Public Class methods</h3>
126
-
127
- <div id="method-M000030" class="method-detail">
128
- <a name="M000030"></a>
129
-
130
- <div class="method-heading">
131
- <a href="BeStaticServerPages.src/M000030.html" target="Code" class="method-signature"
132
- onclick="popupCode('BeStaticServerPages.src/M000030.html');return false;">
133
- <span class="method-name">new</span><span class="method-args">()</span>
134
- </a>
135
- </div>
136
-
137
- <div class="method-description">
138
- </div>
139
- </div>
140
-
141
- <h3 class="section-bar">Public Instance methods</h3>
142
-
143
- <div id="method-M000033" class="method-detail">
144
- <a name="M000033"></a>
145
-
146
- <div class="method-heading">
147
- <a href="BeStaticServerPages.src/M000033.html" target="Code" class="method-signature"
148
- onclick="popupCode('BeStaticServerPages.src/M000033.html');return false;">
149
- <span class="method-name">description</span><span class="method-args">()</span>
150
- </a>
151
- </div>
152
-
153
- <div class="method-description">
154
- </div>
155
- </div>
156
-
157
- <div id="method-M000032" class="method-detail">
158
- <a name="M000032"></a>
159
-
160
- <div class="method-heading">
161
- <a href="BeStaticServerPages.src/M000032.html" target="Code" class="method-signature"
162
- onclick="popupCode('BeStaticServerPages.src/M000032.html');return false;">
163
- <span class="method-name">failure_message</span><span class="method-args">()</span>
164
- </a>
165
- </div>
166
-
167
- <div class="method-description">
168
- </div>
169
- </div>
170
-
171
- <div id="method-M000031" class="method-detail">
172
- <a name="M000031"></a>
173
-
174
- <div class="method-heading">
175
- <a href="BeStaticServerPages.src/M000031.html" target="Code" class="method-signature"
176
- onclick="popupCode('BeStaticServerPages.src/M000031.html');return false;">
177
- <span class="method-name">matches?</span><span class="method-args">(actual)</span>
178
- </a>
179
- </div>
180
-
181
- <div class="method-description">
182
- </div>
183
- </div>
184
-
185
-
186
- </div>
187
-
188
-
189
- </div>
190
-
191
-
192
- <div id="validator-badges">
193
- <p><small><a href="http://validator.w3.org/check/referer">[Validate]</a></small></p>
194
- </div>
195
-
196
- </body>
197
- </html>
data/doc/classes/BeStaticServerPages.src/M000030.html DELETED
@@ -1,19 +0,0 @@
1
- <?xml version="1.0" encoding="iso-8859-1"?>
2
- <!DOCTYPE html
3
- PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
4
- "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
5
-
6
- <html>
7
- <head>
8
- <title>new (BeStaticServerPages)</title>
9
- <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
10
- <link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
11
- </head>
12
- <body class="standalone-code">
13
- <pre><span class="ruby-comment cmt"># File spec/spec_helper.rb, line 14</span>
14
- <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">initialize</span>
15
- <span class="ruby-ivar">@pages</span> = [<span class="ruby-value str">'http://localhost:8888/'</span>, <span class="ruby-value str">'http://localhost:8888/foo'</span>]
16
- <span class="ruby-ivar">@actual</span> = <span class="ruby-keyword kw">nil</span>
17
- <span class="ruby-keyword kw">end</span></pre>
18
- </body>
19
- </html>
data/doc/classes/BeStaticServerPages.src/M000031.html DELETED
@@ -1,19 +0,0 @@
1
- <?xml version="1.0" encoding="iso-8859-1"?>
2
- <!DOCTYPE html
3
- PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
4
- "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
5
-
6
- <html>
7
- <head>
8
- <title>matches? (BeStaticServerPages)</title>
9
- <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
10
- <link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
11
- </head>
12
- <body class="standalone-code">
13
- <pre><span class="ruby-comment cmt"># File spec/spec_helper.rb, line 21</span>
14
- <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">matches?</span>(<span class="ruby-identifier">actual</span>)
15
- <span class="ruby-ivar">@actual</span> = <span class="ruby-identifier">actual</span>
16
- <span class="ruby-identifier">actual</span> <span class="ruby-operator">==</span> <span class="ruby-ivar">@pages</span>
17
- <span class="ruby-keyword kw">end</span></pre>
18
- </body>
19
- </html>
data/doc/classes/BeStaticServerPages.src/M000032.html DELETED
@@ -1,18 +0,0 @@
1
- <?xml version="1.0" encoding="iso-8859-1"?>
2
- <!DOCTYPE html
3
- PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
4
- "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
5
-
6
- <html>
7
- <head>
8
- <title>failure_message (BeStaticServerPages)</title>
9
- <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
10
- <link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
11
- </head>
12
- <body class="standalone-code">
13
- <pre><span class="ruby-comment cmt"># File spec/spec_helper.rb, line 26</span>
14
- <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">failure_message</span>
15
- <span class="ruby-node">&quot;expected #{@pages.inspect}, got #{@actual.inspect}&quot;</span>
16
- <span class="ruby-keyword kw">end</span></pre>
17
- </body>
18
- </html>
data/doc/classes/BeStaticServerPages.src/M000033.html DELETED
@@ -1,18 +0,0 @@
1
- <?xml version="1.0" encoding="iso-8859-1"?>
2
- <!DOCTYPE html
3
- PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
4
- "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
5
-
6
- <html>
7
- <head>
8
- <title>description (BeStaticServerPages)</title>
9
- <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
10
- <link rel="stylesheet" href="../.././rdoc-style.css" type="text/css" media="screen" />
11
- </head>
12
- <body class="standalone-code">
13
- <pre><span class="ruby-comment cmt"># File spec/spec_helper.rb, line 30</span>
14
- <span class="ruby-keyword kw">def</span> <span class="ruby-identifier">description</span>
15
- <span class="ruby-node">&quot;be the pages returned by the static server (#{@pages.inspect})&quot;</span>
16
- <span class="ruby-keyword kw">end</span></pre>
17
- </body>
18
- </html>
data/doc/classes/IncludedInMemcached.html DELETED
@@ -1,199 +0,0 @@
1
- <?xml version="1.0" encoding="iso-8859-1"?>
2
- <!DOCTYPE html
3
- PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN"
4
- "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
5
-
6
- <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
7
- <head>
8
- <title>Class: IncludedInMemcached</title>
9
- <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
10
- <meta http-equiv="Content-Script-Type" content="text/javascript" />
11
- <link rel="stylesheet" href=".././rdoc-style.css" type="text/css" media="screen" />
12
- <script type="text/javascript">
13
- // <![CDATA[
14
-
15
- function popupCode( url ) {
16
- window.open(url, "Code", "resizable=yes,scrollbars=yes,toolbar=no,status=no,height=150,width=400")
17
- }
18
-
19
- function toggleCode( id ) {
20
- if ( document.getElementById )
21
- elem = document.getElementById( id );
22
- else if ( document.all )
23
- elem = eval( "document.all." + id );
24
- else
25
- return false;
26
-
27
- elemStyle = elem.style;
28
-
29
- if ( elemStyle.display != "block" ) {
30
- elemStyle.display = "block"
31
- } else {
32
- elemStyle.display = "none"
33
- }
34
-
35
- return true;
36
- }
37
-
38
- // Make codeblocks hidden by default
39
- document.writeln( "<style type=\"text/css\">div.method-source-code { display: none }</style>" )
40
-
41
- // ]]>
42
- </script>
43
-
44
- </head>
45
- <body>
46
-
47
-
48
-
49
- <div id="classHeader">
50
- <table class="header-table">
51
- <tr class="top-aligned-row">
52
- <td><strong>Class</strong></td>
53
- <td class="class-name-in-header">IncludedInMemcached</td>
54
- </tr>
55
- <tr class="top-aligned-row">
56
- <td><strong>In:</strong></td>
57
- <td>
58
- <a href="../files/lib/spider/included_in_memcached_rb.html">
59
- lib/spider/included_in_memcached.rb
60
- </a>
61
- <br />
62
- </td>
63
- </tr>
64
-
65
- <tr class="top-aligned-row">
66
- <td><strong>Parent:</strong></td>
67
- <td>
68
- Object
69
- </td>
70
- </tr>
71
- </table>
72
- </div>
73
- <!-- banner header -->
74
-
75
- <div id="bodyContent">
76
-
77
-
78
-
79
- <div id="contextContent">
80
-
81
- <div id="description">
82
- <p>
83
- A specialized class using memcached to track items stored. It supports
84
- three operations: <a href="IncludedInMemcached.html#M000015">new</a>,
85
- &lt;&lt;, and <a href="IncludedInMemcached.html#M000017">include?</a> .
86
- Together these can be used to add items to the memcache, then determine
87
- whether the item has been added.
88
- </p>
89
- <p>
90
- To use it with <a href="Spider.html">Spider</a> use the
91
- check_already_seen_with method:
92
- </p>
93
- <pre>
94
- Spider.start_at('http://example.com/') do |s|
95
- s.check_already_seen_with IncludedInMemcached.new('localhost:11211')
96
- end
97
- </pre>
98
-
99
- </div>
100
-
101
-
102
- </div>
103
-
104
- <div id="method-list">
105
- <h3 class="section-bar">Methods</h3>
106
-
107
- <div class="name-list">
108
- <a href="#M000016">&lt;&lt;</a>&nbsp;&nbsp;
109
- <a href="#M000017">include?</a>&nbsp;&nbsp;
110
- <a href="#M000015">new</a>&nbsp;&nbsp;
111
- </div>
112
- </div>
113
-
114
- </div>
115
-
116
-
117
- <!-- if includes -->
118
-
119
- <div id="section">
120
-
121
-
122
-
123
-
124
-
125
-
126
-
127
-
128
- <!-- if method_list -->
129
- <div id="methods">
130
- <h3 class="section-bar">Public Class methods</h3>
131
-
132
- <div id="method-M000015" class="method-detail">
133
- <a name="M000015"></a>
134
-
135
- <div class="method-heading">
136
- <a href="IncludedInMemcached.src/M000015.html" target="Code" class="method-signature"
137
- onclick="popupCode('IncludedInMemcached.src/M000015.html');return false;">
138
- <span class="method-name">new</span><span class="method-args">(*a)</span>
139
- </a>
140
- </div>
141
-
142
- <div class="method-description">
143
- <p>
144
- Construct a <a href="IncludedInMemcached.html#M000015">new</a> <a
145
- href="IncludedInMemcached.html">IncludedInMemcached</a> instance. All
146
- arguments here are passed to MemCache (part of the memcache-client gem).
147
- </p>
148
- </div>
149
- </div>
150
-
151
- <h3 class="section-bar">Public Instance methods</h3>
152
-
153
- <div id="method-M000016" class="method-detail">
154
- <a name="M000016"></a>
155
-
156
- <div class="method-heading">
157
- <a href="IncludedInMemcached.src/M000016.html" target="Code" class="method-signature"
158
- onclick="popupCode('IncludedInMemcached.src/M000016.html');return false;">
159
- <span class="method-name">&lt;&lt;</span><span class="method-args">(v)</span>
160
- </a>
161
- </div>
162
-
163
- <div class="method-description">
164
- <p>
165
- Add an item to the memcache.
166
- </p>
167
- </div>
168
- </div>
169
-
170
- <div id="method-M000017" class="method-detail">
171
- <a name="M000017"></a>
172
-
173
- <div class="method-heading">
174
- <a href="IncludedInMemcached.src/M000017.html" target="Code" class="method-signature"
175
- onclick="popupCode('IncludedInMemcached.src/M000017.html');return false;">
176
- <span class="method-name">include?</span><span class="method-args">(v)</span>
177
- </a>
178
- </div>
179
-
180
- <div class="method-description">
181
- <p>
182
- True if the item is in the memcache.
183
- </p>
184
- </div>
185
- </div>
186
-
187
-
188
- </div>
189
-
190
-
191
- </div>
192
-
193
-
194
- <div id="validator-badges">
195
- <p><small><a href="http://validator.w3.org/check/referer">[Validate]</a></small></p>
196
- </div>
197
-
198
- </body>
199
- </html>