codez-tarantula 0.5.4 → 0.5.5
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/relevance/tarantula/crawler.rb +13 -6
- data/lib/relevance/tarantula/version.rb +1 -1
- metadata +4 -3
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 58b9109eec26602cd9fea4ba7108d8cb58ac967e
|
4
|
+
data.tar.gz: 6fee5dc4225b85b62b91cfef379b7433605fb781
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a75f3990631e0be2ad14b6788070a3d2ae95b82a83337a7e68536ff697edd300d5a05b51596d0605dd9102dd51089801486e1766f140ca5c72c070fd3c748416
|
7
|
+
data.tar.gz: f00117d584a9d30f865d1c78f61f09b746e575e1e75f59b82bf0b9f3e7b2db158121afb712a265e2b5fc9771b9eea52b51ca2d2aedbb10f66c579b0ecb037105
|
@@ -208,14 +208,21 @@ module Relevance
|
|
208
208
|
end
|
209
209
|
end
|
210
210
|
|
211
|
-
# append delete requests to the end of the queue, all others just before the first delete request
|
212
211
|
def append_to_queue(request)
|
213
|
-
|
214
|
-
|
215
|
-
|
216
|
-
|
212
|
+
@crawl_queue.insert(index_to_insert(request), request)
|
213
|
+
end
|
214
|
+
|
215
|
+
# append get requests before others, delete requests to the end of the queue,
|
216
|
+
# all others just before the first delete request
|
217
|
+
def index_to_insert(request)
|
218
|
+
case request.meth
|
219
|
+
when 'get'
|
220
|
+
last_get = @crawl_queue.rindex { |r| r.meth == 'get' } || -1
|
221
|
+
last_get + 1
|
222
|
+
when 'delete'
|
223
|
+
@crawl_queue.index {|r| r.meth == 'delete' && request.url.start_with?(r.url) } || -1
|
217
224
|
else
|
218
|
-
@crawl_queue
|
225
|
+
@crawl_queue.index {|r| r.meth == 'delete' } || -1
|
219
226
|
end
|
220
227
|
end
|
221
228
|
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: codez-tarantula
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.5.
|
4
|
+
version: 0.5.5
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Relevance, Inc.
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2015-
|
12
|
+
date: 2015-12-01 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: htmlentities
|
@@ -223,8 +223,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
223
223
|
version: '0'
|
224
224
|
requirements: []
|
225
225
|
rubyforge_project:
|
226
|
-
rubygems_version: 2.4.
|
226
|
+
rubygems_version: 2.4.8
|
227
227
|
signing_key:
|
228
228
|
specification_version: 4
|
229
229
|
summary: A big hairy fuzzy spider that crawls your site, wreaking havoc
|
230
230
|
test_files: []
|
231
|
+
has_rdoc:
|