arachnid2 0.3.7 → 0.3.8
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile.lock +14 -15
- data/lib/arachnid2/version.rb +1 -1
- data/lib/arachnid2/watir.rb +4 -5
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: fb4b9682882053c38b4169d9b5644cec08e431afa1ae3c22323abe8091124b92
|
|
4
|
+
data.tar.gz: 79a08fec495dc55a95d41b65dad4c59cb08061ef13adc8d18bbcc374d46eb3f1
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: e60c38afd2429656b477b017edf661124b4f40c8598dbeed106d8eec94bc77fbd2cd5f974133bd0118ee447b736869c837b20ce396a22e2ebd0661163ec9a493
|
|
7
|
+
data.tar.gz: 6cd1317b0de54db95772f6e30daafb95484fda231a4ef93c9b2e16a0fe6de8e2728aec9a293d81b6d95c5da8ce84b2c51184f2bf74d268c16c1c9cbe5999e421
|
data/Gemfile.lock
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
PATH
|
|
2
2
|
remote: .
|
|
3
3
|
specs:
|
|
4
|
-
arachnid2 (0.3.
|
|
4
|
+
arachnid2 (0.3.8)
|
|
5
5
|
addressable
|
|
6
6
|
adomain
|
|
7
7
|
bloomfilter-rb
|
|
@@ -21,23 +21,22 @@ GEM
|
|
|
21
21
|
logger
|
|
22
22
|
bloomfilter-rb (2.1.1)
|
|
23
23
|
redis
|
|
24
|
-
childprocess (
|
|
25
|
-
rake (< 13.0)
|
|
24
|
+
childprocess (3.0.0)
|
|
26
25
|
diff-lcs (1.3)
|
|
27
26
|
ethon (0.12.0)
|
|
28
27
|
ffi (>= 1.3.0)
|
|
29
28
|
facets (3.1.0)
|
|
30
|
-
ffi (1.
|
|
31
|
-
json (2.
|
|
32
|
-
logger (1.4.
|
|
29
|
+
ffi (1.12.2)
|
|
30
|
+
json (2.3.0)
|
|
31
|
+
logger (1.4.2)
|
|
33
32
|
mini_portile2 (2.4.0)
|
|
34
|
-
nokogiri (1.10.
|
|
33
|
+
nokogiri (1.10.8)
|
|
35
34
|
mini_portile2 (~> 2.4.0)
|
|
36
35
|
os (1.0.1)
|
|
37
36
|
psych (3.1.0)
|
|
38
|
-
public_suffix (4.0.
|
|
37
|
+
public_suffix (4.0.3)
|
|
39
38
|
rake (10.5.0)
|
|
40
|
-
redis (4.1.
|
|
39
|
+
redis (4.1.3)
|
|
41
40
|
regexp_parser (1.6.0)
|
|
42
41
|
rspec (3.8.0)
|
|
43
42
|
rspec-core (~> 3.8.0)
|
|
@@ -52,10 +51,10 @@ GEM
|
|
|
52
51
|
diff-lcs (>= 1.2.0, < 2.0)
|
|
53
52
|
rspec-support (~> 3.8.0)
|
|
54
53
|
rspec-support (3.8.0)
|
|
55
|
-
rubyzip (
|
|
56
|
-
selenium-webdriver (3.142.
|
|
57
|
-
childprocess (>= 0.5, <
|
|
58
|
-
rubyzip (
|
|
54
|
+
rubyzip (2.2.0)
|
|
55
|
+
selenium-webdriver (3.142.7)
|
|
56
|
+
childprocess (>= 0.5, < 4.0)
|
|
57
|
+
rubyzip (>= 1.2.2)
|
|
59
58
|
typhoeus (1.3.1)
|
|
60
59
|
ethon (>= 0.9.0)
|
|
61
60
|
watir (6.16.5)
|
|
@@ -67,9 +66,9 @@ GEM
|
|
|
67
66
|
os
|
|
68
67
|
psych
|
|
69
68
|
selenium-webdriver (>= 3.4.0)
|
|
70
|
-
webdrivers (4.
|
|
69
|
+
webdrivers (4.2.0)
|
|
71
70
|
nokogiri (~> 1.6)
|
|
72
|
-
rubyzip (
|
|
71
|
+
rubyzip (>= 1.3.0)
|
|
73
72
|
selenium-webdriver (>= 3.0, < 4.0)
|
|
74
73
|
|
|
75
74
|
PLATFORMS
|
data/lib/arachnid2/version.rb
CHANGED
data/lib/arachnid2/watir.rb
CHANGED
|
@@ -13,9 +13,9 @@ class Arachnid2
|
|
|
13
13
|
def crawl(opts)
|
|
14
14
|
preflight(opts)
|
|
15
15
|
watir_preflight
|
|
16
|
+
@already_retried = false
|
|
16
17
|
|
|
17
18
|
until @global_queue.empty?
|
|
18
|
-
@already_retried = false
|
|
19
19
|
q = @global_queue.shift
|
|
20
20
|
links = nil
|
|
21
21
|
|
|
@@ -44,13 +44,10 @@ class Arachnid2
|
|
|
44
44
|
yield browser
|
|
45
45
|
|
|
46
46
|
vacuum(links, browser.url)
|
|
47
|
+
rescue Selenium::WebDriver::Error::NoSuchWindowError, Net::ReadTimeout => e
|
|
47
48
|
rescue => e
|
|
48
|
-
next if e.class == Net::ReadTimeout
|
|
49
|
-
|
|
50
49
|
raise e if raise_before_retry?(e.class)
|
|
51
|
-
|
|
52
50
|
reset_for_retry
|
|
53
|
-
retry
|
|
54
51
|
end
|
|
55
52
|
|
|
56
53
|
end # until @global_queue.empty?
|
|
@@ -69,6 +66,8 @@ class Arachnid2
|
|
|
69
66
|
def reset_for_retry
|
|
70
67
|
@browser.close if @browser rescue nil
|
|
71
68
|
@headless.destroy if @headless rescue nil
|
|
69
|
+
@driver.quit if @headless rescue nil
|
|
70
|
+
@driver = nil
|
|
72
71
|
@browser = nil
|
|
73
72
|
@already_retried = true
|
|
74
73
|
end
|