get_proxy_list 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +8 -8
- data/lib/debug.rb +1 -1
- data/lib/get_proxy_list/from_proxy_cn.rb +1 -1
- data/lib/get_proxy_list/version.rb +1 -1
- data/lib/get_proxy_list.rb +33 -0
- metadata +1 -1
checksums.yaml
CHANGED
|
@@ -1,15 +1,15 @@
|
|
|
1
1
|
---
|
|
2
2
|
!binary "U0hBMQ==":
|
|
3
3
|
metadata.gz: !binary |-
|
|
4
|
-
|
|
4
|
+
NWMwNDdjYjQwZTMwZTE2MTg1ZmIxNGI2NTk4NDlhMDdmNGM5MDEyMg==
|
|
5
5
|
data.tar.gz: !binary |-
|
|
6
|
-
|
|
6
|
+
ZGNhOTBjMDAwNzIzZTZjNGY1YWY5MTU4MDdmNWMxZWQzZGM3Zjc1Mg==
|
|
7
7
|
!binary "U0hBNTEy":
|
|
8
8
|
metadata.gz: !binary |-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
9
|
+
OGJmMjIwODcyNzcwZWM5Mjk4M2UxZmYxMGMzNWRmZmYyNjFhYjMxZjU2NmU0
|
|
10
|
+
MzI3NzVjMTY3N2IxOWNkMTU1NzhiODk4MDM0MWExMDlkMmM5MTIwMGU2NGM4
|
|
11
|
+
MTNhNzA4Mzk5Y2UzNzlmZTg3OTFiNjEyYjQyZWQyZTkyMzE4ZmI=
|
|
12
12
|
data.tar.gz: !binary |-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
13
|
+
MTA4MDllYTEzYjE3NzAyNjAyMjA5Yzk4NzEwOWU2MjgzZjgxNmRlNTc0YTU1
|
|
14
|
+
Y2U5Y2E2YjcxYjBhNDcwNzBiZWM5OGU5NGQ2NGE5Y2QyZmNiMWZjZWIyYjk2
|
|
15
|
+
MjAyOTY0OTM5NmRmZGIwMTk4NzlhYzkwYThmYmVmZTMxNGFlZTU=
|
data/lib/debug.rb
CHANGED
|
@@ -6,7 +6,7 @@ require File.expand_path("../get_proxy_list", __FILE__)
|
|
|
6
6
|
class Debug
|
|
7
7
|
# To change this template use File | Settings | File Templates.
|
|
8
8
|
include GetProxyList
|
|
9
|
-
list= GetProxyList.get_list(
|
|
9
|
+
list= GetProxyList.get_list(0,4)
|
|
10
10
|
p "Size: #{list.size}"
|
|
11
11
|
p "list: #{list}"
|
|
12
12
|
end
|
|
@@ -16,7 +16,7 @@ module GetProxyList
|
|
|
16
16
|
def _getproxylist(url)
|
|
17
17
|
begin
|
|
18
18
|
hash_port = {"c"=>"1","a"=>"2","z"=>"3","m"=>"4","b"=>"5","w"=>"6","i"=>"7","x"=>"8","l"=>"9","f"=>"0"}
|
|
19
|
-
html = open(url
|
|
19
|
+
html = open(url).read
|
|
20
20
|
html.force_encoding("gbk")
|
|
21
21
|
html.encode!("utf-8", :undef => :replace, :replace => "", :invalid => :replace)
|
|
22
22
|
doc=Nokogiri::HTML.parse html
|
data/lib/get_proxy_list.rb
CHANGED
|
@@ -10,4 +10,37 @@ module GetProxyList
|
|
|
10
10
|
proxylist = base.get_proxy_in_time_limit(limit,proxylist)
|
|
11
11
|
return proxylist
|
|
12
12
|
end
|
|
13
|
+
|
|
14
|
+
def self.get_proxy_in_time_limt(limit,proxylist)
|
|
15
|
+
limit_list = []
|
|
16
|
+
proxylist.each do |proxy|
|
|
17
|
+
url = proxy
|
|
18
|
+
time_start = Time.now.to_i
|
|
19
|
+
begin
|
|
20
|
+
timeout(limit+1) do
|
|
21
|
+
doc = Nokogiri::HTML(open("http://www.baidu.com",:proxy=> url))
|
|
22
|
+
x = doc.css("em")
|
|
23
|
+
end
|
|
24
|
+
time_end = Time.now.to_i
|
|
25
|
+
time_use = time_end - time_start
|
|
26
|
+
p "url use_time:#{time_use}"
|
|
27
|
+
rescue Exception =>e
|
|
28
|
+
case e
|
|
29
|
+
when Errno::ETIMEDOUT
|
|
30
|
+
p "Use #{url} timeout"
|
|
31
|
+
when Timeout::Error
|
|
32
|
+
p "Use #{url} timeout"
|
|
33
|
+
when Errno::ECONNREFUSED
|
|
34
|
+
p "Use #{url} Error connection"
|
|
35
|
+
else
|
|
36
|
+
p "Use #{url} Error:#{e.to_s}"
|
|
37
|
+
end
|
|
38
|
+
time_use = -1
|
|
39
|
+
end
|
|
40
|
+
if(time_use > 0 &&time_use < limit)
|
|
41
|
+
limit_list << url
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
limit_list
|
|
45
|
+
end
|
|
13
46
|
end
|