rest-tor 0.1.3 → 0.1.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +23 -1
- data/lib/rest-tor.rb +5 -1
- data/lib/rest_tor/instance.rb +5 -1
- data/lib/rest_tor/tor.rb +22 -16
- data/lib/rest_tor/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 2e8915b30c4321b069ab153e473351376943242e
|
4
|
+
data.tar.gz: d1091b5f18c5ab86d234786aa2111871539beda2
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 75a266319482a39a9ac9df64e140c395aeac2c6ca1fe476d5966974f7afeca8277979920ead6d9ee63a1473159a331ee6976e8deba5d53d272d70ce5529cecdf
|
7
|
+
data.tar.gz: 5736e68c8a496d9c354de9591f9b4bacdfc5c79a4fd9cd63a6e03a3481b55e2426ec13c05da37698aa17826f4b9ee87e7c432cf2120d127e40f294beedc69598
|
data/README.md
CHANGED
@@ -8,8 +8,17 @@ Tor.request will take an instance if has tor instanc, otherwise initialize an i
|
|
8
8
|
|
9
9
|
- Web Crawler
|
10
10
|
|
11
|
+
## Requirements
|
12
|
+
|
13
|
+
- Tor
|
14
|
+
|
15
|
+
Mac os: brew install tor
|
16
|
+
|
17
|
+
Linux: apt-get install tor
|
18
|
+
|
11
19
|
## Installation
|
12
20
|
|
21
|
+
|
13
22
|
Add this line to your application's Gemfile:
|
14
23
|
|
15
24
|
```ruby
|
@@ -44,6 +53,14 @@ Or install it yourself as:
|
|
44
53
|
Tor.request(url: '...', mode: :default) # Priority to use the highest number of successes, default is :default
|
45
54
|
Tor.request(url: '...', mode: :order) # in order
|
46
55
|
|
56
|
+
# Customize an mode
|
57
|
+
Tor::Dispatcher.register :custom do
|
58
|
+
Tor.store.all.sort_by do |(port, tor)|
|
59
|
+
tor.c_success
|
60
|
+
end
|
61
|
+
end
|
62
|
+
Tor.request(url: '...', mode: :custom)
|
63
|
+
|
47
64
|
# show all instances
|
48
65
|
Tor.store.all
|
49
66
|
=> {"9001"=>#<Tor::Instance:0x007fb7798d2498 @port="9001", @ip="64.113.32.29", @using=nil, @counter=#<Counter success: 5, fail: 1, succss_at: 2017-12-15 19:52:44 +0800, fail_at:2017-12-15 19:52:26 +0800>>}
|
@@ -52,6 +69,12 @@ Or install it yourself as:
|
|
52
69
|
# Stop instance
|
53
70
|
Tor.stop(port)
|
54
71
|
|
72
|
+
# proxy default is true
|
73
|
+
Tor.request(url: 'http://ip.plusor.cn',raw: false, proxy: false)
|
74
|
+
[2017-12-16 16:19:34.545] [95646-70129455200740] Started GET "http://ip.plusor.cn" (port:rest-client | mode:default)
|
75
|
+
[2017-12-16 16:19:34.785] [95646-70129455200740] Completed 200 OK in 0.2s (Size: 15 Bytes)
|
76
|
+
=> "172.104.89.163\n"
|
77
|
+
|
55
78
|
# Initialize tors
|
56
79
|
Tor.init
|
57
80
|
I, [2017-12-15T20:40:36.189835 #60422] INFO -- : Open tor with port:9001
|
@@ -97,7 +120,6 @@ Or install it yourself as:
|
|
97
120
|
- Optimization code
|
98
121
|
- Add config
|
99
122
|
- Configurable restart strategy
|
100
|
-
- Configurable dispatcher
|
101
123
|
|
102
124
|
## Development
|
103
125
|
|
data/lib/rest-tor.rb
CHANGED
@@ -7,6 +7,10 @@ Dir.glob(File.expand_path('../rest_tor/**/*.rb', __FILE__)).each { |file| requir
|
|
7
7
|
|
8
8
|
module Tor
|
9
9
|
def self.logger
|
10
|
-
Logger.new(STDOUT)
|
10
|
+
Logger.new(STDOUT).tap do |log|
|
11
|
+
log.formatter = proc do |severity, datetime, progname, msg|
|
12
|
+
"[#{datetime.strftime('%Y-%m-%d %H:%M:%S.%3N')}] [#{Process.pid}-#{Thread.current.object_id}] #{msg}\n"
|
13
|
+
end
|
14
|
+
end
|
11
15
|
end
|
12
16
|
end
|
data/lib/rest_tor/instance.rb
CHANGED
@@ -93,7 +93,11 @@ module Tor
|
|
93
93
|
end
|
94
94
|
|
95
95
|
def avaliable?
|
96
|
-
|
96
|
+
begin
|
97
|
+
pid && Process.getpgid( pid ) && true
|
98
|
+
rescue Errno::ESRCH
|
99
|
+
false
|
100
|
+
end
|
97
101
|
end
|
98
102
|
end
|
99
103
|
|
data/lib/rest_tor/tor.rb
CHANGED
@@ -23,7 +23,8 @@ module Tor extend self
|
|
23
23
|
@store ||= Redis::HashKey.new('tor', marshal: true).tap { |s| s.send(:extend, Builder) }
|
24
24
|
end
|
25
25
|
|
26
|
-
def hold_tor(mode: :default, &block)
|
26
|
+
def hold_tor(mode: :default, rest: false, &block)
|
27
|
+
return yield if rest
|
27
28
|
if tor=Thread.current[:tor]
|
28
29
|
port = tor.port
|
29
30
|
else
|
@@ -42,25 +43,28 @@ module Tor extend self
|
|
42
43
|
|
43
44
|
def request(options={}, &block)
|
44
45
|
url = options[:url]
|
45
|
-
|
46
|
+
mobile = options[:mobile]
|
47
|
+
proxy = options[:proxy] == false ? false : true
|
46
48
|
raw = options[:raw].nil? ? true : false
|
47
|
-
|
49
|
+
mode = options[:mode] || :default
|
50
|
+
method = options[:method] || :get
|
48
51
|
payload = options[:payload] || {}
|
49
52
|
timeout = options[:timeout] || 10
|
50
|
-
format = options[:format]
|
51
|
-
|
53
|
+
format = options[:format] || (raw ? :html : :string)
|
54
|
+
headers = options[:headers] || options[:header] || {}
|
52
55
|
default_header = { 'User-Agent' => mobile ? MOBILE_AGENT : USER_AGENT }
|
53
56
|
time, body = Time.now, nil
|
54
57
|
|
55
|
-
hold_tor(mode: mode) do |port, tor|
|
56
|
-
logger.info "Started #{method.to_s.upcase} #{url.inspect} (port:#{port} | mode:#{mode}"
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
58
|
+
hold_tor(mode: mode, rest: !proxy) do |port, tor|
|
59
|
+
logger.info "Started #{method.to_s.upcase} #{url.inspect} (port:#{port || 'rest-client'} | mode:#{mode})"
|
60
|
+
proxy = proxy && "socks5://127.0.0.1:#{port}" or nil
|
61
|
+
params = {
|
62
|
+
method: method,
|
63
|
+
url: url,
|
64
|
+
payload: payload,
|
65
|
+
proxy: proxy,
|
66
|
+
timeout: timeout,
|
67
|
+
headers: default_header.merge(headers)
|
64
68
|
}
|
65
69
|
|
66
70
|
begin
|
@@ -68,12 +72,14 @@ module Tor extend self
|
|
68
72
|
yield(res, req, headers ) if block_given?
|
69
73
|
res
|
70
74
|
end
|
71
|
-
tor
|
75
|
+
tor&.success!
|
72
76
|
body = response.body
|
73
77
|
logger.info "Completed #{response.try(:code)} OK in #{(Time.now-time).round(1)}s (Size: #{Utils.number_to_human_size(body.bytesize)})"
|
74
78
|
rescue Exception => e
|
75
79
|
tor.fail!(e)
|
76
|
-
|
80
|
+
if tor
|
81
|
+
logger.info "#{e.class}: #{e.message}, <Tor#(success: #{tor.counter.success}, fail: #{tor.counter.fail}, port: #{tor.port})>"
|
82
|
+
end
|
77
83
|
raise e
|
78
84
|
end
|
79
85
|
end
|
data/lib/rest_tor/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: rest-tor
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.4
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- plusor
|
8
8
|
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-12-
|
11
|
+
date: 2017-12-16 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rest-client
|