rest-tor 0.1.1 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +52 -3
- data/lib/rest-tor.rb +2 -1
- data/lib/rest_tor/instance.rb +7 -0
- data/lib/rest_tor/tor.rb +25 -14
- data/lib/rest_tor/version.rb +1 -1
- metadata +1 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7633b710960f440ecc94cf9d03ca0373d39daa10
|
4
|
+
data.tar.gz: ee7c1d41043e5a7f53c00b9fc2cc51586f6b1537
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 59bc545c50834cf402202aea5c91a93fdc72265ea19de9a380174b9df3453ac83cfdafc628931793b623e4b4d8685a4cdaeac2ec3f1d68a493d2cedf284192c4
|
7
|
+
data.tar.gz: cac174ee711d9c92b0ec296734465ee2a1eac8f41048c20496aa9a420ba00044205e6b215a44d3d057c461231df4a72bfb1e9b0a9852aab0387f77ecf89588d8
|
data/README.md
CHANGED
@@ -1,8 +1,12 @@
|
|
1
|
-
# Rest
|
1
|
+
# Rest Tor
|
2
2
|
|
3
|
-
|
3
|
+
Tor for ruby. Support for multiple processes. Each request will take an tor instance.
|
4
4
|
|
5
|
-
|
5
|
+
Tor.request will take an instance if has tor instanc, otherwise initialize an instance.
|
6
|
+
|
7
|
+
## What can it do
|
8
|
+
|
9
|
+
- Web Crawler
|
6
10
|
|
7
11
|
## Installation
|
8
12
|
|
@@ -39,6 +43,51 @@ Or install it yourself as:
|
|
39
43
|
Tor.request(url: '...', mode: :default) # Priority to use the highest number of successes, default is :default
|
40
44
|
Tor.request(url: '...', mode: :order) # in order
|
41
45
|
|
46
|
+
# show all instances
|
47
|
+
Tor.store.all
|
48
|
+
=> {"9001"=>#<Tor::Instance:0x007fb7798d2498 @port="9001", @ip="64.113.32.29", @using=nil, @counter=#<Counter success: 5, fail: 1, succss_at: 2017-12-15 19:52:44 +0800, fail_at:2017-12-15 19:52:26 +0800>>}
|
49
|
+
# Clear instances
|
50
|
+
Tor.clear
|
51
|
+
# Stop instance
|
52
|
+
Tor.stop(port)
|
53
|
+
|
54
|
+
# Initialize tors
|
55
|
+
Tor.init
|
56
|
+
I, [2017-12-15T20:40:36.189835 #60422] INFO -- : Open tor with port:9001
|
57
|
+
I, [2017-12-15T20:40:41.242452 #60422] INFO -- : Testing tor 9001
|
58
|
+
I, [2017-12-15T20:40:42.881857 #60422] INFO -- : IP: 163.172.67.180
|
59
|
+
I, [2017-12-15T20:40:42.882737 #60422] INFO -- : Open tor with port:9002
|
60
|
+
I, [2017-12-15T20:40:47.931555 #60422] INFO -- : Testing tor 9002
|
61
|
+
I, [2017-12-15T20:41:18.924004 #60422] INFO -- : IP: 155.4.230.97
|
62
|
+
I, [2017-12-15T20:41:18.925154 #60422] INFO -- : Open tor with port:9003
|
63
|
+
I, [2017-12-15T20:41:23.971647 #60422] INFO -- : Testing tor 9003
|
64
|
+
I, [2017-12-15T20:41:46.111952 #60422] INFO -- : IP: 46.17.97.112
|
65
|
+
I, [2017-12-15T20:41:46.112896 #60422] INFO -- : Open tor with port:9004
|
66
|
+
I, [2017-12-15T20:41:51.164009 #60422] INFO -- : Testing tor 9004
|
67
|
+
I, [2017-12-15T20:42:20.096998 #60422] INFO -- : IP: 109.201.133.100
|
68
|
+
I, [2017-12-15T20:42:20.097992 #60422] INFO -- : Open tor with port:9005
|
69
|
+
I, [2017-12-15T20:42:25.148808 #60422] INFO -- : Testing tor 9005
|
70
|
+
I, [2017-12-15T20:43:01.699053 #60422] INFO -- : IP: 62.210.129.246
|
71
|
+
I, [2017-12-15T20:43:01.700151 #60422] INFO -- : Open tor with port:9006
|
72
|
+
I, [2017-12-15T20:43:06.756239 #60422] INFO -- : Testing tor 9006
|
73
|
+
I, [2017-12-15T20:43:39.118469 #60422] INFO -- : IP: 163.172.160.182
|
74
|
+
I, [2017-12-15T20:43:39.119310 #60422] INFO -- : Open tor with port:9007
|
75
|
+
I, [2017-12-15T20:43:44.194975 #60422] INFO -- : Testing tor 9007
|
76
|
+
I, [2017-12-15T20:44:23.526274 #60422] INFO -- : IP: 89.144.12.14
|
77
|
+
I, [2017-12-15T20:44:23.527287 #60422] INFO -- : Open tor with port:9008
|
78
|
+
I, [2017-12-15T20:44:28.575018 #60422] INFO -- : Testing tor 9008
|
79
|
+
I, [2017-12-15T20:44:51.746405 #60422] INFO -- : IP: 195.22.126.147
|
80
|
+
I, [2017-12-15T20:44:51.747258 #60422] INFO -- : Open tor with port:9009
|
81
|
+
I, [2017-12-15T20:44:56.793446 #60422] INFO -- : Testing tor 9009
|
82
|
+
I, [2017-12-15T20:45:32.469774 #60422] INFO -- : IP: 62.210.129.246
|
83
|
+
I, [2017-12-15T20:45:32.470560 #60422] INFO -- : Open tor with port:9010
|
84
|
+
I, [2017-12-15T20:45:37.528114 #60422] INFO -- : Testing tor 9010
|
85
|
+
I, [2017-12-15T20:46:37.533066 #60422] INFO -- : Stop tor port:9010
|
86
|
+
I, [2017-12-15T20:46:37.533927 #60422] INFO -- : RestClient::Exceptions::OpenTimeout:Timed out connecting to server
|
87
|
+
I, [2017-12-15T20:46:37.534015 #60422] INFO -- : Open tor with port:9010
|
88
|
+
I, [2017-12-15T20:46:42.588824 #60422] INFO -- : Testing tor 9010
|
89
|
+
I, [2017-12-15T20:47:02.179464 #60422] INFO -- : IP: 173.254.216.66
|
90
|
+
|
42
91
|
|
43
92
|
## TODO
|
44
93
|
|
data/lib/rest-tor.rb
CHANGED
@@ -2,10 +2,11 @@ require 'nokogiri'
|
|
2
2
|
require 'rest-client'
|
3
3
|
require 'redis-objects'
|
4
4
|
require 'active_support/all'
|
5
|
+
require 'logger'
|
5
6
|
Dir.glob(File.expand_path('../rest_tor/**/*.rb', __FILE__)).each { |file| require file }
|
6
7
|
|
7
8
|
module Tor
|
8
9
|
def self.logger
|
9
10
|
Logger.new(STDOUT)
|
10
11
|
end
|
11
|
-
end
|
12
|
+
end
|
data/lib/rest_tor/instance.rb
CHANGED
@@ -21,6 +21,13 @@ module Tor
|
|
21
21
|
@counter= Counter.new(self, counter || {})
|
22
22
|
end
|
23
23
|
|
24
|
+
def pid
|
25
|
+
path = Tor::TOR_DIR.join("#{port}/tor.pid")
|
26
|
+
if File.exists?(path)
|
27
|
+
File.read(path).chomp.to_i
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
24
31
|
def attributes
|
25
32
|
{ ip: @ip, port: @port, using: @using, counter: @counter.to_h }
|
26
33
|
end
|
data/lib/rest_tor/tor.rb
CHANGED
@@ -1,19 +1,20 @@
|
|
1
|
-
require '
|
1
|
+
require 'pathname'
|
2
2
|
module Tor extend self
|
3
3
|
extend Lock
|
4
4
|
class Error < StandardError; end
|
5
5
|
class UnvaliablePort < Error; end
|
6
6
|
class DiedPortError < Error; end
|
7
7
|
class InvalidFormat < Error; end
|
8
|
-
USER_AGENT
|
9
|
-
MOBILE_AGENT
|
8
|
+
USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36'
|
9
|
+
MOBILE_AGENT = 'ANDROID_KFZ_COM_2.0.9_M6 Note_7.1.2'
|
10
10
|
TOR_COUNT = 10
|
11
11
|
TOR_PORT_START_WITH = 9000
|
12
|
+
TOR_DIR = Pathname.new('/tmp/tor')
|
12
13
|
|
13
14
|
def init
|
14
15
|
lock("tor:init", expires: 1.minutes) do
|
15
|
-
|
16
|
-
listen(TOR_PORT_START_WITH + i)
|
16
|
+
TOR_COUNT.times do |i|
|
17
|
+
listen(TOR_PORT_START_WITH + i + 1)
|
17
18
|
end
|
18
19
|
end
|
19
20
|
end
|
@@ -87,24 +88,26 @@ module Tor extend self
|
|
87
88
|
|
88
89
|
def stop(port)
|
89
90
|
logger.info "Stop tor port:#{port}"
|
90
|
-
|
91
|
+
instance = store[port]
|
92
|
+
if instance && instance.pid
|
93
|
+
Process.kill("KILL", instance.pid)
|
94
|
+
end
|
95
|
+
FileUtils.rm_rf(TOR_DIR.join(port.to_s))
|
96
|
+
rescue Exception
|
97
|
+
|
91
98
|
ensure
|
92
99
|
store.delete(port)
|
93
100
|
end
|
94
101
|
|
95
102
|
def listen(port)
|
96
103
|
return if port.blank? || !port.to_s.match(/^\d+$/)
|
97
|
-
# if `lsof -i -P -n |grep -E :#{port}`.chomp.present?
|
98
|
-
# logger.error "Unavailable port:#{port}, it used!"
|
99
|
-
# return
|
100
|
-
# end
|
101
104
|
|
102
105
|
logger.info "Open tor with port:#{port}"
|
103
106
|
|
104
107
|
control_port = 6000 + port.to_i
|
105
108
|
|
106
109
|
tor = 'tor --RunAsDaemon 1 --CookieAuthentication 0 --HashedControlPassword ""'
|
107
|
-
tor+= " --ControlPort #{ control_port } --PidFile tor
|
110
|
+
tor+= " --ControlPort #{ control_port } --PidFile tor.pid --SocksPort #{port} --DataDirectory #{dir(port)}"
|
108
111
|
tor+= " --CircuitBuildTimeout 5 --KeepalivePeriod 60 --NewCircuitPeriod 15 --NumEntryGuards 8"# make tor faster
|
109
112
|
tor+= " --quiet" # unless Rails.env.production?
|
110
113
|
system tor
|
@@ -130,7 +133,7 @@ module Tor extend self
|
|
130
133
|
end
|
131
134
|
|
132
135
|
def dir(port)
|
133
|
-
"
|
136
|
+
TOR_DIR.join("#{port}").tap do |dir|
|
134
137
|
FileUtils.mkpath(dir) if not Dir.exists?(dir)
|
135
138
|
end
|
136
139
|
end
|
@@ -150,12 +153,20 @@ module Tor extend self
|
|
150
153
|
end
|
151
154
|
|
152
155
|
def clear
|
153
|
-
|
156
|
+
Dir.glob(TOR_DIR.join("**/*.pid")).each do |path|
|
157
|
+
begin
|
158
|
+
Process.kill("KILL", File.read(path).chomp.to_i)
|
159
|
+
rescue Errno::ESRCH
|
160
|
+
end
|
161
|
+
end
|
162
|
+
FileUtils.rm_rf(TOR_DIR)
|
163
|
+
true
|
164
|
+
ensure
|
154
165
|
store.clear
|
155
166
|
end
|
156
167
|
|
157
168
|
def count
|
158
|
-
|
169
|
+
Dir.glob(TOR_DIR.join("*")).count
|
159
170
|
end
|
160
171
|
|
161
172
|
def unused
|
data/lib/rest_tor/version.rb
CHANGED