rest-tor 0.1.1 → 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 9e0212c56daffeb88632e8b62c0d2223a3baae6e
4
- data.tar.gz: d2830fcc80de05933abe3239d0733406992fdb13
3
+ metadata.gz: 7633b710960f440ecc94cf9d03ca0373d39daa10
4
+ data.tar.gz: ee7c1d41043e5a7f53c00b9fc2cc51586f6b1537
5
5
  SHA512:
6
- metadata.gz: ede11fc0af2cd7d8b7c633529bd86a019801cb627b6865aa3c505ad62a40b6297cc6ab2bb36d1693cadaa41f395faac389bf683c9379cdc5191839ae2b48de76
7
- data.tar.gz: 6ee03b7714f3e057b8ec3db1b24631f85df7e13c10e020a86cc279de47a7192cc424f7a25fdf360a1d0a2c42c5ae6304cc6ce22b1b8a3e712bb296181d561544
6
+ metadata.gz: 59bc545c50834cf402202aea5c91a93fdc72265ea19de9a380174b9df3453ac83cfdafc628931793b623e4b4d8685a4cdaeac2ec3f1d68a493d2cedf284192c4
7
+ data.tar.gz: cac174ee711d9c92b0ec296734465ee2a1eac8f41048c20496aa9a420ba00044205e6b215a44d3d057c461231df4a72bfb1e9b0a9852aab0387f77ecf89588d8
data/README.md CHANGED
@@ -1,8 +1,12 @@
1
- # Rest::Tor
1
+ # Rest Tor
2
2
 
3
- Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/rest/tor`. To experiment with that code, run `bin/console` for an interactive prompt.
3
+ Tor for ruby. Support for multiple processes. Each request will take an tor instance.
4
4
 
5
- TODO: Delete this and the text above, and describe your gem
5
+ Tor.request will take an instance if has tor instanc, otherwise initialize an instance.
6
+
7
+ ## What can it do
8
+
9
+ - Web Crawler
6
10
 
7
11
  ## Installation
8
12
 
@@ -39,6 +43,51 @@ Or install it yourself as:
39
43
  Tor.request(url: '...', mode: :default) # Priority to use the highest number of successes, default is :default
40
44
  Tor.request(url: '...', mode: :order) # in order
41
45
 
46
+ # show all instances
47
+ Tor.store.all
48
+ => {"9001"=>#<Tor::Instance:0x007fb7798d2498 @port="9001", @ip="64.113.32.29", @using=nil, @counter=#<Counter success: 5, fail: 1, succss_at: 2017-12-15 19:52:44 +0800, fail_at:2017-12-15 19:52:26 +0800>>}
49
+ # Clear instances
50
+ Tor.clear
51
+ # Stop instance
52
+ Tor.stop(port)
53
+
54
+ # Initialize tors
55
+ Tor.init
56
+ I, [2017-12-15T20:40:36.189835 #60422] INFO -- : Open tor with port:9001
57
+ I, [2017-12-15T20:40:41.242452 #60422] INFO -- : Testing tor 9001
58
+ I, [2017-12-15T20:40:42.881857 #60422] INFO -- : IP: 163.172.67.180
59
+ I, [2017-12-15T20:40:42.882737 #60422] INFO -- : Open tor with port:9002
60
+ I, [2017-12-15T20:40:47.931555 #60422] INFO -- : Testing tor 9002
61
+ I, [2017-12-15T20:41:18.924004 #60422] INFO -- : IP: 155.4.230.97
62
+ I, [2017-12-15T20:41:18.925154 #60422] INFO -- : Open tor with port:9003
63
+ I, [2017-12-15T20:41:23.971647 #60422] INFO -- : Testing tor 9003
64
+ I, [2017-12-15T20:41:46.111952 #60422] INFO -- : IP: 46.17.97.112
65
+ I, [2017-12-15T20:41:46.112896 #60422] INFO -- : Open tor with port:9004
66
+ I, [2017-12-15T20:41:51.164009 #60422] INFO -- : Testing tor 9004
67
+ I, [2017-12-15T20:42:20.096998 #60422] INFO -- : IP: 109.201.133.100
68
+ I, [2017-12-15T20:42:20.097992 #60422] INFO -- : Open tor with port:9005
69
+ I, [2017-12-15T20:42:25.148808 #60422] INFO -- : Testing tor 9005
70
+ I, [2017-12-15T20:43:01.699053 #60422] INFO -- : IP: 62.210.129.246
71
+ I, [2017-12-15T20:43:01.700151 #60422] INFO -- : Open tor with port:9006
72
+ I, [2017-12-15T20:43:06.756239 #60422] INFO -- : Testing tor 9006
73
+ I, [2017-12-15T20:43:39.118469 #60422] INFO -- : IP: 163.172.160.182
74
+ I, [2017-12-15T20:43:39.119310 #60422] INFO -- : Open tor with port:9007
75
+ I, [2017-12-15T20:43:44.194975 #60422] INFO -- : Testing tor 9007
76
+ I, [2017-12-15T20:44:23.526274 #60422] INFO -- : IP: 89.144.12.14
77
+ I, [2017-12-15T20:44:23.527287 #60422] INFO -- : Open tor with port:9008
78
+ I, [2017-12-15T20:44:28.575018 #60422] INFO -- : Testing tor 9008
79
+ I, [2017-12-15T20:44:51.746405 #60422] INFO -- : IP: 195.22.126.147
80
+ I, [2017-12-15T20:44:51.747258 #60422] INFO -- : Open tor with port:9009
81
+ I, [2017-12-15T20:44:56.793446 #60422] INFO -- : Testing tor 9009
82
+ I, [2017-12-15T20:45:32.469774 #60422] INFO -- : IP: 62.210.129.246
83
+ I, [2017-12-15T20:45:32.470560 #60422] INFO -- : Open tor with port:9010
84
+ I, [2017-12-15T20:45:37.528114 #60422] INFO -- : Testing tor 9010
85
+ I, [2017-12-15T20:46:37.533066 #60422] INFO -- : Stop tor port:9010
86
+ I, [2017-12-15T20:46:37.533927 #60422] INFO -- : RestClient::Exceptions::OpenTimeout:Timed out connecting to server
87
+ I, [2017-12-15T20:46:37.534015 #60422] INFO -- : Open tor with port:9010
88
+ I, [2017-12-15T20:46:42.588824 #60422] INFO -- : Testing tor 9010
89
+ I, [2017-12-15T20:47:02.179464 #60422] INFO -- : IP: 173.254.216.66
90
+
42
91
 
43
92
  ## TODO
44
93
 
data/lib/rest-tor.rb CHANGED
@@ -2,10 +2,11 @@ require 'nokogiri'
2
2
  require 'rest-client'
3
3
  require 'redis-objects'
4
4
  require 'active_support/all'
5
+ require 'logger'
5
6
  Dir.glob(File.expand_path('../rest_tor/**/*.rb', __FILE__)).each { |file| require file }
6
7
 
7
8
  module Tor
8
9
  def self.logger
9
10
  Logger.new(STDOUT)
10
11
  end
11
- end
12
+ end
@@ -21,6 +21,13 @@ module Tor
21
21
  @counter= Counter.new(self, counter || {})
22
22
  end
23
23
 
24
+ def pid
25
+ path = Tor::TOR_DIR.join("#{port}/tor.pid")
26
+ if File.exists?(path)
27
+ File.read(path).chomp.to_i
28
+ end
29
+ end
30
+
24
31
  def attributes
25
32
  { ip: @ip, port: @port, using: @using, counter: @counter.to_h }
26
33
  end
data/lib/rest_tor/tor.rb CHANGED
@@ -1,19 +1,20 @@
1
- require 'open3'
1
+ require 'pathname'
2
2
  module Tor extend self
3
3
  extend Lock
4
4
  class Error < StandardError; end
5
5
  class UnvaliablePort < Error; end
6
6
  class DiedPortError < Error; end
7
7
  class InvalidFormat < Error; end
8
- USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36'
9
- MOBILE_AGENT = 'ANDROID_KFZ_COM_2.0.9_M6 Note_7.1.2'
8
+ USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36'
9
+ MOBILE_AGENT = 'ANDROID_KFZ_COM_2.0.9_M6 Note_7.1.2'
10
10
  TOR_COUNT = 10
11
11
  TOR_PORT_START_WITH = 9000
12
+ TOR_DIR = Pathname.new('/tmp/tor')
12
13
 
13
14
  def init
14
15
  lock("tor:init", expires: 1.minutes) do
15
- Parallel.each(0...TOR_COUNT, in_threads: 20, progress: 'Init Tor') do |i|
16
- listen(TOR_PORT_START_WITH + i)
16
+ TOR_COUNT.times do |i|
17
+ listen(TOR_PORT_START_WITH + i + 1)
17
18
  end
18
19
  end
19
20
  end
@@ -87,24 +88,26 @@ module Tor extend self
87
88
 
88
89
  def stop(port)
89
90
  logger.info "Stop tor port:#{port}"
90
- Open3.pipeline("ps aux", "grep tor", "grep RunAsDaemon", "grep #{port}", "awk '{print $2}'", "xargs kill -9")
91
+ instance = store[port]
92
+ if instance && instance.pid
93
+ Process.kill("KILL", instance.pid)
94
+ end
95
+ FileUtils.rm_rf(TOR_DIR.join(port.to_s))
96
+ rescue Exception
97
+
91
98
  ensure
92
99
  store.delete(port)
93
100
  end
94
101
 
95
102
  def listen(port)
96
103
  return if port.blank? || !port.to_s.match(/^\d+$/)
97
- # if `lsof -i -P -n |grep -E :#{port}`.chomp.present?
98
- # logger.error "Unavailable port:#{port}, it used!"
99
- # return
100
- # end
101
104
 
102
105
  logger.info "Open tor with port:#{port}"
103
106
 
104
107
  control_port = 6000 + port.to_i
105
108
 
106
109
  tor = 'tor --RunAsDaemon 1 --CookieAuthentication 0 --HashedControlPassword ""'
107
- tor+= " --ControlPort #{ control_port } --PidFile tor#{port}.pid --SocksPort #{port} --DataDirectory #{dir(port)}"
110
+ tor+= " --ControlPort #{ control_port } --PidFile tor.pid --SocksPort #{port} --DataDirectory #{dir(port)}"
108
111
  tor+= " --CircuitBuildTimeout 5 --KeepalivePeriod 60 --NewCircuitPeriod 15 --NumEntryGuards 8"# make tor faster
109
112
  tor+= " --quiet" # unless Rails.env.production?
110
113
  system tor
@@ -130,7 +133,7 @@ module Tor extend self
130
133
  end
131
134
 
132
135
  def dir(port)
133
- "/tmp/tor/#{port}".tap do |dir|
136
+ TOR_DIR.join("#{port}").tap do |dir|
134
137
  FileUtils.mkpath(dir) if not Dir.exists?(dir)
135
138
  end
136
139
  end
@@ -150,12 +153,20 @@ module Tor extend self
150
153
  end
151
154
 
152
155
  def clear
153
- `ps aux | grep tor | grep RunAsDaemon|grep -v "ps aux" | awk '{print $2}' | xargs kill -9`
156
+ Dir.glob(TOR_DIR.join("**/*.pid")).each do |path|
157
+ begin
158
+ Process.kill("KILL", File.read(path).chomp.to_i)
159
+ rescue Errno::ESRCH
160
+ end
161
+ end
162
+ FileUtils.rm_rf(TOR_DIR)
163
+ true
164
+ ensure
154
165
  store.clear
155
166
  end
156
167
 
157
168
  def count
158
- `ps aux | grep tor | grep RunAsDaemon|grep -v "ps aux"`.to_s.strip.split("\n").keep_if { |s| s.strip.present? }.count
169
+ Dir.glob(TOR_DIR.join("*")).count
159
170
  end
160
171
 
161
172
  def unused
@@ -1,3 +1,3 @@
1
1
  module Tor
2
- VERSION = "0.1.1"
2
+ VERSION = "0.1.2"
3
3
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: rest-tor
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.1
4
+ version: 0.1.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - plusor