rest-tor 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9e0212c56daffeb88632e8b62c0d2223a3baae6e
-  data.tar.gz: d2830fcc80de05933abe3239d0733406992fdb13
+  metadata.gz: 7633b710960f440ecc94cf9d03ca0373d39daa10
+  data.tar.gz: ee7c1d41043e5a7f53c00b9fc2cc51586f6b1537
 SHA512:
-  metadata.gz: ede11fc0af2cd7d8b7c633529bd86a019801cb627b6865aa3c505ad62a40b6297cc6ab2bb36d1693cadaa41f395faac389bf683c9379cdc5191839ae2b48de76
-  data.tar.gz: 6ee03b7714f3e057b8ec3db1b24631f85df7e13c10e020a86cc279de47a7192cc424f7a25fdf360a1d0a2c42c5ae6304cc6ce22b1b8a3e712bb296181d561544
+  metadata.gz: 59bc545c50834cf402202aea5c91a93fdc72265ea19de9a380174b9df3453ac83cfdafc628931793b623e4b4d8685a4cdaeac2ec3f1d68a493d2cedf284192c4
+  data.tar.gz: cac174ee711d9c92b0ec296734465ee2a1eac8f41048c20496aa9a420ba00044205e6b215a44d3d057c461231df4a72bfb1e9b0a9852aab0387f77ecf89588d8

data/README.md

@@ -1,8 +1,12 @@
-# Rest::Tor
+# Rest Tor
 
-Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/rest/tor`. To experiment with that code, run `bin/console` for an interactive prompt.
+Tor for ruby.  Support for multiple processes.  Each request will take an tor instance.
 
-TODO: Delete this and the text above, and describe your gem
+Tor.request will  take an instance if has tor instanc, otherwise initialize an instance.
+
+## What can it do
+
+- Web Crawler
 
 ## Installation
 
@@ -39,6 +43,51 @@ Or install it yourself as:
       Tor.request(url: '...', mode: :default)  # Priority to use the highest number of successes, default is :default
       Tor.request(url: '...', mode: :order) # in order
 
+      # show all instances
+      Tor.store.all
+      => {"9001"=>#<Tor::Instance:0x007fb7798d2498 @port="9001", @ip="64.113.32.29", @using=nil, @counter=#<Counter success: 5, fail: 1, succss_at: 2017-12-15 19:52:44 +0800, fail_at:2017-12-15 19:52:26 +0800>>}
+      # Clear instances
+      Tor.clear
+      # Stop instance
+      Tor.stop(port)
+
+      # Initialize tors
+      Tor.init
+      I, [2017-12-15T20:40:36.189835 #60422]  INFO -- : Open tor with port:9001
+      I, [2017-12-15T20:40:41.242452 #60422]  INFO -- : Testing tor 9001
+      I, [2017-12-15T20:40:42.881857 #60422]  INFO -- :   IP: 163.172.67.180 
+      I, [2017-12-15T20:40:42.882737 #60422]  INFO -- : Open tor with port:9002
+      I, [2017-12-15T20:40:47.931555 #60422]  INFO -- : Testing tor 9002
+      I, [2017-12-15T20:41:18.924004 #60422]  INFO -- :   IP: 155.4.230.97 
+      I, [2017-12-15T20:41:18.925154 #60422]  INFO -- : Open tor with port:9003
+      I, [2017-12-15T20:41:23.971647 #60422]  INFO -- : Testing tor 9003
+      I, [2017-12-15T20:41:46.111952 #60422]  INFO -- :   IP: 46.17.97.112 
+      I, [2017-12-15T20:41:46.112896 #60422]  INFO -- : Open tor with port:9004
+      I, [2017-12-15T20:41:51.164009 #60422]  INFO -- : Testing tor 9004
+      I, [2017-12-15T20:42:20.096998 #60422]  INFO -- :   IP: 109.201.133.100 
+      I, [2017-12-15T20:42:20.097992 #60422]  INFO -- : Open tor with port:9005
+      I, [2017-12-15T20:42:25.148808 #60422]  INFO -- : Testing tor 9005
+      I, [2017-12-15T20:43:01.699053 #60422]  INFO -- :   IP: 62.210.129.246 
+      I, [2017-12-15T20:43:01.700151 #60422]  INFO -- : Open tor with port:9006
+      I, [2017-12-15T20:43:06.756239 #60422]  INFO -- : Testing tor 9006
+      I, [2017-12-15T20:43:39.118469 #60422]  INFO -- :   IP: 163.172.160.182 
+      I, [2017-12-15T20:43:39.119310 #60422]  INFO -- : Open tor with port:9007
+      I, [2017-12-15T20:43:44.194975 #60422]  INFO -- : Testing tor 9007
+      I, [2017-12-15T20:44:23.526274 #60422]  INFO -- :   IP: 89.144.12.14 
+      I, [2017-12-15T20:44:23.527287 #60422]  INFO -- : Open tor with port:9008
+      I, [2017-12-15T20:44:28.575018 #60422]  INFO -- : Testing tor 9008
+      I, [2017-12-15T20:44:51.746405 #60422]  INFO -- :   IP: 195.22.126.147 
+      I, [2017-12-15T20:44:51.747258 #60422]  INFO -- : Open tor with port:9009
+      I, [2017-12-15T20:44:56.793446 #60422]  INFO -- : Testing tor 9009
+      I, [2017-12-15T20:45:32.469774 #60422]  INFO -- :   IP: 62.210.129.246 
+      I, [2017-12-15T20:45:32.470560 #60422]  INFO -- : Open tor with port:9010
+      I, [2017-12-15T20:45:37.528114 #60422]  INFO -- : Testing tor 9010
+      I, [2017-12-15T20:46:37.533066 #60422]  INFO -- : Stop tor port:9010
+      I, [2017-12-15T20:46:37.533927 #60422]  INFO -- : RestClient::Exceptions::OpenTimeout:Timed out connecting to server
+      I, [2017-12-15T20:46:37.534015 #60422]  INFO -- : Open tor with port:9010
+      I, [2017-12-15T20:46:42.588824 #60422]  INFO -- : Testing tor 9010
+      I, [2017-12-15T20:47:02.179464 #60422]  INFO -- :   IP: 173.254.216.66
+
 
 ## TODO
 

data/lib/rest-tor.rb

@@ -2,10 +2,11 @@ require 'nokogiri'
 require 'rest-client'
 require 'redis-objects'
 require 'active_support/all'
+require 'logger'
 Dir.glob(File.expand_path('../rest_tor/**/*.rb', __FILE__)).each { |file| require file }
 
 module Tor
   def self.logger
     Logger.new(STDOUT)
   end
-end
+end

data/lib/rest_tor/instance.rb

@@ -21,6 +21,13 @@ module Tor
       @counter= Counter.new(self, counter || {})
     end
 
+    def pid
+      path = Tor::TOR_DIR.join("#{port}/tor.pid")
+      if File.exists?(path)
+        File.read(path).chomp.to_i
+      end
+    end
+
     def attributes
       { ip: @ip, port: @port, using: @using, counter: @counter.to_h }
     end

data/lib/rest_tor/tor.rb

@@ -1,19 +1,20 @@
-require 'open3'
+require 'pathname'
 module Tor extend self
   extend Lock
   class Error < StandardError; end
   class UnvaliablePort < Error; end
   class DiedPortError < Error; end
   class InvalidFormat < Error; end
-  USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36'
-  MOBILE_AGENT = 'ANDROID_KFZ_COM_2.0.9_M6 Note_7.1.2'
+  USER_AGENT          = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.36'
+  MOBILE_AGENT        = 'ANDROID_KFZ_COM_2.0.9_M6 Note_7.1.2'
   TOR_COUNT           = 10
   TOR_PORT_START_WITH = 9000
+  TOR_DIR             = Pathname.new('/tmp/tor')
 
   def init
     lock("tor:init", expires: 1.minutes) do
-      Parallel.each(0...TOR_COUNT, in_threads: 20, progress: 'Init Tor') do |i|
-        listen(TOR_PORT_START_WITH + i)
+      TOR_COUNT.times do |i|
+        listen(TOR_PORT_START_WITH + i + 1)
       end
     end
   end
@@ -87,24 +88,26 @@ module Tor extend self
 
   def stop(port)
     logger.info "Stop tor port:#{port}"
-    Open3.pipeline("ps aux", "grep tor", "grep RunAsDaemon", "grep #{port}", "awk '{print $2}'", "xargs kill -9")
+    instance = store[port]
+    if instance && instance.pid
+      Process.kill("KILL", instance.pid)
+    end
+    FileUtils.rm_rf(TOR_DIR.join(port.to_s))
+  rescue Exception
+    
   ensure
     store.delete(port)
   end
 
   def listen(port)
     return if port.blank? || !port.to_s.match(/^\d+$/)
-    # if `lsof -i -P -n |grep -E :#{port}`.chomp.present?
-    #   logger.error "Unavailable port:#{port}, it used!"
-    #   return
-    # end
 
     logger.info "Open tor with port:#{port}"
 
     control_port = 6000 + port.to_i
 
     tor = 'tor --RunAsDaemon 1 --CookieAuthentication 0 --HashedControlPassword ""'
-    tor+= " --ControlPort #{ control_port } --PidFile tor#{port}.pid --SocksPort #{port} --DataDirectory #{dir(port)}"
+    tor+= " --ControlPort #{ control_port } --PidFile tor.pid --SocksPort #{port} --DataDirectory #{dir(port)}"
     tor+= " --CircuitBuildTimeout 5 --KeepalivePeriod 60 --NewCircuitPeriod 15 --NumEntryGuards 8"# make tor faster
     tor+= " --quiet" # unless Rails.env.production?
     system tor
@@ -130,7 +133,7 @@ module Tor extend self
   end
 
   def dir(port)
-    "/tmp/tor/#{port}".tap do |dir|
+    TOR_DIR.join("#{port}").tap do |dir|
       FileUtils.mkpath(dir) if not Dir.exists?(dir)
     end
   end
@@ -150,12 +153,20 @@ module Tor extend self
   end
 
   def clear
-    `ps aux | grep tor | grep RunAsDaemon|grep -v "ps aux" | awk '{print $2}' | xargs kill -9`
+    Dir.glob(TOR_DIR.join("**/*.pid")).each do |path|
+      begin
+        Process.kill("KILL", File.read(path).chomp.to_i)
+      rescue Errno::ESRCH
+      end
+    end
+    FileUtils.rm_rf(TOR_DIR)
+    true
+  ensure
     store.clear
   end
 
   def count
-    `ps aux | grep tor | grep RunAsDaemon|grep -v "ps aux"`.to_s.strip.split("\n").keep_if { |s| s.strip.present? }.count
+    Dir.glob(TOR_DIR.join("*")).count
   end
 
   def unused

data/lib/rest_tor/version.rb

@@ -1,3 +1,3 @@
 module Tor
-  VERSION = "0.1.1"
+  VERSION = "0.1.2"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rest-tor
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - plusor