r_proxy 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f8a74c2bf49d46d1c83f61c84c5d93e489180f73fe6d6bb1a691700db3a55e66
-  data.tar.gz: 03d4d23c639f5b410f3d50260d0d7db241bb0393cbc0c0f1b6a3c2c584a22c4d
+  metadata.gz: eb665c8aedadf57b45585d927e08de7e572dc1e5bfdca207a0f889b41791d4c9
+  data.tar.gz: 8fea768841cce3fc8741efdefdbee247b0a48abf680bf906d35f97e6fcbf2e43
 SHA512:
-  metadata.gz: a8cde243c051d33e13a64160f31582a5fed08db3077a470b6823db087c5fcd36092dbaec32d80b51f620b07fe3dd317747ef800bb964ee0e0c2e00ef0cf523b1
-  data.tar.gz: '0903bb936989764a934a10f0c8c3070324f97ec1c3416be22082a97eaaa28283da8ff749faa11d4e3d98ee0d23e25de279f7edaef5e70382cd92b128f5c2cf84'
+  metadata.gz: e747ed1002a3a05330312691c9dbe2648c96859ff8af184dcd3c1e5f2738f58f37642343806f5ade14735c7a931e84f58a20279adf8bf3c088d5d71f66b5539e
+  data.tar.gz: b850bad10bf4b3d361bfcb8fa9d1fbe395dc6816c1e14afe3799d9be876fab53f1db3c8b25419116af121fb91e9f9141c7f0ca4cee0418ca15da93691c351ce8

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    r_proxy (0.1.1)
+    r_proxy (0.2.0)
       eventmachine (~> 1.2, >= 1.2.7)
       redis (~> 4.1, >= 4.1.4)
 

data/README.md

@@ -2,7 +2,7 @@
 
 a powerful ruby http proxy server, base on eventmachine
 
-able to run in multi-process like nginx works
+able to run in multi-process like nginx workers
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -79,5 +79,5 @@ server.run!
 
 ## Contributing
 
-Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/r_proxy.
+Bug reports and pull requests are welcome on GitHub at https://github.com/nickoan/r_proxy.
 

data/example.rb

@@ -7,11 +7,16 @@ server.set(:port, 8080)
 
 server.set(:instances, 3)
 
-server.set(:disable_auth, true)
-# server.set(:disable_unbind_cb, true)
-server.set(:enable_ssl, false)
+#server.set(:disable_auth, true)
+server.set(:disable_unbind_cb, false)
+server.set(:enable_ssl, true)
 
-server.set(:callback_url,'http://127.0.0.1:1234')
+server.set(:callback_url,'http://localhost:3000/api/proxy_callback')
+
+server.set(:no_cache_below, 1 * 1024 * 1024 * 1024)
+server.set(:cache_clear_threshold, 1)
+server.set(:enable_force_quit, true)
+server.set(:enable_cache, true)
 
 server.set(:redis_url, "redis://@localhost:6379/1")
 

data/lib/r_proxy/cache_pool.rb

@@ -0,0 +1,37 @@
+module RProxy
+  class CachePool
+
+    def initialize
+      @pool = {}
+      @able_write = true
+    end
+
+    def []=(key, value)
+      return value if !@able_write
+      @pool[key] = value
+    end
+
+
+    def [](key)
+      @pool[key]
+    end
+
+    def writable?
+      @able_write
+    end
+
+    def disable_write!
+      @able_write = false
+    end
+
+    def enable_write!
+      @able_write = true
+    end
+
+    def flush
+      tmp = @pool
+      @pool = {}
+      tmp
+    end
+  end
+end

data/lib/r_proxy/callback_service.rb

@@ -1,6 +1,7 @@
 module RProxy
   class CallbackService
     def self.call(url, user, pass, value)
+      return if url.nil? || url.empty?
       uri = URI(url)
       tls = uri.scheme == 'https'
 

data/lib/r_proxy/check_snapshot_service.rb

@@ -20,9 +20,13 @@ module RProxy
         tmp = snapshot_value.to_i - result.to_i
 
         if tmp >= @usage_threshold
-          connection = RProxy::CallbackService.call(@cb_url, user, pass, tmp)
-          connection.assign_logger(@logger)
-          @redis.setex(s_key, @snapshot_expire_in, result)
+          begin
+            connection = RProxy::CallbackService.call(@cb_url, user, pass, tmp)
+            connection.assign_logger(@logger)
+            @redis.setex(s_key, @snapshot_expire_in, result)
+          rescue => e
+            @logger.error("callback service: @id:#{s_key}, #{e.message}") if @logger
+          end
         end
       end
     end

data/lib/r_proxy/config.rb

@@ -40,6 +40,12 @@ module RProxy
     add_config(:usage_threshold, 1 * 1024 * 1024 * 1024)
     add_config(:proxy_buffer, 1024 * 1024 * 10) # default is 10M
 
+    add_config(:enable_cache, true)
+    add_config(:cache_clear_threshold, 200)
+    add_config(:no_cache_below, 500 * 1024 * 1024)
+
+    add_config(:enable_force_quit, true)
+
     add_config(:disable_auth, false)
     add_config(:disable_unbind_cb, false)
 

data/lib/r_proxy/connection_handler.rb

@@ -1,10 +1,9 @@
 module RProxy
   class ConnectionHandler < EM::Connection
-    def initialize(config)
+    def initialize(config, cache_pool)
       @config = config
       @logger = @config.logger
       @redis = RProxy::RedisService.instance(@config.redis_url)
-      @http_parser = HttpProxyParser.new(@redis)
       @disable_auth = @config.disable_auth
       @disable_unbind_cb = @config.disable_unbind_cb
       @buffer_size = @config.proxy_buffer
@@ -12,9 +11,11 @@ module RProxy
       @username = nil
       @password = nil
       @target_connection = nil
-
-      @unbind_service = UnbindService.new(config, @redis)
+      @cache_pool = cache_pool
+      @usage_manager = RProxy::UsageManager.new(config, @cache_pool, @redis)
+      @http_parser = HttpProxyParser.new(@usage_manager)
       @snapshot_service = RProxy::CheckSnapshotService.new(@redis, @config)
+      @enable_force_quit = config.enable_force_quit
     end
 
     def post_init
@@ -27,9 +28,11 @@ module RProxy
         end
         @port, @ip = Socket.unpack_sockaddr_in(get_peername)
 
-        @timer = EventMachine.add_timer(20) do
-          self.close_connection(false)
-          @timer = nil
+        if @enable_force_quit
+          @timer = EventMachine.add_timer(20) do
+            self.close_connection(false)
+            @timer = nil
+          end
         end
       rescue => e
         if @logger
@@ -50,7 +53,7 @@ module RProxy
                   self,
                   @disable_unbind_cb,
                   @buffer_size,
-                  @unbind_service)
+                  @usage_manager)
         @target_connection.assign_logger(@logger)
         if !@disable_auth
           @username = @http_parser.username
@@ -79,8 +82,11 @@ module RProxy
     end
 
     def unbind
+      if @timer
+        EventMachine.cancel_timer(@timer)
+      end
       return if @disable_unbind_cb
-      @unbind_service.call(@username, @password, get_proxied_bytes)
+      @usage_manager.report_usage(@username, @password, get_proxied_bytes)
     end
   end
 end

data/lib/r_proxy/http_post_template.rb

@@ -36,7 +36,7 @@ module RProxy
 
     def init_headers
       {
-        'User-Agent' => "RSocks/#{RProxy::VERSION}",
+        'User-Agent' => "RProxy/#{RProxy::VERSION}",
         'Content-Type' => 'application/json',
       }
     end

data/lib/r_proxy/http_proxy_parser.rb

@@ -5,8 +5,8 @@ module RProxy
 
     attr_reader :username, :password
 
-    def initialize(redis)
-      @redis = redis
+    def initialize(usage_manager)
+      @usage_manager = usage_manager
       @max_connection_size = 4 * 1024
     end
 
@@ -31,11 +31,10 @@ module RProxy
       rescue
         raise RProxy::HTTPNotSupport, "token parse failed #{token}"
       end
-      key = "proxy:#{@username}-#{@password}"
-      value = @redis.get(key)
 
-      raise RProxy::HTTPAuthFailed if value.nil?
-      value
+      auth_result = @usage_manager.auth_user(@username, @password)
+      raise RProxy::HTTPAuthFailed if auth_result.nil?
+      auth_result
     end
 
     def parse_connect_request(data)

data/lib/r_proxy/master_process.rb

@@ -42,11 +42,11 @@ module RProxy
       instance_amount = @config.instances
       server = TCPServer.new(@config.host, @config.port)
       instance_amount.times do
-        timestamp = Time.now.to_i
         pid =  Process.fork do
+          timestamp = (Time.now.to_f * 1000).round
           begin
             @logger.info("r_proxy @#{timestamp} process start....") if @logger
-            RProxy::ProxyServer.new(server, @config).run!
+            RProxy::ProxyServer.new(server, @config, timestamp).run!
           rescue Interrupt
             @logger.info("r_proxy TPC server instance @#{timestamp} closed now....") if @logger
           rescue => e
@@ -57,6 +57,7 @@ module RProxy
 
         Process.detach(pid)
         @pids << pid
+        sleep(0.1)
       end
 
       EventMachine.kqueue=(true)

data/lib/r_proxy/proxy_server.rb

@@ -1,14 +1,50 @@
 module RProxy
   class ProxyServer
-    def initialize(sock, config)
+    def initialize(sock, config, instance_id)
       @sock = sock
       @config = config
+      @cache_pool = RProxy::CachePool.new
+      @logger = @config.logger
+      @instance_id = instance_id
     end
 
     def run!
       Signal.trap("TERM") { exit! }
       EventMachine.run do
-        EventMachine.attach_server(@sock, RProxy::ConnectionHandler, @config)
+
+        if @config.enable_cache
+          @period_timer = EventMachine.add_periodic_timer(30) do
+            @cache_pool.disable_write!
+
+            tmp = @cache_pool.flush
+
+            report_and_clean_cache(tmp)
+
+            @cache_pool.enable_write!
+          end
+        end
+
+        EventMachine.attach_server(@sock, RProxy::ConnectionHandler, @config, @cache_pool)
+      end
+    end
+
+    private
+
+    def report_and_clean_cache(cache)
+      return if cache.empty?
+
+      redis = RProxy::RedisService.instance(@config.redis_url)
+      time_start = (Time.now.to_f * 1000).floor
+      cache.each do |k, value|
+        used = value[:used]
+        next if used.nil? || used.zero?
+        redis.decrby(k, used)
+      end
+      time_end = (Time.now.to_f * 1000).floor
+      spend = time_end - time_start
+
+      if spend > @config.cache_clear_threshold
+        @logger.info("@#{@instance_id} clear cache: #{spend} ms, remove: #{cache.size} items") if @logger
       end
     end
   end

data/lib/r_proxy/target_connection.rb

@@ -1,11 +1,11 @@
 module RProxy
   class TargetConnection < EM::Connection
 
-    def initialize(client, disable_cb, buffer_size, unbind)
+    def initialize(client, disable_cb, buffer_size, usage_manager)
       @disable_unbind_callback = disable_cb
       @client_connection = client
       @buffer_size = buffer_size
-      @unbind_service = unbind
+      @usage_manager = usage_manager
     end
 
     def assign_logger(logger)
@@ -27,7 +27,7 @@ module RProxy
 
     def unbind
       return if @disable_unbind_callback
-      @unbind_service.call(@username, @password, get_proxied_bytes)
+      @usage_manager.report_usage(@username, @password, get_proxied_bytes)
     end
 
     private

data/lib/r_proxy/usage_manager.rb

@@ -0,0 +1,59 @@
+module RProxy
+  class UsageManager
+    def initialize(config, cache_pool, redis)
+      @enable_cache = config.enable_cache
+      @cache_pool = cache_pool
+      @redis = redis
+      @no_cache_below = config.no_cache_below
+    end
+
+    def auth_user(user, pass)
+      key = proxy_key(user, pass)
+
+      value = fetch_usage(key)
+
+      return value if !value.nil? && value.to_i > 0
+      nil
+    end
+
+    def report_usage(user, pass, value)
+      return if user.nil? || pass.nil? || value.nil?
+
+      key = proxy_key(user, pass)
+      cache = @cache_pool[key]
+
+      if cache.nil? || !@cache_pool.writable?
+        @redis.decrby(key, value)
+      else
+        tmp = cache[:used]
+        @cache_pool[key][:used] = tmp + value
+      end
+    end
+
+    private
+
+    def fetch_usage(key)
+      return @redis.get(key) if !@enable_cache || !@cache_pool.writable?
+
+      cache = @cache_pool[key]
+
+      if cache.nil?
+        value = @redis.get(key)
+
+        return value if !value.nil? && value.to_i <= @no_cache_below
+
+        @cache_pool[key] = {
+          usage: value,
+          used: 0
+        }
+        return value
+      end
+
+      cache[:usage]
+    end
+
+    def proxy_key(user, pass)
+      "proxy:#{user}-#{pass}"
+    end
+  end
+end

data/lib/r_proxy/version.rb

@@ -1,3 +1,3 @@
 module RProxy
-  VERSION = "0.1.1"
+  VERSION = "0.2.0"
 end

data/lib/r_proxy.rb

@@ -3,6 +3,8 @@ require 'r_proxy/version'
 require 'eventmachine'
 require 'redis'
 require 'r_proxy/config'
+require 'r_proxy/cache_pool'
+require 'r_proxy/usage_manager'
 require 'r_proxy/constants'
 require 'r_proxy/check_snapshot_service'
 require 'r_proxy/http_proxy_parser'
@@ -13,7 +15,6 @@ require 'r_proxy/process_handler'
 
 require 'r_proxy/target_connection'
 require 'r_proxy/connection_handler'
-require 'r_proxy/unbind_service'
 
 require 'r_proxy/callback_connection'
 require 'r_proxy/http_post_template'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: r_proxy
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - Nick An
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2020-05-08 00:00:00.000000000 Z
+date: 2020-05-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: eventmachine
@@ -68,6 +68,7 @@ files:
 - bin/setup
 - example.rb
 - lib/r_proxy.rb
+- lib/r_proxy/cache_pool.rb
 - lib/r_proxy/callback_connection.rb
 - lib/r_proxy/callback_service.rb
 - lib/r_proxy/check_snapshot_service.rb
@@ -81,7 +82,7 @@ files:
 - lib/r_proxy/proxy_server.rb
 - lib/r_proxy/redis_service.rb
 - lib/r_proxy/target_connection.rb
-- lib/r_proxy/unbind_service.rb
+- lib/r_proxy/usage_manager.rb
 - lib/r_proxy/version.rb
 - r_proxy.gemspec
 homepage: https://github.com/nickoan/r_proxy

data/lib/r_proxy/unbind_service.rb

@@ -1,24 +0,0 @@
-module RProxy
-  class UnbindService
-
-    def initialize(config, redis)
-      @config = config
-      @cb_url = config.callback_url
-      @redis = redis
-      @usage_threshold = @config.usage_threshold
-      @snapshot_expire_in = 15 * 60 # 15 min
-    end
-
-    def call(user, pass, usage)
-
-      return if user.nil? || pass.nil? || usage.nil?
-
-      key = proxy_key(user, pass)
-      @redis.decrby(key, usage)
-    end
-
-    def proxy_key(user, pass)
-      "proxy:#{user}-#{pass}"
-    end
-  end
-end