girl 0.86.0 → 0.90.0

data/lib/girl/proxyd.rb

@@ -13,61 +13,41 @@ module Girl
   class Proxyd
 
     def initialize( config_path = nil )
-      if config_path
-        unless File.exist?( config_path )
-          raise "not found config file #{ config_path }"
-        end
-
+      if config_path then
+        raise "not found config file #{ config_path }" unless File.exist?( config_path )
         conf = JSON.parse( IO.binread( config_path ), symbolize_names: true )
         proxyd_port = conf[ :proxyd_port ]
-        proxyd_tmp_dir = conf[ :proxyd_tmp_dir ]
         worker_count = conf[ :worker_count ]
       end
 
-      unless proxyd_port
+      unless proxyd_port then
         proxyd_port = 6060
       end
 
-      unless proxyd_tmp_dir
-        proxyd_tmp_dir = '/tmp/girl.proxyd'
-      end
-
-      unless File.exist?( proxyd_tmp_dir )
-        Dir.mkdir( proxyd_tmp_dir )
-      end
-
-      dst_chunk_dir = File.join( proxyd_tmp_dir, 'dst.chunk' )
-
-      unless Dir.exist?( dst_chunk_dir )
-        Dir.mkdir( dst_chunk_dir )
-      end
-
-      tund_chunk_dir = File.join( proxyd_tmp_dir, 'tund.chunk' )
-
-      unless Dir.exist?( tund_chunk_dir )
-        Dir.mkdir( tund_chunk_dir )
-      end
-
       nprocessors = Etc.nprocessors
 
-      if worker_count.nil? || worker_count <= 0 || worker_count > nprocessors
+      if worker_count.nil? || worker_count <= 0 || worker_count > nprocessors then
         worker_count = nprocessors
       end
 
       title = "girl proxyd #{ Girl::VERSION }"
       puts title
       puts "proxyd port #{ proxyd_port }"
-      puts "dst chunk dir #{ dst_chunk_dir }"
-      puts "tund chunk dir #{ tund_chunk_dir }"
       puts "worker count #{ worker_count }"
 
+      len = CONSTS.map{ | name | name.size }.max
+
+      CONSTS.each do | name |
+        puts "#{ name.gsub( '_', ' ' ).ljust( len ) } #{ Girl.const_get( name ) }"
+      end
+
       $0 = title
       workers = []
 
       worker_count.times do | i |
         workers << fork do
           $0 = 'girl proxyd worker'
-          worker = Girl::ProxydWorker.new( proxyd_port, dst_chunk_dir, tund_chunk_dir )
+          worker = Girl::ProxydWorker.new( proxyd_port )
 
           Signal.trap( :TERM ) do
             puts "w#{ i } exit"

data/lib/girl/proxyd_worker.rb

@@ -4,17 +4,16 @@ module Girl
     ##
     # initialize
     #
-    def initialize( proxyd_port, dst_chunk_dir, tund_chunk_dir )
-      @dst_chunk_dir = dst_chunk_dir
-      @tund_chunk_dir = tund_chunk_dir
+    def initialize( proxyd_port )
       @custom = Girl::ProxydCustom.new
       @mutex = Mutex.new
       @reads = []
       @writes = []
-      @roles = {}           # sock => :dotr / :proxyd / :dst / :tund
-      @dst_infos = {}       # dst => {}
-      @tunds = {}           # port => tund
+      @roles = {}           # sock => :dotr / :proxyd / :dst / :tund / :tcpd / :streamd
       @tund_infos = {}      # tund => {}
+      @tcpd_infos = {}      # tcpd => {}
+      @dst_infos = {}       # dst => {}
+      @streamd_infos = {}   # streamd => {}
       @tunneling_tunds = {} # tunneling_addr => tund
       @resolv_caches = {}   # domain => [ ip, created_at ]
 
@@ -30,34 +29,40 @@ module Girl
     def looping
       puts "p#{ Process.pid } #{ Time.new } looping"
       loop_check_expire
-      loop_check_status
+      loop_check_resume
 
       loop do
         rs, ws = IO.select( @reads, @writes )
 
         @mutex.synchronize do
-          # 先写，再读
-          ws.each do | sock |
-            case @roles[ sock ]
-            when :proxyd
-              write_proxyd( sock )
-            when :dst
-              write_dst( sock )
-            when :tund
-              write_tund( sock )
-            end
-          end
-
+          # 先读，再写，避免打上关闭标记后读到
           rs.each do | sock |
             case @roles[ sock ]
-            when :dotr
+            when :dotr then
               read_dotr( sock )
-            when :proxyd
+            when :proxyd then
               read_proxyd( sock )
-            when :dst
-              read_dst( sock )
-            when :tund
+            when :tund then
               read_tund( sock )
+            when :tcpd then
+              read_tcpd( sock )
+            when :dst then
+              read_dst( sock )
+            when :streamd then
+              read_streamd( sock )
+            end
+          end
+
+          ws.each do | sock |
+            case @roles[ sock ]
+            when :proxyd then
+              write_proxyd( sock )
+            when :tund then
+              write_tund( sock )
+            when :dst then
+              write_dst( sock )
+            when :streamd then
+              write_streamd( sock )
             end
           end
         end
@@ -74,7 +79,7 @@ module Girl
       data = [ 0, TUND_FIN ].pack( 'Q>C' )
 
       @tund_infos.each do | tund, tund_info |
-        if !tund.closed? && tund_info[ :tun_addr ]
+        if !tund.closed? && tund_info[ :tun_addr ] then
           # puts "debug1 send tund fin"
           tund.sendmsg( data, 0, tund_info[ :tun_addr ] )
         end
@@ -87,153 +92,172 @@ module Girl
     private
 
     ##
-    # loop check expire
+    # add proxyd ctlmsg
     #
-    def loop_check_expire
-      Thread.new do
-        loop do
-          sleep CHECK_EXPIRE_INTERVAL
-
-          @mutex.synchronize do
-            need_trigger = false
-            now = Time.new
+    def add_proxyd_ctlmsg_tund_port( tund_info )
+      data = [ 0, TUND_PORT, tund_info[ :port ], tund_info[ :tcpd_port ] ].pack( 'Q>Cnn' )
+      @proxyd_info[ :ctlmsgs ] << [ data, tund_info[ :tun_addr ] ]
+      add_write( @proxyd )
+    end
 
-            @tund_infos.each do | tund, tund_info |
-              unless tund.closed?
-                is_expired = tund_info[ :last_recv_at ] ? ( now - tund_info[ :last_recv_at ] > EXPIRE_AFTER ) : ( now - tund_info[ :created_at ] > EXPIRE_NEW )
-
-                if is_expired
-                  puts "p#{ Process.pid } #{ Time.new } expire tund #{ tund_info[ :port ] }"
-                  set_is_closing( tund )
-                else
-                  data = [ 0, HEARTBEAT, rand( 128 ) ].pack( 'Q>CC' )
-                  # puts "debug1 #{ Time.new } #{ tund_info[ :port ] } heartbeat"
-                  add_tund_ctlmsg( tund, data )
-
-                  tund_info[ :dst_exts ].each do | dst_local_port, dst_ext |
-                    if dst_ext[ :dst ].closed? && ( now - dst_ext[ :last_continue_at ] > EXPIRE_AFTER )
-                      puts "p#{ Process.pid } #{ Time.new } expire dst ext #{ dst_ext[ :domain_port ] }"
-                      del_dst_ext( tund, dst_local_port )
-                    end
-                  end
-                end
-
-                need_trigger = true
-              end
-            end
+    ##
+    # add ctlmsg
+    #
+    def add_ctlmsg( tund, data )
+      tund_info = @tund_infos[ tund ]
+      tund_info[ :ctlmsgs ] << data
+      add_write( tund )
+    end
 
-            @dst_infos.each do | dst, dst_info |
-              if now - dst_info[ :last_continue_at ] > EXPIRE_AFTER
-                puts "p#{ Process.pid } #{ Time.new } expire dst #{ dst_info[ :domain_port ] }"
-                set_is_closing( dst )
-                need_trigger = true
-              end
-            end
+    ##
+    # add read
+    #
+    def add_read( sock, role = nil )
+      unless @reads.include?( sock ) then
+        @reads << sock
 
-            if need_trigger
-              next_tick
-            end
-          end
+        if role then
+          @roles[ sock ] = role
         end
       end
     end
 
     ##
-    # loop check status
+    # add write
     #
-    def loop_check_status
-      Thread.new do
-        loop do
-          sleep CHECK_STATUS_INTERVAL
-
-          if @tunds.any?
-            @mutex.synchronize do
-              need_trigger = false
-
-              @tunds.each do | tund_port, tund |
-                tund_info = @tund_infos[ tund ]
-
-                if tund_info[ :dst_exts ].any?
-                  now = Time.new
-
-                  tund_info[ :dst_exts ].each do | dst_local_port, dst_ext |
-                    if now - dst_ext[ :last_continue_at ] < SEND_STATUS_UNTIL
-                      data = [ 0, DEST_STATUS, dst_local_port, dst_ext[ :relay_pack_id ], dst_ext[ :continue_src_pack_id ] ].pack( 'Q>CnQ>Q>' )
-                      add_tund_ctlmsg( tund, data )
-                      need_trigger = true
-                    end
-                  end
-                end
-
-                if tund_info[ :paused ] && ( tund_info[ :dst_exts ].map{ | _, dst_ext | dst_ext[ :wmems ].size }.sum < RESUME_BELOW )
-                  puts "p#{ Process.pid } #{ Time.new } resume tund"
-                  tund_info[ :paused ] = false
-                  add_write( tund )
-                  need_trigger = true
-                end
-              end
-
-              if need_trigger
-                next_tick
-              end
-            end
-          end
-        end
+    def add_write( sock )
+      unless @writes.include?( sock ) then
+        @writes << sock
       end
     end
 
     ##
-    # resolve domain
+    # close dst
     #
-    def resolve_domain( tund, src_id, destination_domain_port )
-      resolv_cache = @resolv_caches[ destination_domain_port ]
+    def close_dst( dst )
+      # puts "debug1 close dst"
+      close_sock( dst )
+      del_dst_info( dst )
+    end
 
-      if resolv_cache
-        destination_addr, created_at = resolv_cache
+    ##
+    # close read dst
+    #
+    def close_read_dst( dst )
+      return if dst.closed?
+      # puts "debug1 close read dst"
+      dst.close_read
+      @reads.delete( dst )
+
+      if dst.closed? then
+        # puts "debug1 delete dst info"
+        @roles.delete( dst )
+        dst_info = del_dst_info( dst )
+      else
+        dst_info = @dst_infos[ dst ]
+      end
 
-        if Time.new - created_at < RESOLV_CACHE_EXPIRE
-          # puts "debug1 #{ destination_domain_port } hit resolv cache #{ Addrinfo.new( destination_addr ).inspect }"
-          deal_with_destination_addr( tund, src_id, destination_addr, destination_domain_port )
-          return
-        end
+      dst_info[ :paused ] = false
+      dst_info
+    end
 
-        # puts "debug1 expire #{ destination_domain_port } resolv cache"
-        @resolv_caches.delete( destination_domain_port )
+    ##
+    # close read streamd
+    #
+    def close_read_streamd( streamd )
+      return if streamd.closed?
+      # puts "debug1 close read streamd"
+      streamd.close_read
+      @reads.delete( streamd )
+
+      if streamd.closed? then
+        # puts "debug1 delete streamd info"
+        @roles.delete( streamd )
+        streamd_info = @streamd_infos.delete( streamd )
+      else
+        streamd_info = @streamd_infos[ streamd ]
       end
 
-      Thread.new do
-        colon_idx = destination_domain_port.rindex( ':' )
+      streamd_info
+    end
 
-        if colon_idx
-          destination_domain = destination_domain_port[ 0...colon_idx ]
-          destination_port = destination_domain_port[ ( colon_idx + 1 )..-1 ].to_i
+    ##
+    # close sock
+    #
+    def close_sock( sock )
+      sock.close
+      @reads.delete( sock )
+      @writes.delete( sock )
+      @roles.delete( sock )
+    end
 
-          begin
-            destination_addr = Socket.sockaddr_in( destination_port, destination_domain )
-          rescue Exception => e
-            puts "p#{ Process.pid } #{ Time.new } sockaddr in #{ destination_domain_port } #{ e.class }"
-          end
-        end
+    ##
+    # close streamd
+    #
+    def close_streamd( streamd )
+      # puts "debug1 close streamd"
+      close_sock( streamd )
+      @streamd_infos.delete( streamd )
+    end
 
-        @mutex.synchronize do
-          if destination_addr
-            # puts "debug1 resolved #{ destination_domain_port } #{ Addrinfo.new( destination_addr ).inspect }"
-            @resolv_caches[ destination_domain_port ] = [ destination_addr, Time.new ]
+    ##
+    # close tund
+    #
+    def close_tund( tund )
+      # puts "debug1 close tund"
+      close_sock( tund )
+      tund_info = @tund_infos.delete( tund )
+      tcpd = tund_info[ :tcpd ]
+      close_sock( tcpd )
+      @tcpd_infos.delete( tcpd )
+      tund_info[ :dsts ].each{ | _, dst | set_dst_closing( dst ) }
+      @tunneling_tunds.delete( tund_info[ :tun_addr ] )
+    end
 
-            unless tund.closed?
-              if deal_with_destination_addr( tund, src_id, destination_addr, destination_domain_port )
-                next_tick
-              end
-            end
-          end
-        end
+    ##
+    # close write dst
+    #
+    def close_write_dst( dst )
+      return if dst.closed?
+      # puts "debug1 close write dst"
+      dst.close_write
+      @writes.delete( dst )
+
+      if dst.closed? then
+        # puts "debug1 delete dst info"
+        @roles.delete( dst )
+        dst_info = del_dst_info( dst )
+      else
+        dst_info = @dst_infos[ dst ]
+      end
+
+      dst_info
+    end
+
+    ##
+    # close write streamd
+    #
+    def close_write_streamd( streamd )
+      return if streamd.closed?
+      # puts "debug1 close write streamd"
+      streamd.close_write
+      @writes.delete( streamd )
+
+      if streamd.closed? then
+        # puts "debug1 delete streamd info"
+        @roles.delete( streamd )
+        streamd_info = @streamd_infos.delete( streamd )
+      else
+        streamd_info = @streamd_infos[ streamd ]
       end
+
+      streamd_info
     end
 
     ##
     # deal with destination addr
     #
-    def deal_with_destination_addr( tund, src_id, destination_addr, destination_domain_port )
+    def deal_with_destination_addr( tund, src_id, destination_addr, domain_port )
       dst = Socket.new( Addrinfo.new( destination_addr ).ipv4? ? Socket::AF_INET : Socket::AF_INET6, Socket::SOCK_STREAM, 0 )
       dst.setsockopt( Socket::SOL_TCP, Socket::TCP_NODELAY, 1 )
 
@@ -245,173 +269,191 @@ module Girl
         return false
       end
 
-      local_port = dst.local_address.ip_port
+      dst_id = dst.local_address.ip_port
 
       @dst_infos[ dst ] = {
-        local_port: local_port,               # 本地端口
-        tund: tund,                           # 对应tund
-        domain_port: destination_domain_port, # 域名和端口
-        biggest_pack_id: 0,                   # 最大包号码
-        wbuff: '',                            # 写前
-        cache: '',                            # 块读出缓存
-        chunks: [],                           # 块队列，写前达到块大小时结一个块 filename
-        spring: 0,                            # 块后缀，结块时，如果块队列不为空，则自增，为空，则置为0
-        last_continue_at: Time.new,           # 上一次发生流量的时间
-        is_closing: false                     # 是否准备关闭
+        id: dst_id,               # id
+        tund: tund,               # 对应tund
+        domain_port: domain_port, # 域名和端口
+        rbuff: '',                # 对应的streamd没准备好，暂存读到的流量
+        streamd: nil,             # 对应的streamd
+        wbuff: '',                # 从streamd读到的流量
+        src_id: src_id,           # 近端src id
+        created_at: Time.new,     # 创建时间
+        last_recv_at: nil,        # 上一次收到新流量（由streamd收到）的时间
+        last_sent_at: nil,        # 上一次发出流量（由streamd发出）的时间
+        paused: false,            # 是否已暂停
+        closing: false,           # 准备关闭
+        closing_read: false,      # 准备关闭读
+        closing_write: false      # 准备关闭写
       }
+
       add_read( dst, :dst )
 
       tund_info = @tund_infos[ tund ]
-      tund_info[ :dst_local_ports ][ src_id ] = local_port
-      tund_info[ :dst_exts ][ local_port ] = {
-        dst: dst,                             # dst
-        src_id: src_id,                       # 近端src id
-        domain_port: destination_domain_port, # 域名和端口
-        wmems: {},                            # 写后 pack_id => data
-        send_ats: {},                         # 上一次发出时间 pack_id => send_at
-        relay_pack_id: 0,                     # 转发到几
-        continue_src_pack_id: 0,              # 收到几
-        pieces: {},                           # 跳号包 src_pack_id => data
-        is_src_closed: false,                 # src是否已关闭
-        biggest_src_pack_id: 0,               # src最大包号码
-        completed_pack_id: 0,                 # 完成到几（对面收到几）
-        last_continue_at: Time.new            # 上一次发生流量的时间
-      }
-
-      data = [ 0, PAIRED, src_id, local_port ].pack( 'Q>CQ>n' )
-      # puts "debug1 add ctlmsg paired #{ data.inspect }"
-      add_tund_ctlmsg( tund, data )
+      tund_info[ :dst_ids ][ src_id ] = dst_id
+      tund_info[ :dsts ][ dst_id ] = dst
 
+      data = [ 0, PAIRED, src_id, dst_id ].pack( 'Q>CQ>n' )
+      # puts "debug1 add ctlmsg paired #{ src_id } #{ dst_id }"
+      add_ctlmsg( tund, data )
       true
     end
 
     ##
-    # new a proxyd
+    # del dst info
     #
-    def new_a_proxyd( proxyd_port )
-      proxyd = Socket.new( Socket::AF_INET, Socket::SOCK_DGRAM, 0 )
-      proxyd.setsockopt( Socket::SOL_SOCKET, Socket::SO_REUSEPORT, 1 )
-      proxyd.bind( Socket.sockaddr_in( proxyd_port, '0.0.0.0' ) )
+    def del_dst_info( dst )
+      dst_info = @dst_infos.delete( dst )
+      tund = dst_info[ :tund ]
 
-      puts "p#{ Process.pid } #{ Time.new } proxyd bind on #{ proxyd_port }"
-      @proxyd = proxyd
-      @proxyd_ctlmsgs = [] # [ to_addr, data ]
-      add_read( proxyd, :proxyd )
-    end
+      unless tund.closed? then
+        tund_info = @tund_infos[ tund ]
+        tund_info[ :dsts ].delete( dst_info[ :id ] )
+        tund_info[ :dst_ids ].delete( dst_info[ :src_id ] )
+      end
 
-    ##
-    # add proxyd ctlmsg
-    #
-    def add_proxyd_ctlmsg( data, to_addr )
-      @proxyd_ctlmsgs << [ to_addr, data ]
-      add_write( @proxyd )
+      dst_info
     end
 
     ##
-    # add tund ctlmsg
+    # loop check expire
     #
-    def add_tund_ctlmsg( tund, data )
-      tund_info = @tund_infos[ tund ]
-      tund_info[ :ctlmsgs ] << data
-      add_write( tund )
-    end
+    def loop_check_expire
+      Thread.new do
+        loop do
+          sleep CHECK_EXPIRE_INTERVAL
 
-    ##
-    # add tund wbuff
-    #
-    def add_tund_wbuff( tund, dst_local_port, pack_id, data )
-      tund_info = @tund_infos[ tund ]
-      tund_info[ :wbuffs ] << [ dst_local_port, pack_id, data ]
-
-      if tund_info[ :wbuffs ].size >= WBUFFS_LIMIT
-        spring = tund_info[ :chunks ].size > 0 ? ( tund_info[ :spring ] + 1 ) : 0
-        filename = "#{ Process.pid }-#{ tund_info[ :port ] }.#{ spring }"
-        chunk_path = File.join( @tund_chunk_dir, filename )
-        datas = tund_info[ :wbuffs ].map{ | _dst_local_port, _pack_id, _data | [ [ _dst_local_port, _pack_id, _data.bytesize ].pack( 'nQ>n' ), _data ].join }
-
-        begin
-          IO.binwrite( chunk_path, datas.join )
-        rescue Errno::ENOSPC => e
-          puts "p#{ Process.pid } #{ Time.new } #{ e.class }, close tund"
-          set_is_closing( tund )
-          return
-        end
+          @mutex.synchronize do
+            trigger = false
+            now = Time.new
 
-        tund_info[ :chunks ] << filename
-        tund_info[ :spring ] = spring
-        tund_info[ :wbuffs ].clear
-      end
+            @tund_infos.each do | tund, tund_info |
+              last_recv_at = tund_info[ :last_recv_at ] || tund_info[ :created_at ]
+              last_sent_at = tund_info[ :last_sent_at ] || tund_info[ :created_at ]
 
-      add_write( tund )
+              if tund_info[ :dsts ].empty? && ( now - last_recv_at >= EXPIRE_AFTER ) && ( now - last_sent_at >= EXPIRE_AFTER ) then
+                puts "p#{ Process.pid } #{ Time.new } expire tund #{ tund_info[ :port ] }"
+                set_tund_closing( tund )
+                trigger = true
+              end
+            end
+
+            @dst_infos.each do | dst, dst_info |
+              last_recv_at = dst_info[ :last_recv_at ] || dst_info[ :created_at ]
+              last_sent_at = dst_info[ :last_sent_at ] || dst_info[ :created_at ]
+
+              if ( now - last_recv_at >= EXPIRE_AFTER ) && ( now - last_sent_at >= EXPIRE_AFTER ) then
+                puts "p#{ Process.pid } #{ Time.new } expire dst #{ dst_info[ :domain_port ] }"
+                set_dst_closing( dst )
+                trigger = true
+              end
+            end
+
+            next_tick if trigger
+          end
+        end
+      end
     end
 
     ##
-    # add dst wbuff
+    # loop check resume
     #
-    def add_dst_wbuff( dst, data )
-      dst_info = @dst_infos[ dst ]
-      dst_info[ :wbuff ] << data
-
-      if dst_info[ :wbuff ].bytesize >= CHUNK_SIZE
-        spring = dst_info[ :chunks ].size > 0 ? ( dst_info[ :spring ] + 1 ) : 0
-        filename = "#{ Process.pid }-#{ dst_info[ :local_port ] }.#{ spring }"
-        chunk_path = File.join( @dst_chunk_dir, filename )
-
-        begin
-          IO.binwrite( chunk_path, dst_info[ :wbuff ] )
-        rescue Errno::ENOSPC => e
-          puts "p#{ Process.pid } #{ Time.new } #{ e.class }, close dst"
-          set_is_closing( dst )
-          return
-        end
+    def loop_check_resume
+      Thread.new do
+        loop do
+          sleep CHECK_RESUME_INTERVAL
 
-        dst_info[ :chunks ] << filename
-        dst_info[ :spring ] = spring
-        dst_info[ :wbuff ].clear
-      end
+          @mutex.synchronize do
+            trigger = false
 
-      add_write( dst )
+            @dst_infos.select{ | _, dst_info | dst_info[ :paused ] }.each do | dst, dst_info |
+              streamd = dst_info[ :streamd ]
+              streamd_info = @streamd_infos[ streamd ]
+
+              if streamd_info[ :wbuff ].size < RESUME_BELOW then
+                puts "p#{ Process.pid } #{ Time.new } resume dst #{ dst_info[ :domain_port ] }"
+                dst_info[ :paused ] = false
+                add_read( dst )
+                trigger = true
+              end
+            end
+
+            next_tick if trigger
+          end
+        end
+      end
     end
 
     ##
-    # add read
+    # new a proxyd
     #
-    def add_read( sock, role )
-      unless @reads.include?( sock )
-        @reads << sock
-      end
+    def new_a_proxyd( proxyd_port )
+      proxyd = Socket.new( Socket::AF_INET, Socket::SOCK_DGRAM, 0 )
+      proxyd.setsockopt( Socket::SOL_SOCKET, Socket::SO_REUSEPORT, 1 )
+      proxyd.bind( Socket.sockaddr_in( proxyd_port, '0.0.0.0' ) )
+
+      puts "p#{ Process.pid } #{ Time.new } proxyd bind on #{ proxyd_port }"
+      @proxyd = proxyd
+      @proxyd_info = {
+        ctlmsgs: [] # [ ctlmsg, to_addr ]
+      }
 
-      @roles[ sock ] = role
+      add_read( proxyd, :proxyd )
     end
 
     ##
-    # add write
+    # next tick
     #
-    def add_write( sock )
-      if sock && !sock.closed? && !@writes.include?( sock )
-        @writes << sock
-      end
+    def next_tick
+      @dotw.write( '.' )
     end
 
     ##
-    # set is closing
+    # resolve domain
     #
-    def set_is_closing( sock )
-      if sock && !sock.closed?
-        role = @roles[ sock ]
-        # puts "debug1 set #{ role.to_s } is closing"
+    def resolve_domain( tund, src_id, domain_port )
+      resolv_cache = @resolv_caches[ domain_port ]
+
+      if resolv_cache then
+        destination_addr, created_at = resolv_cache
+
+        if Time.new - created_at < RESOLV_CACHE_EXPIRE then
+          # puts "debug1 #{ domain_port } hit resolv cache #{ Addrinfo.new( destination_addr ).inspect }"
+          deal_with_destination_addr( tund, src_id, destination_addr, domain_port )
+          return
+        end
+
+        # puts "debug1 expire #{ domain_port } resolv cache"
+        @resolv_caches.delete( domain_port )
+      end
 
-        case role
-        when :dst
-          dst_info = @dst_infos[ sock ]
-          dst_info[ :is_closing ] = true
-        when :tund
-          tund_info = @tund_infos[ sock ]
-          tund_info[ :is_closing ] = true
+      Thread.new do
+        colon_idx = domain_port.rindex( ':' )
+
+        if colon_idx then
+          destination_domain = domain_port[ 0...colon_idx ]
+          destination_port = domain_port[ ( colon_idx + 1 )..-1 ].to_i
+
+          begin
+            destination_addr = Socket.sockaddr_in( destination_port, destination_domain )
+          rescue Exception => e
+            puts "p#{ Process.pid } #{ Time.new } sockaddr in #{ domain_port } #{ e.class }"
+          end
         end
 
-        @reads.delete( sock )
-        add_write( sock )
+        @mutex.synchronize do
+          if destination_addr then
+            # puts "debug1 resolved #{ domain_port } #{ Addrinfo.new( destination_addr ).inspect }"
+            @resolv_caches[ domain_port ] = [ destination_addr, Time.new ]
+
+            unless tund.closed? then
+              if deal_with_destination_addr( tund, src_id, destination_addr, domain_port ) then
+                next_tick
+              end
+            end
+          end
+        end
       end
     end
 
@@ -420,358 +462,217 @@ module Girl
     #
     def send_data( sock, data, to_addr )
       begin
-        sock.sendmsg( data, 0, to_addr )
+        written = sock.sendmsg_nonblock( data, 0, to_addr )
       rescue IO::WaitWritable, Errno::EINTR
-        return false
+        return :wait
       rescue Errno::EHOSTUNREACH, Errno::ENETUNREACH, Errno::ENETDOWN => e
-        if @roles[ sock ] == :tund
-          puts "#{ Time.new } #{ e.class }, close tund"
-          close_tund( sock )
-          return false
-        end
+        puts "p#{ Process.pid } #{ Time.new } sendmsg #{ e.class }"
+        return :fatal
       end
 
-      true
+      written
     end
 
     ##
-    # close dst
+    # set dst closing
     #
-    def close_dst( dst )
-      # puts "debug1 close dst"
-      close_sock( dst )
-      dst_info = @dst_infos.delete( dst )
-
-      dst_info[ :chunks ].each do | filename |
-        begin
-          File.delete( File.join( @dst_chunk_dir, filename ) )
-        rescue Errno::ENOENT
-        end
-      end
+    def set_dst_closing( dst )
+      return if dst.closed?
+      dst_info = @dst_infos[ dst ]
+      dst_info[ :closing ] = true
+      @reads.delete( dst )
+      add_write( dst )
+    end
 
-      tund = dst_info[ :tund ]
-      return if tund.closed?
+    ##
+    # set dst closing write
+    #
+    def set_dst_closing_write( dst )
+      return if dst.closed?
+      dst_info = @dst_infos[ dst ]
+      dst_info[ :closing_write ] = true
+      add_write( dst )
+    end
 
-      tund_info = @tund_infos[ tund ]
-      local_port = dst_info[ :local_port ]
-      dst_ext = tund_info[ :dst_exts ][ local_port ]
-      return unless dst_ext
-
-      if dst_ext[ :is_src_closed ]
-        # puts "debug1 4-3. after close dst -> src closed ? yes -> del dst ext -> send fin2"
-        del_dst_ext( tund, local_port )
-        data = [ 0, FIN2, local_port ].pack( 'Q>Cn' )
-        add_tund_ctlmsg( tund, data )
-      else
-        # puts "debug1 3-1. after close dst -> src closed ? no -> send fin1"
-        data = [ 0, FIN1, local_port, dst_info[ :biggest_pack_id ], dst_ext[ :continue_src_pack_id ] ].pack( 'Q>CnQ>Q>' )
-        add_tund_ctlmsg( tund, data )
-      end
+    ##
+    # set streamd closing
+    #
+    def set_streamd_closing( streamd )
+      return if streamd.closed?
+      streamd_info = @streamd_infos[ streamd ]
+      streamd_info[ :closing ] = true
+      @reads.delete( streamd )
+      add_write( streamd )
     end
 
     ##
-    # close tun
+    # set streamd closing write
     #
-    def close_tund( tund )
-      # puts "debug1 close tund"
-      close_sock( tund )
-
-      tund_info = @tund_infos.delete( tund )
-      tund_info[ :chunks ].each do | filename |
-        begin
-          File.delete( File.join( @tund_chunk_dir, filename ) )
-        rescue Errno::ENOENT
-        end
-      end
-
-      tund_info[ :dst_exts ].each{ | _, dst_ext | set_is_closing( dst_ext[ :dst ] ) }
-      @tunneling_tunds.delete( tund_info[ :tun_addr ] )
-      @tunds.delete( tund_info[ :port ] )
+    def set_streamd_closing_write( streamd )
+      return if streamd.closed?
+      streamd_info = @streamd_infos[ streamd ]
+      streamd_info[ :closing_write ] = true
+      add_write( streamd )
     end
 
     ##
-    # close sock
+    # set tund is closing
     #
-    def close_sock( sock )
-      sock.close
-      @reads.delete( sock )
-      @writes.delete( sock )
-      @roles.delete( sock )
+    def set_tund_closing( tund )
+      return if tund.closed?
+      tund_info = @tund_infos[ tund ]
+      tund_info[ :closing ] = true
+      @reads.delete( tund )
+      add_write( tund )
     end
 
     ##
-    # del dst ext
+    # read dotr
     #
-    def del_dst_ext( tund, dst_local_port )
-      tund_info = @tund_infos[ tund ]
-      dst_ext = tund_info[ :dst_exts ].delete( dst_local_port )
-
-      if dst_ext
-        tund_info[ :dst_local_ports ].delete( dst_ext[ :src_id ] )
-      end
+    def read_dotr( dotr )
+      dotr.read( 1 )
     end
 
     ##
-    # release wmems
+    # read proxyd
     #
-    def release_wmems( dst_ext, completed_pack_id )
-      if completed_pack_id > dst_ext[ :completed_pack_id ]
-        # puts "debug2 update completed pack #{ completed_pack_id }"
+    def read_proxyd( proxyd )
+      data, addrinfo, rflags, *controls = proxyd.recvmsg
+      from_addr = addrinfo.to_sockaddr
 
-        pack_ids = dst_ext[ :wmems ].keys.select { | pack_id | pack_id <= completed_pack_id }
+      if @tunneling_tunds.include?( from_addr ) then
+        tund = @tunneling_tunds[ from_addr ]
+        tund_info = @tund_infos[ tund ]
+        puts "p#{ Process.pid } #{ Time.new } resend tund port #{ tund_info[ :port ] }, #{ tund_info[ :stream_port ] }"
+        add_proxyd_ctlmsg_tund_port( tund_info )
+        return
+      end
 
-        pack_ids.each do | pack_id |
-          dst_ext[ :wmems ].delete( pack_id )
-          dst_ext[ :send_ats ].delete( pack_id )
-        end
+      result = @custom.check( data, addrinfo )
 
-        dst_ext[ :completed_pack_id ] = completed_pack_id
+      if result != :success then
+        puts "p#{ Process.pid } #{ Time.new } #{ result }"
+        return
       end
-    end
 
-    ##
-    # next tick
-    #
-    def next_tick
-      @dotw.write( '.' )
-    end
-
-    ##
-    # write proxyd
-    #
-    def write_proxyd( proxyd )
-      while @proxyd_ctlmsgs.any?
-        to_addr, data = @proxyd_ctlmsgs.first
+      tund = Socket.new( Socket::AF_INET, Socket::SOCK_DGRAM, 0 )
+      tund.bind( Socket.sockaddr_in( 0, '0.0.0.0' ) )
+      tund_port = tund.local_address.ip_port
+      add_read( tund, :tund )
 
-        unless send_data( proxyd, data, to_addr )
-          return
-        end
+      tcpd = Socket.new( Socket::AF_INET, Socket::SOCK_STREAM, 0 )
+      tcpd.setsockopt( Socket::SOL_TCP, Socket::TCP_NODELAY, 1 ) if RUBY_PLATFORM.include?( 'linux' )
+      tcpd.bind( Socket.sockaddr_in( 0, '0.0.0.0' ) )
+      tcpd_port = tcpd.local_address.ip_port
+      tcpd.listen( 127 )
+      add_read( tcpd, :tcpd )
+
+      tund_info = {
+        port: tund_port,      # 端口
+        tcpd: tcpd,           # 对应的tcpd
+        tcpd_port: tcpd_port, # tcpd端口
+        ctlmsgs: [],          # [ ctlmsg, to_addr ]
+        tun_addr: from_addr,  # tun地址
+        dsts: {},             # dst_id => dst
+        dst_ids: {},          # src_id => dst_id
+        created_at: Time.new, # 创建时间
+        last_recv_at: nil,    # 上一次收到流量的时间
+        last_sent_at: nil,    # 上一次发出流量的时间
+        closing: false,       # 准备关闭
+        closing_read: false,  # 准备关闭读
+        closing_write: false, # 准备关闭写
+        changed_tun_addr: nil # 记录到和tun addr不符的来源地址
+      }
 
-        @proxyd_ctlmsgs.shift
-      end
+      @tunneling_tunds[ from_addr ] = tund
+      @tund_infos[ tund ] = tund_info
+      @tcpd_infos[ tcpd ] = {
+        tund: tund
+      }
 
-      @writes.delete( proxyd )
+      puts "p#{ Process.pid } #{ Time.new } a new tunnel #{ addrinfo.ip_unpack.inspect } - #{ tund_port }, #{ tcpd_port }, #{ @tund_infos.size } tunds"
+      add_proxyd_ctlmsg_tund_port( tund_info )
     end
 
     ##
-    # write dst
+    # read tund
     #
-    def write_dst( dst )
-      dst_info = @dst_infos[ dst ]
-      from, data = :cache, dst_info[ :cache ]
-
-      if data.empty?
-        if dst_info[ :chunks ].any?
-          path = File.join( @dst_chunk_dir, dst_info[ :chunks ].shift )
-
-          begin
-            data = dst_info[ :cache ] = IO.binread( path )
-            File.delete( path )
-          rescue Errno::ENOENT => e
-            puts "p#{ Process.pid } #{ Time.new } read #{ path } #{ e.class }"
-            close_dst( dst )
-            return
-          end
-        else
-          from, data = :wbuff, dst_info[ :wbuff ]
-        end
-      end
-
-      if data.empty?
-        if dst_info[ :is_closing ]
-          close_dst( dst )
-        else
-          @writes.delete( dst )
-        end
-
-        return
-      end
-
+    def read_tund( tund )
       begin
-        written = dst.write_nonblock( data )
-      rescue IO::WaitWritable, Errno::EINTR
-        return
-      rescue Exception => e
-        # puts "debug1 write dst #{ e.class }"
-        close_dst( dst )
+        data, addrinfo, rflags, *controls = tund.recvmsg_nonblock
+      rescue IO::WaitReadable, Errno::EINTR
+        print 'r'
         return
       end
 
-      # puts "debug2 write dst #{ written }"
-      data = data[ written..-1 ]
-      dst_info[ from ] = data
-      dst_info[ :last_continue_at ] = Time.new
-    end
-
-    ##
-    # write tund
-    #
-    def write_tund( tund )
-      now = Time.new
+      from_addr = addrinfo.to_sockaddr
       tund_info = @tund_infos[ tund ]
 
-      if tund_info[ :is_closing ]
-        if tund_info[ :changed_tun_addr ]
-          data = [ 0, IP_CHANGED ].pack( 'Q>C' )
-          send_data( tund, data, tund_info[ :changed_tun_addr ] )
-        end
-
-        close_tund( tund )
+      if from_addr != tund_info[ :tun_addr ] then
+        # 通常是光猫刷新ip和端口，但万一不是，为了避免脏数据注入，关闭tund
+        puts "p#{ Process.pid } #{ Time.new } from #{ addrinfo.inspect } not match tun addr #{ Addrinfo.new( tund_info[ :tun_addr ] ).inspect }"
+        tund_info[ :changed_tun_addr ] = from_addr
+        set_tund_closing( tund )
         return
       end
 
-      # 传ctlmsg
-      while tund_info[ :ctlmsgs ].any?
-        data = tund_info[ :ctlmsgs ].first
-
-        unless send_data( tund, data, tund_info[ :tun_addr ] )
-          return
-        end
-
-        tund_info[ :ctlmsgs ].shift
-      end
+      tund_info[ :last_recv_at ] = Time.new
+      pack_id = data[ 0, 8 ].unpack( 'Q>' ).first
+      return if pack_id != 0
 
-      # 重传
-      while tund_info[ :resendings ].any?
-        dst_local_port, pack_id = tund_info[ :resendings ].first
-        dst_ext = tund_info[ :dst_exts ][ dst_local_port ]
+      ctl_num = data[ 8 ].unpack( 'C' ).first
 
-        if dst_ext
-          data = dst_ext[ :wmems ][ pack_id ]
+      case ctl_num
+      when A_NEW_SOURCE then
+        src_id = data[ 9, 8 ].unpack( 'Q>' ).first
+        dst_id = tund_info[ :dst_ids ][ src_id ]
 
-          if data
-            unless send_data( tund, data, tund_info[ :tun_addr ] )
-              return
-            end
+        if dst_id then
+          dst = tund_info[ :dsts ][ dst_id ]
+          return unless dst
 
-            dst_ext[ :last_continue_at ] = now
+          if dst.closed? then
+            dst_id = 0
           end
-        end
 
-        tund_info[ :resendings ].shift
-      end
-
-      # 若写后达到上限，暂停取写前
-      if tund_info[ :dst_exts ].map{ | _, dst_ext | dst_ext[ :wmems ].size }.sum >= WMEMS_LIMIT
-        unless tund_info[ :paused ]
-          puts "p#{ Process.pid } #{ Time.new } pause tund #{ tund_info[ :port ] }"
-          tund_info[ :paused ] = true
-        end
-
-        @writes.delete( tund )
-        return
-      end
-
-      # 取写前
-      if tund_info[ :caches ].any?
-        datas = tund_info[ :caches ]
-      elsif tund_info[ :chunks ].any?
-        path = File.join( @tund_chunk_dir, tund_info[ :chunks ].shift )
-
-        begin
-          data = IO.binread( path )
-          File.delete( path )
-        rescue Errno::ENOENT => e
-          puts "p#{ Process.pid } #{ Time.new } read #{ path } #{ e.class }"
-          close_tund( tund )
+          # puts "debug1 resend paired #{ dst_id }"
+          data2 = [ 0, PAIRED, src_id, dst_id ].pack( 'Q>CQ>n' )
+          add_ctlmsg( tund, data2 )
           return
         end
 
-        caches = []
-
-        until data.empty?
-          _dst_local_port, _pack_id, pack_size = data[ 0, 12 ].unpack( 'nQ>n' )
-          caches << [ _dst_local_port, _pack_id, data[ 12, pack_size ] ]
-          data = data[ ( 12 + pack_size )..-1 ]
-        end
-
-        datas = tund_info[ :caches ] = caches
-      elsif tund_info[ :wbuffs ].any?
-        datas = tund_info[ :wbuffs ]
-      else
-        @writes.delete( tund )
-        return
+        data = data[ 17..-1 ]
+        domain_port = @custom.decode( data )
+        # puts "debug1 a new source #{ src_id } #{ domain_port }"
+        resolve_domain( tund, src_id, domain_port )
+      when TUN_FIN then
+        puts "p#{ Process.pid } #{ Time.new } recv tun fin"
+        set_tund_closing( tund )
       end
-
-      while datas.any?
-        dst_local_port, pack_id, data = datas.first
-        dst_ext = tund_info[ :dst_exts ][ dst_local_port ]
-
-        if dst_ext
-          if pack_id <= CONFUSE_UNTIL
-            data = @custom.encode( data )
-            # puts "debug1 encoded pack #{ pack_id }"
-          end
-
-          data = [ [ pack_id, dst_local_port ].pack( 'Q>n' ), data ].join
-
-          unless send_data( tund, data, tund_info[ :tun_addr ] )
-            return
-          end
-
-          # puts "debug2 written pack #{ pack_id }"
-          dst_ext[ :relay_pack_id ] = pack_id
-          dst_ext[ :wmems ][ pack_id ] = data
-          dst_ext[ :send_ats ][ pack_id ] = now
-          dst_ext[ :last_continue_at ] = now
-        end
-
-        datas.shift
-      end
-    end
-
-    ##
-    # read dotr
-    #
-    def read_dotr( dotr )
-      dotr.read( 1 )
     end
 
     ##
-    # read proxyd
+    # read tcpd
     #
-    def read_proxyd( proxyd )
-      data, addrinfo, rflags, *controls = proxyd.recvmsg
-      from_addr = addrinfo.to_sockaddr
-
-      return if @tunneling_tunds.include?( from_addr )
-
-      result = @custom.check( data, addrinfo )
-
-      if result != :success
-        puts "p#{ Process.pid } #{ Time.new } #{ result }"
+    def read_tcpd( tcpd )
+      begin
+        streamd, addrinfo = tcpd.accept_nonblock
+      rescue IO::WaitReadable, Errno::EINTR
+        print 'r'
         return
       end
 
-      tund = Socket.new( Socket::AF_INET, Socket::SOCK_DGRAM, 0 )
-      tund.bind( Socket.sockaddr_in( 0, '0.0.0.0' ) )
-      port = tund.local_address.ip_port
+      # puts "debug1 accept a streamd"
+      tcpd_info = @tcpd_infos[ tcpd ]
+      tund = tcpd_info[ :tund ]
 
-      @tunneling_tunds[ from_addr ] = tund
-      @tunds[ port ] = tund
-      @tund_infos[ tund ] = {
-        port: port,           # 端口
-        ctlmsgs: [],          # data
-        wbuffs: [],           # 写前缓存 [ dst_local_port, pack_id, data ]
-        caches: [],           # 块读出缓存 [ dst_local_port, pack_id, data ]
-        chunks: [],           # 块队列 filename
-        spring: 0,            # 块后缀，结块时，如果块队列不为空，则自增，为空，则置为0
-        tun_addr: from_addr,  # tun地址
-        dst_exts: {},         # dst额外信息 dst_local_port => {}
-        dst_local_ports: {},  # src_id => dst_local_port
-        paused: false,        # 是否暂停写
-        resendings: [],       # 重传队列 [ dst_local_port, pack_id ]
-        created_at: Time.new, # 创建时间
-        last_recv_at: nil,    # 上一次收到流量的时间，过期关闭
-        is_closing: false,    # 是否准备关闭
-        changed_tun_addr: nil # 记录到和tun addr不符的来源地址
+      @streamd_infos[ streamd ] = {
+        tund: tund, # 对应tund
+        dst: nil,   # 对应dst
+        wbuff: ''   # 写前，写往近端stream
       }
 
-      add_read( tund, :tund )
-
-      data = [ 0, TUND_PORT, port ].pack( 'Q>Cn' )
-      puts "p#{ Process.pid } #{ Time.new } a new tunnel #{ addrinfo.ip_unpack.inspect } - #{ port }, #{ @tunds.size } tunds"
-      add_proxyd_ctlmsg( data, from_addr )
+      add_read( streamd, :streamd )
     end
 
     ##
@@ -779,211 +680,271 @@ module Girl
     #
     def read_dst( dst )
       begin
-        data = dst.read_nonblock( PACK_SIZE )
+        data = dst.read_nonblock( READ_SIZE )
       rescue IO::WaitReadable, Errno::EINTR
+        print 'r'
         return
       rescue Exception => e
         # puts "debug1 read dst #{ e.class }"
-        set_is_closing( dst )
+        dst_info = close_read_dst( dst )
+        streamd = dst_info[ :streamd ]
+        set_streamd_closing_write( streamd ) if streamd
         return
       end
 
-      # puts "debug2 read dst #{ data.inspect }"
       dst_info = @dst_infos[ dst ]
-      dst_info[ :last_continue_at ] = Time.new
-      tund = dst_info[ :tund ]
+      streamd = dst_info[ :streamd ]
+
+      if streamd then
+        unless streamd.closed? then
+          streamd_info = @streamd_infos[ streamd ]
+          data = @custom.encode( data )
+          # puts "debug2 add streamd.wbuff encoded #{ data.bytesize }"
+          streamd_info[ :wbuff ] << data
+          add_write( streamd )
+
+          if streamd_info[ :wbuff ].bytesize >= WBUFF_LIMIT then
+            puts "p#{ Process.pid } #{ Time.new } pause dst #{ dst_info[ :id ] } #{ dst_info[ :domain_port ] }"
+            dst_info[ :paused ] = true
+            @reads.delete( dst )
+          end
+        end
+      else
+        dst_info[ :rbuff ] << data
 
-      if tund.closed?
-        puts "p#{ Process.pid } #{ Time.new } tund closed, close dst"
-        set_is_closing( dst )
-        return
+        if dst_info[ :rbuff ].bytesize >= WBUFF_LIMIT then
+          # puts "debug1 dst.rbuff full"
+          set_dst_closing( dst )
+        end
       end
-
-      pack_id = dst_info[ :biggest_pack_id ] + 1
-      dst_info[ :biggest_pack_id ] = pack_id
-      add_tund_wbuff( tund, dst_info[ :local_port ], pack_id, data )
     end
 
     ##
-    # read tund
+    # read streamd
     #
-    def read_tund( tund )
-      data, addrinfo, rflags, *controls = tund.recvmsg
-      from_addr = addrinfo.to_sockaddr
-      now = Time.new
-      tund_info = @tund_infos[ tund ]
-
-      if from_addr != tund_info[ :tun_addr ]
-        # 通常是光猫刷新ip（端口也会变），但万一不是，为了避免脏数据注入，关闭tund
-        puts "p#{ Process.pid } #{ Time.new } from #{ addrinfo.inspect } not match tun addr #{ Addrinfo.new( tund_info[ :tun_addr ] ).inspect }"
-        tund_info[ :changed_tun_addr ] = from_addr
-        set_is_closing( tund )
+    def read_streamd( streamd )
+      begin
+        data = streamd.read_nonblock( READ_SIZE )
+      rescue IO::WaitReadable, Errno::EINTR
+        print 'r'
+        return
+      rescue Exception => e
+        # puts "debug1 read streamd #{ e.class }"
+        streamd_info = close_read_streamd( streamd )
+        dst = streamd_info[ :dst ]
+        set_dst_closing_write( dst ) if dst
         return
       end
 
-      tund_info[ :last_recv_at ] = now
-      pack_id = data[ 0, 8 ].unpack( 'Q>' ).first
+      streamd_info = @streamd_infos[ streamd ]
+      dst = streamd_info[ :dst ]
 
-      if pack_id == 0
-        ctl_num = data[ 8 ].unpack( 'C' ).first
+      unless dst then
+        dst_id = data[ 0, 2 ].unpack( 'n' ).first
+        tund = streamd_info[ :tund ]
 
-        case ctl_num
-        when A_NEW_SOURCE
-          src_id = data[ 9, 8 ].unpack( 'Q>' ).first
-          dst_local_port = tund_info[ :dst_local_ports ][ src_id ]
-
-          if dst_local_port
-            dst_ext = tund_info[ :dst_exts ][ dst_local_port ]
-            return unless dst_ext
-
-            if dst_ext[ :dst ].closed?
-              dst_local_port = 0
-            end
+        if tund.closed? then
+          set_streamd_closing( streamd )
+          return
+        end
 
-            # puts "debug1 readd ctlmsg paired #{ dst_local_port }"
-            data2 = [ 0, PAIRED, src_id, dst_local_port ].pack( 'Q>CQ>n' )
-            add_tund_ctlmsg( tund, data2 )
-            return
-          end
+        tund_info = @tund_infos[ tund ]
+        dst = tund_info[ :dsts ][ dst_id ]
 
-          data = data[ 17..-1 ]
-          destination_domain_port = @custom.decode( data )
-          puts "p#{ Process.pid } #{ Time.new } a new source #{ src_id } #{ destination_domain_port }"
-          resolve_domain( tund, src_id, destination_domain_port )
-        when SOURCE_STATUS
-          src_id, relay_src_pack_id, continue_dst_pack_id  = data[ 9, 24 ].unpack( 'Q>Q>Q>' )
+        unless dst then
+          set_streamd_closing( streamd )
+          return
+        end
 
-          dst_local_port = tund_info[ :dst_local_ports ][ src_id ]
-          return unless dst_local_port
+        # puts "debug1 set streamd.dst #{ dst_id }"
+        streamd_info[ :dst ] = dst
+        dst_info = @dst_infos[ dst ]
 
-          dst_ext = tund_info[ :dst_exts ][ dst_local_port ]
-          return unless dst_ext
+        unless dst_info[ :rbuff ].empty? then
+          # puts "debug1 encode and move dst.rbuff to streamd.wbuff"
+          streamd_info[ :wbuff ] << @custom.encode( dst_info[ :rbuff ] )
+        end
 
-          # puts "debug2 got source status"
+        dst_info[ :streamd ] = streamd
+        data = data[ 2..-1 ]
 
-          release_wmems( dst_ext, continue_dst_pack_id )
+        return if data.empty?
+      end
 
-          # 发miss
-          if !dst_ext[ :dst ].closed? && ( dst_ext[ :continue_src_pack_id ] < relay_src_pack_id )
-            ranges = []
-            ignored = false
-            curr_pack_id = dst_ext[ :continue_src_pack_id ] + 1
+      unless dst.closed? then
+        dst_info = @dst_infos[ dst ]
+        data = @custom.decode( data )
+        # puts "debug2 add dst.wbuff decoded #{ data.bytesize }"
+        dst_info[ :wbuff ] << data
+        dst_info[ :last_recv_at ] = Time.new
+        add_write( dst )
+      end
+    end
 
-            dst_ext[ :pieces ].keys.sort.each do | pack_id |
-              if pack_id > curr_pack_id
-                ranges << [ curr_pack_id, pack_id - 1 ]
+    ##
+    # write proxyd
+    #
+    def write_proxyd( proxyd )
+      # 发ctlmsg
+      while @proxyd_info[ :ctlmsgs ].any? do
+        data, to_addr = @proxyd_info[ :ctlmsgs ].first
+        sent = send_data( proxyd, data, to_addr )
 
-                if ranges.size >= MISS_RANGE_LIMIT
-                  puts "p#{ Process.pid } #{ Time.new } break add miss range at #{ pack_id }"
-                  ignored = true
-                  break
-                end
-              end
+        if sent == :wait then
+          puts "p#{ Process.pid } #{ Time.new } wait proxyd send ctlmsg, left #{ @proxyd_info[ :ctlmsgs ].size }"
+          return
+        else
+          @proxyd_info[ :ctlmsgs ].shift
+        end
+      end
 
-              curr_pack_id = pack_id + 1
-            end
+      @writes.delete( proxyd )
+    end
 
-            if !ignored && ( curr_pack_id <= relay_src_pack_id )
-              ranges << [ curr_pack_id, relay_src_pack_id ]
-            end
+    ##
+    # write tund
+    #
+    def write_tund( tund )
+      tund_info = @tund_infos[ tund ]
 
-            # puts "debug1 continue/relay #{ dst_ext[ :continue_src_pack_id ] }/#{ relay_src_pack_id } send MISS #{ ranges.size }"
+      # 处理关闭
+      if tund_info[ :closing ] then
+        if tund_info[ :changed_tun_addr ] then
+          data = [ 0, IP_CHANGED ].pack( 'Q>C' )
+          send_data( tund, data, tund_info[ :changed_tun_addr ] )
+        end
 
-            ranges.each do | pack_id_begin, pack_id_end |
-              data2 = [ 0, MISS, src_id, pack_id_begin, pack_id_end ].pack( 'Q>CQ>Q>Q>' )
-              add_tund_ctlmsg( tund, data2 )
-            end
-          end
-        when MISS
-          return if tund_info[ :resendings ].size >= RESENDING_LIMIT
+        close_tund( tund )
+        return
+      end
 
-          dst_local_port, pack_id_begin, pack_id_end = data[ 9, 18 ].unpack( 'nQ>Q>' )
+      now = Time.new
 
-          dst_ext = tund_info[ :dst_exts ][ dst_local_port ]
-          return unless dst_ext
+      # 发ctlmsg
+      while tund_info[ :ctlmsgs ].any? do
+        data = tund_info[ :ctlmsgs ].first
+        sent = send_data( tund, data, tund_info[ :tun_addr ] )
 
-          ( pack_id_begin..pack_id_end ).each do | pack_id |
-            send_at = dst_ext[ :send_ats ][ pack_id ]
+        if sent == :fatal then
+          close_tund( tund )
+          return
+        elsif sent == :wait then
+          puts "p#{ Process.pid } #{ Time.new } wait tund #{ tund_info[ :port ] } send ctlmsg, left #{ tund_info[ :ctlmsgs ].size }"
+          tund_info[ :last_sent_at ] = now
+          return
+        end
 
-            if send_at
-              break if now - send_at < CHECK_STATUS_INTERVAL
-              tund_info[ :resendings ] << [ dst_local_port, pack_id ]
-            end
-          end
+        tund_info[ :ctlmsgs ].shift
+      end
 
-          add_write( tund )
-        when FIN1
-          src_id, biggest_src_pack_id, continue_dst_pack_id = data[ 9, 24 ].unpack( 'Q>Q>Q>' )
+      tund_info[ :last_sent_at ] = now
+      @writes.delete( tund )
+    end
 
-          dst_local_port = tund_info[ :dst_local_ports ][ src_id ]
-          return unless dst_local_port
+    ##
+    # write dst
+    #
+    def write_dst( dst )
+      return if dst.closed?
+      dst_info = @dst_infos[ dst ]
+      streamd = dst_info[ :streamd ]
 
-          dst_ext = tund_info[ :dst_exts ][ dst_local_port ]
-          return unless dst_ext
+      # 处理关闭
+      if dst_info[ :closing ] then
+        close_dst( dst )
 
-          # puts "debug1 got fin1 #{ src_id } biggest src pack #{ biggest_src_pack_id } completed dst pack #{ continue_dst_pack_id }"
-          dst_ext[ :is_src_closed ] = true
-          dst_ext[ :biggest_src_pack_id ] = biggest_src_pack_id
-          release_wmems( dst_ext, continue_dst_pack_id )
+        if streamd then
+          close_read_streamd( streamd )
+          set_streamd_closing_write( streamd )
+        end
 
-          if biggest_src_pack_id == dst_ext[ :continue_src_pack_id ]
-            # puts "debug1 4-1. tund recv fin1 -> all traffic received ? -> close dst after write"
-            set_is_closing( dst_ext[ :dst ] )
-          end
-        when FIN2
-          src_id = data[ 9, 8 ].unpack( 'Q>' ).first
+        return
+      end
 
-          dst_local_port = tund_info[ :dst_local_ports ][ src_id ]
-          return unless dst_local_port
+      data = dst_info[ :wbuff ]
 
-          # puts "debug1 3-2. tund recv fin2 -> del dst ext"
-          del_dst_ext( tund, dst_local_port )
-        when TUN_FIN
-          puts "p#{ Process.pid } #{ Time.new } recv tun fin"
-          set_is_closing( tund )
+      # 写前为空，处理关闭写
+      if data.empty? then
+        if dst_info[ :closing_write ] then
+          close_write_dst( dst )
+        else
+          @writes.delete( dst )
         end
 
         return
       end
 
-      src_id = data[ 8, 8 ].unpack( 'Q>' ).first
-
-      dst_local_port = tund_info[ :dst_local_ports ][ src_id ]
-      return unless dst_local_port
+      # 写入
+      begin
+        written = dst.write_nonblock( data )
+      rescue IO::WaitWritable, Errno::EINTR
+        print 'w'
+        return
+      rescue Exception => e
+        # puts "debug1 write dst #{ e.class }"
+        close_write_dst( dst )
+        close_read_streamd( streamd ) if streamd
+        return
+      end
 
-      dst_ext = tund_info[ :dst_exts ][ dst_local_port ]
-      return if dst_ext.nil? || dst_ext[ :dst ].closed?
-      return if ( pack_id <= dst_ext[ :continue_src_pack_id ] ) || dst_ext[ :pieces ].include?( pack_id )
+      # puts "debug2 written dst #{ written }"
+      data = data[ written..-1 ]
+      dst_info[ :wbuff ] = data
+    end
 
-      data = data[ 16..-1 ]
-      # puts "debug2 got pack #{ pack_id }"
+    ##
+    # write streamd
+    #
+    def write_streamd( streamd )
+      return if streamd.closed?
+      streamd_info = @streamd_infos[ streamd ]
+      dst = streamd_info[ :dst ]
+
+      # 处理关闭
+      if streamd_info[ :closing ] then
+        close_streamd( streamd )
+
+        if dst then
+          close_read_dst( dst )
+          set_dst_closing_write( dst )
+        end
 
-      if pack_id <= CONFUSE_UNTIL
-        # puts "debug2 #{ data.inspect }"
-        data = @custom.decode( data )
-        # puts "debug1 decoded pack #{ pack_id }"
+        return
       end
 
-      # 放进写前，跳号放碎片缓存
-      if pack_id - dst_ext[ :continue_src_pack_id ] == 1
-        while dst_ext[ :pieces ].include?( pack_id + 1 )
-          data << dst_ext[ :pieces ].delete( pack_id + 1 )
-          pack_id += 1
+      data = streamd_info[ :wbuff ]
+
+      # 写前为空，处理关闭写
+      if data.empty? then
+        if streamd_info[ :closing_write ] then
+          close_write_streamd( streamd )
+        else
+          @writes.delete( streamd )
         end
 
-        dst_ext[ :continue_src_pack_id ] = pack_id
-        dst_ext[ :last_continue_at ] = now
-        add_dst_wbuff( dst_ext[ :dst ], data )
-        # puts "debug2 update continue src pack #{ pack_id }"
+        return
+      end
 
-        # 接到流量，若对面已关闭，且流量正好收全，关闭dst
-        if dst_ext[ :is_src_closed ] && ( pack_id == dst_ext[ :biggest_src_pack_id ] )
-          # puts "debug1 4-2. tund recv traffic -> src closed and all traffic received ? -> close dst after write"
-          set_is_closing( dst_ext[ :dst ] )
-          return
-        end
-      else
-        dst_ext[ :pieces ][ pack_id ] = data
+      # 写入
+      begin
+        written = streamd.write_nonblock( data )
+      rescue IO::WaitWritable, Errno::EINTR
+        print 'w'
+        return
+      rescue Exception => e
+        # puts "debug1 write streamd #{ e.class }"
+        close_write_streamd( streamd )
+        close_read_dst( dst ) if dst
+        return
+      end
+
+      # puts "debug2 written streamd #{ written }"
+      data = data[ written..-1 ]
+      streamd_info[ :wbuff ] = data
+
+      if dst && !dst.closed? then
+        dst_info = @dst_infos[ dst ]
+        dst_info[ :last_sent_at ] = Time.new
       end
     end