girl 0.87.0 → 0.91.0

data/lib/girl/proxyd.rb

@@ -13,23 +13,20 @@ module Girl
   class Proxyd
 
     def initialize( config_path = nil )
-      if config_path
-        unless File.exist?( config_path )
-          raise "not found config file #{ config_path }"
-        end
-
+      if config_path then
+        raise "not found config file #{ config_path }" unless File.exist?( config_path )
         conf = JSON.parse( IO.binread( config_path ), symbolize_names: true )
         proxyd_port = conf[ :proxyd_port ]
         worker_count = conf[ :worker_count ]
       end
 
-      unless proxyd_port
+      unless proxyd_port then
         proxyd_port = 6060
       end
 
       nprocessors = Etc.nprocessors
 
-      if worker_count.nil? || worker_count <= 0 || worker_count > nprocessors
+      if worker_count.nil? || worker_count <= 0 || worker_count > nprocessors then
         worker_count = nprocessors
       end
 
@@ -38,25 +35,9 @@ module Girl
       puts "proxyd port #{ proxyd_port }"
       puts "worker count #{ worker_count }"
 
-      names = %w[
-        PACK_SIZE
-        READ_SIZE
-        WMEMS_LIMIT
-        RESUME_BELOW
-        EXPIRE_NEW
-        EXPIRE_AFTER
-        CHECK_EXPIRE_INTERVAL
-        CHECK_STATUS_INTERVAL
-        SEND_STATUS_UNTIL
-        MULTI_MISS_SIZE
-        MISS_RANGE_LIMIT
-        CONFUSE_UNTIL
-        RESOLV_CACHE_EXPIRE
-      ]
-
-      len = names.map{ | name | name.size }.max
+      len = CONSTS.map{ | name | name.size }.max
 
-      names.each do | name |
+      CONSTS.each do | name |
         puts "#{ name.gsub( '_', ' ' ).ljust( len ) } #{ Girl.const_get( name ) }"
       end
 

data/lib/girl/proxyd_worker.rb

@@ -9,11 +9,11 @@ module Girl
       @mutex = Mutex.new
       @reads = []
       @writes = []
-      @pause_dsts = []
-      @roles = {}           # sock => :dotr / :proxyd / :dst / :tund
-      @dst_infos = {}       # dst => {}
-      @tunds = {}           # port => tund
+      @roles = {}           # sock => :dotr / :proxyd / :dst / :tund / :tcpd / :streamd
       @tund_infos = {}      # tund => {}
+      @tcpd_infos = {}      # tcpd => {}
+      @dst_infos = {}       # dst => {}
+      @streamd_infos = {}   # streamd => {}
       @tunneling_tunds = {} # tunneling_addr => tund
       @resolv_caches = {}   # domain => [ ip, created_at ]
 
@@ -29,32 +29,40 @@ module Girl
     def looping
       puts "p#{ Process.pid } #{ Time.new } looping"
       loop_check_expire
-      loop_check_status
+      loop_check_resume
 
       loop do
         rs, ws = IO.select( @reads, @writes )
 
         @mutex.synchronize do
-          # 先写，再读
-          ws.each do | sock |
-            case @roles[ sock ]
-            when :dst
-              write_dst( sock )
-            when :tund
-              write_tund( sock )
-            end
-          end
-
+          # 先读，再写，避免打上关闭标记后读到
           rs.each do | sock |
             case @roles[ sock ]
-            when :dotr
+            when :dotr then
               read_dotr( sock )
-            when :proxyd
+            when :proxyd then
               read_proxyd( sock )
-            when :dst
-              read_dst( sock )
-            when :tund
+            when :tund then
               read_tund( sock )
+            when :tcpd then
+              read_tcpd( sock )
+            when :dst then
+              read_dst( sock )
+            when :streamd then
+              read_streamd( sock )
+            end
+          end
+
+          ws.each do | sock |
+            case @roles[ sock ]
+            when :proxyd then
+              write_proxyd( sock )
+            when :tund then
+              write_tund( sock )
+            when :dst then
+              write_dst( sock )
+            when :streamd then
+              write_streamd( sock )
             end
           end
         end
@@ -71,7 +79,7 @@ module Girl
       data = [ 0, TUND_FIN ].pack( 'Q>C' )
 
       @tund_infos.each do | tund, tund_info |
-        if !tund.closed? && tund_info[ :tun_addr ]
+        if !tund.closed? && tund_info[ :tun_addr ] then
           # puts "debug1 send tund fin"
           tund.sendmsg( data, 0, tund_info[ :tun_addr ] )
         end
@@ -84,191 +92,166 @@ module Girl
     private
 
     ##
-    # loop check expire
+    # add proxyd ctlmsg
     #
-    def loop_check_expire
-      Thread.new do
-        loop do
-          sleep CHECK_EXPIRE_INTERVAL
-
-          @mutex.synchronize do
-            need_trigger = false
-            now = Time.new
+    def add_proxyd_ctlmsg_tund_port( tund_info )
+      data = [ 0, TUND_PORT, tund_info[ :port ], tund_info[ :tcpd_port ] ].pack( 'Q>Cnn' )
+      @proxyd_info[ :ctlmsgs ] << [ data, tund_info[ :tun_addr ] ]
+      add_write( @proxyd )
+    end
 
-            @tund_infos.each do | tund, tund_info |
-              unless tund.closed?
-                is_expired = tund_info[ :last_recv_at ] ? ( now - tund_info[ :last_recv_at ] > EXPIRE_AFTER ) : ( now - tund_info[ :created_at ] > EXPIRE_NEW )
-
-                if is_expired
-                  puts "p#{ Process.pid } #{ Time.new } expire tund #{ tund_info[ :port ] }"
-                  set_is_closing( tund )
-                  need_trigger = true
-                else
-                  data = [ 0, HEARTBEAT, rand( 128 ) ].pack( 'Q>CC' )
-                  # puts "debug1 #{ Time.new } #{ tund_info[ :port ] } heartbeat"
-                  send_data( tund, data, tund_info[ :tun_addr ] )
-                  del_dst_ids = []
-
-                  tund_info[ :dsts ].each do | dst_id, dst |
-                    if dst.closed?
-                      dst_info = @dst_infos[ dst ]
-
-                      if dst_info && ( now - dst_info[ :last_continue_at ] > EXPIRE_AFTER )
-                        puts "p#{ Process.pid } #{ Time.new } expire dst ext #{ dst_info[ :domain_port ] }"
-                        tund_info[ :wmems ].delete_if { | port_and_pack_id, _ | port_and_pack_id.first == dst_id }
-                        tund_info[ :dst_ids ].delete( dst_info[ :src_id ] )
-                        @dst_infos.delete( dst )
-                        del_dst_ids << dst_id
-                      end
-                    end
-                  end
-
-                  if del_dst_ids.any?
-                    tund_info[ :dsts ].delete_if { | dst_id, _ | del_dst_ids.include?( dst_id ) }
-                  end
-                end
-              end
-            end
+    ##
+    # add ctlmsg
+    #
+    def add_ctlmsg( tund, data )
+      tund_info = @tund_infos[ tund ]
+      tund_info[ :ctlmsgs ] << data
+      add_write( tund )
+    end
 
-            @dst_infos.each do | dst, dst_info |
-              if now - dst_info[ :last_continue_at ] > EXPIRE_AFTER
-                puts "p#{ Process.pid } #{ Time.new } expire dst #{ dst_info[ :domain_port ] }"
-                set_is_closing( dst )
-                need_trigger = true
-              end
-            end
+    ##
+    # add read
+    #
+    def add_read( sock, role = nil )
+      unless @reads.include?( sock ) then
+        @reads << sock
 
-            if need_trigger
-              next_tick
-            end
-          end
+        if role then
+          @roles[ sock ] = role
         end
       end
     end
 
     ##
-    # loop check status
+    # add write
     #
-    def loop_check_status
-      Thread.new do
-        loop do
-          sleep CHECK_STATUS_INTERVAL
-
-          @mutex.synchronize do
-            need_trigger = false
-
-            if @tunds.any?
-              @tunds.each do | tund_port, tund |
-                tund_info = @tund_infos[ tund ]
-
-                if tund_info[ :dsts ].any?
-                  now = Time.new
-
-                  tund_info[ :dsts ].each do | dst_id, dst |
-                    dst_info = @dst_infos[ dst ]
-
-                    if dst_info && ( now - dst_info[ :last_continue_at ] < SEND_STATUS_UNTIL )
-                      data = [ 0, DEST_STATUS, dst_id, dst_info[ :biggest_pack_id ], dst_info[ :continue_src_pack_id ] ].pack( 'Q>CnQ>Q>' )
-                      send_data( tund, data, tund_info[ :tun_addr ] )
-                    end
-                  end
-                end
-              end
-            end
+    def add_write( sock )
+      unless @writes.include?( sock ) then
+        @writes << sock
+      end
+    end
 
-            if @pause_dsts.any?
-              resume_dsts = []
-              ignore_dsts = []
-
-              @pause_dsts.each do | dst |
-                dst_info = @dst_infos[ dst ]
-
-                if dst_info
-                  tund = dst_info[ :tund ]
-
-                  if tund.closed?
-                    ignore_dsts << dst
-                  else
-                    tund_info = @tund_infos[ tund ]
-
-                    if tund_info[ :wmems ].size < RESUME_BELOW
-                      puts "p#{ Process.pid } #{ Time.new } resume dst #{ dst_info[ :domain_port ] }"
-                      resume_dsts << dst
-                    end
-                  end
-                else
-                  ignore_dsts << dst
-                end
-              end
+    ##
+    # close dst
+    #
+    def close_dst( dst )
+      # puts "debug1 close dst"
+      close_sock( dst )
+      del_dst_info( dst )
+    end
 
-              if resume_dsts.any?
-                resume_dsts.each do | dst |
-                  add_read( dst )
-                end
+    ##
+    # close read dst
+    #
+    def close_read_dst( dst )
+      return if dst.closed?
+      # puts "debug1 close read dst"
+      dst.close_read
+      @reads.delete( dst )
 
-                @pause_dsts -= resume_dsts
-                need_trigger = true
-              end
+      if dst.closed? then
+        # puts "debug1 delete dst info"
+        @roles.delete( dst )
+        dst_info = del_dst_info( dst )
+      else
+        dst_info = @dst_infos[ dst ]
+      end
 
-              if ignore_dsts.any?
-                @pause_dsts -= ignore_dsts
-              end
-            end
+      dst_info[ :paused ] = false
+      dst_info
+    end
 
-            if need_trigger
-              next_tick
-            end
-          end
-        end
+    ##
+    # close read streamd
+    #
+    def close_read_streamd( streamd )
+      return if streamd.closed?
+      # puts "debug1 close read streamd"
+      streamd.close_read
+      @reads.delete( streamd )
+
+      if streamd.closed? then
+        # puts "debug1 delete streamd info"
+        @roles.delete( streamd )
+        streamd_info = @streamd_infos.delete( streamd )
+      else
+        streamd_info = @streamd_infos[ streamd ]
       end
+
+      streamd_info
     end
 
     ##
-    # resolve domain
+    # close sock
     #
-    def resolve_domain( tund, src_id, domain_port )
-      resolv_cache = @resolv_caches[ domain_port ]
+    def close_sock( sock )
+      sock.close
+      @reads.delete( sock )
+      @writes.delete( sock )
+      @roles.delete( sock )
+    end
 
-      if resolv_cache
-        destination_addr, created_at = resolv_cache
+    ##
+    # close streamd
+    #
+    def close_streamd( streamd )
+      # puts "debug1 close streamd"
+      close_sock( streamd )
+      @streamd_infos.delete( streamd )
+    end
 
-        if Time.new - created_at < RESOLV_CACHE_EXPIRE
-          # puts "debug1 #{ domain_port } hit resolv cache #{ Addrinfo.new( destination_addr ).inspect }"
-          deal_with_destination_addr( tund, src_id, destination_addr, domain_port )
-          return
-        end
+    ##
+    # close tund
+    #
+    def close_tund( tund )
+      # puts "debug1 close tund"
+      close_sock( tund )
+      tund_info = @tund_infos.delete( tund )
+      tcpd = tund_info[ :tcpd ]
+      close_sock( tcpd )
+      @tcpd_infos.delete( tcpd )
+      tund_info[ :dsts ].each{ | _, dst | set_dst_closing( dst ) }
+      @tunneling_tunds.delete( tund_info[ :tun_addr ] )
+    end
 
-        # puts "debug1 expire #{ domain_port } resolv cache"
-        @resolv_caches.delete( domain_port )
+    ##
+    # close write dst
+    #
+    def close_write_dst( dst )
+      return if dst.closed?
+      # puts "debug1 close write dst"
+      dst.close_write
+      @writes.delete( dst )
+
+      if dst.closed? then
+        # puts "debug1 delete dst info"
+        @roles.delete( dst )
+        dst_info = del_dst_info( dst )
+      else
+        dst_info = @dst_infos[ dst ]
       end
 
-      Thread.new do
-        colon_idx = domain_port.rindex( ':' )
-
-        if colon_idx
-          destination_domain = domain_port[ 0...colon_idx ]
-          destination_port = domain_port[ ( colon_idx + 1 )..-1 ].to_i
-
-          begin
-            destination_addr = Socket.sockaddr_in( destination_port, destination_domain )
-          rescue Exception => e
-            puts "p#{ Process.pid } #{ Time.new } sockaddr in #{ domain_port } #{ e.class }"
-          end
-        end
-
-        @mutex.synchronize do
-          if destination_addr
-            # puts "debug1 resolved #{ domain_port } #{ Addrinfo.new( destination_addr ).inspect }"
-            @resolv_caches[ domain_port ] = [ destination_addr, Time.new ]
+      dst_info
+    end
 
-            unless tund.closed?
-              if deal_with_destination_addr( tund, src_id, destination_addr, domain_port )
-                next_tick
-              end
-            end
-          end
-        end
+    ##
+    # close write streamd
+    #
+    def close_write_streamd( streamd )
+      return if streamd.closed?
+      # puts "debug1 close write streamd"
+      streamd.close_write
+      @writes.delete( streamd )
+
+      if streamd.closed? then
+        # puts "debug1 delete streamd info"
+        @roles.delete( streamd )
+        streamd_info = @streamd_infos.delete( streamd )
+      else
+        streamd_info = @streamd_infos[ streamd ]
       end
+
+      streamd_info
     end
 
     ##
@@ -282,27 +265,27 @@ module Girl
         dst.connect_nonblock( destination_addr )
       rescue IO::WaitWritable
       rescue Exception => e
-        puts "p#{ Process.pid } #{ Time.new } connect destination #{ e.class }"
+        puts "p#{ Process.pid } #{ Time.new } connect destination #{ domain_port } #{ e.class }"
         return false
       end
 
       dst_id = dst.local_address.ip_port
 
       @dst_infos[ dst ] = {
-        id: dst_id,                 # id
-        tund: tund,                 # 对应tund
-        domain_port: domain_port,   # 域名和端口
-        biggest_pack_id: 0,         # 最大包号码
-        wbuff: '',                  # 写前
-        src_id: src_id,             # 近端src id
-        send_ats: {},               # 上一次发出时间 pack_id => send_at
-        continue_src_pack_id: 0,    # 收到几
-        pieces: {},                 # 跳号包 src_pack_id => data
-        is_src_closed: false,       # src是否已关闭
-        biggest_src_pack_id: 0,     # src最大包号码
-        completed_pack_id: 0,       # 完成到几（对面收到几）
-        last_continue_at: Time.new, # 上一次发生流量的时间
-        is_closing: false           # 是否准备关闭
+        id: dst_id,               # id
+        tund: tund,               # 对应tund
+        domain_port: domain_port, # 目的地和端口
+        rbuff: '',                # 对应的streamd没准备好，暂存读到的流量
+        streamd: nil,             # 对应的streamd
+        wbuff: '',                # 从streamd读到的流量
+        src_id: src_id,           # 近端src id
+        created_at: Time.new,     # 创建时间
+        last_recv_at: nil,        # 上一次收到新流量（由streamd收到）的时间
+        last_sent_at: nil,        # 上一次发出流量（由streamd发出）的时间
+        paused: false,            # 是否已暂停读
+        closing: false,           # 准备关闭
+        closing_read: false,      # 准备关闭读
+        closing_write: false      # 准备关闭写
       }
 
       add_read( dst, :dst )
@@ -312,124 +295,177 @@ module Girl
       tund_info[ :dsts ][ dst_id ] = dst
 
       data = [ 0, PAIRED, src_id, dst_id ].pack( 'Q>CQ>n' )
-      # puts "debug1 send paired #{ data.inspect }"
-      send_data( tund, data, tund_info[ :tun_addr ] )
-
+      # puts "debug1 add ctlmsg paired #{ src_id } #{ dst_id }"
+      add_ctlmsg( tund, data )
       true
     end
 
     ##
-    # new a proxyd
+    # del dst info
     #
-    def new_a_proxyd( proxyd_port )
-      proxyd = Socket.new( Socket::AF_INET, Socket::SOCK_DGRAM, 0 )
-      proxyd.setsockopt( Socket::SOL_SOCKET, Socket::SO_REUSEPORT, 1 )
-      proxyd.bind( Socket.sockaddr_in( proxyd_port, '0.0.0.0' ) )
+    def del_dst_info( dst )
+      dst_info = @dst_infos.delete( dst )
+      tund = dst_info[ :tund ]
 
-      puts "p#{ Process.pid } #{ Time.new } proxyd bind on #{ proxyd_port }"
-      @proxyd = proxyd
-      add_read( proxyd, :proxyd )
+      unless tund.closed? then
+        tund_info = @tund_infos[ tund ]
+        tund_info[ :dsts ].delete( dst_info[ :id ] )
+        tund_info[ :dst_ids ].delete( dst_info[ :src_id ] )
+      end
+
+      dst_info
     end
 
     ##
-    # add read
+    # loop check expire
     #
-    def add_read( sock, role = nil )
-      if sock && !sock.closed? && !@reads.include?( sock )
-        @reads << sock
+    def loop_check_expire
+      Thread.new do
+        loop do
+          sleep CHECK_EXPIRE_INTERVAL
 
-        if role
-          @roles[ sock ] = role
+          @mutex.synchronize do
+            trigger = false
+            now = Time.new
+
+            @tund_infos.each do | tund, tund_info |
+              last_recv_at = tund_info[ :last_recv_at ] || tund_info[ :created_at ]
+              last_sent_at = tund_info[ :last_sent_at ] || tund_info[ :created_at ]
+
+              if tund_info[ :dsts ].empty? && ( now - last_recv_at >= EXPIRE_AFTER ) && ( now - last_sent_at >= EXPIRE_AFTER ) then
+                puts "p#{ Process.pid } #{ Time.new } expire tund #{ tund_info[ :port ] }"
+                set_tund_closing( tund )
+                trigger = true
+              end
+            end
+
+            @dst_infos.each do | dst, dst_info |
+              last_recv_at = dst_info[ :last_recv_at ] || dst_info[ :created_at ]
+              last_sent_at = dst_info[ :last_sent_at ] || dst_info[ :created_at ]
+
+              if ( now - last_recv_at >= EXPIRE_AFTER ) && ( now - last_sent_at >= EXPIRE_AFTER ) then
+                puts "p#{ Process.pid } #{ Time.new } expire dst #{ dst_info[ :domain_port ] }"
+                set_dst_closing( dst )
+                trigger = true
+              end
+            end
+
+            next_tick if trigger
+          end
         end
       end
     end
 
     ##
-    # add write
+    # loop check resume
     #
-    def add_write( sock )
-      if sock && !sock.closed? && !@writes.include?( sock )
-        @writes << sock
+    def loop_check_resume
+      Thread.new do
+        loop do
+          sleep CHECK_RESUME_INTERVAL
+
+          @mutex.synchronize do
+            trigger = false
+
+            @dst_infos.select{ | _, dst_info | dst_info[ :paused ] }.each do | dst, dst_info |
+              streamd = dst_info[ :streamd ]
+              streamd_info = @streamd_infos[ streamd ]
+
+              if streamd_info[ :wbuff ].size < RESUME_BELOW then
+                puts "p#{ Process.pid } #{ Time.new } resume dst #{ dst_info[ :domain_port ] }"
+                dst_info[ :paused ] = false
+                add_read( dst )
+                trigger = true
+              end
+            end
+
+            @streamd_infos.select{ | _, streamd_info | streamd_info[ :paused ] }.each do | streamd, streamd_info |
+              dst = streamd_info[ :dst ]
+              dst_info = @dst_infos[ dst ]
+
+              if dst_info[ :wbuff ].size < RESUME_BELOW then
+                puts "p#{ Process.pid } #{ Time.new } resume streamd #{ streamd_info[ :domain_port ] }"
+                streamd_info[ :paused ] = false
+                add_read( streamd )
+                trigger = true
+              end
+            end
+
+            next_tick if trigger
+          end
+        end
       end
     end
 
     ##
-    # add pause dst
+    # new a proxyd
     #
-    def add_pause_dst( dst )
-      @reads.delete( dst )
+    def new_a_proxyd( proxyd_port )
+      proxyd = Socket.new( Socket::AF_INET, Socket::SOCK_DGRAM, 0 )
+      proxyd.setsockopt( Socket::SOL_SOCKET, Socket::SO_REUSEPORT, 1 )
+      proxyd.bind( Socket.sockaddr_in( proxyd_port, '0.0.0.0' ) )
 
-      unless @pause_dsts.include?( dst )
-        @pause_dsts << dst
-      end
+      puts "p#{ Process.pid } #{ Time.new } proxyd bind on #{ proxyd_port }"
+      @proxyd = proxyd
+      @proxyd_info = {
+        ctlmsgs: [] # [ ctlmsg, to_addr ]
+      }
+
+      add_read( proxyd, :proxyd )
     end
 
     ##
-    # set is closing
+    # next tick
     #
-    def set_is_closing( sock )
-      if sock && !sock.closed?
-        role = @roles[ sock ]
-        # puts "debug1 set #{ role.to_s } is closing"
-
-        case role
-        when :dst
-          dst_info = @dst_infos[ sock ]
-          dst_info[ :is_closing ] = true
-        when :tund
-          tund_info = @tund_infos[ sock ]
-          tund_info[ :is_closing ] = true
-        end
-
-        @reads.delete( sock )
-        add_write( sock )
-      end
+    def next_tick
+      @dotw.write( '.' )
     end
 
     ##
-    # tunnel data
+    # resolve domain
     #
-    def tunnel_data( dst, data )
-      dst_info = @dst_infos[ dst ]
-      tund = dst_info[ :tund ]
+    def resolve_domain( tund, src_id, domain_port )
+      resolv_cache = @resolv_caches[ domain_port ]
 
-      if tund.closed?
-        puts "p#{ Process.pid } #{ Time.new } tund closed, close dst"
-        set_is_closing( dst )
-        return
+      if resolv_cache then
+        destination_addr, created_at = resolv_cache
+
+        if Time.new - created_at < RESOLV_CACHE_EXPIRE then
+          # puts "debug1 #{ domain_port } hit resolv cache #{ Addrinfo.new( destination_addr ).inspect }"
+          deal_with_destination_addr( tund, src_id, destination_addr, domain_port )
+          return
+        end
+
+        # puts "debug1 expire #{ domain_port } resolv cache"
+        @resolv_caches.delete( domain_port )
       end
 
-      tund_info = @tund_infos[ tund ]
-      dst_id = dst_info[ :id ]
-      now = Time.new
-      pack_id = dst_info[ :biggest_pack_id ]
-      idx = 0
-      len = data.bytesize
+      Thread.new do
+        colon_idx = domain_port.rindex( ':' )
 
-      while idx < len
-        chunk = data[ idx, PACK_SIZE ]
-        pack_id += 1
+        if colon_idx then
+          destination_domain = domain_port[ 0...colon_idx ]
+          destination_port = domain_port[ ( colon_idx + 1 )..-1 ].to_i
 
-        if pack_id <= CONFUSE_UNTIL
-          chunk = @custom.encode( chunk )
-          # puts "debug1 encoded chunk #{ pack_id }"
+          begin
+            destination_addr = Socket.sockaddr_in( destination_port, destination_domain )
+          rescue Exception => e
+            puts "p#{ Process.pid } #{ Time.new } sockaddr in #{ domain_port } #{ e.class }"
+          end
         end
 
-        data2 = [ [ pack_id, dst_id ].pack( 'Q>n' ), chunk ].join
-        sent = send_data( tund, data2, tund_info[ :tun_addr ] )
-        # puts "debug2 written pack #{ pack_id } #{ sent }"
-        tund_info[ :wmems ][ [ dst_id, pack_id ] ] = data2
-        dst_info[ :send_ats ][ pack_id ] = now
-        idx += PACK_SIZE
-      end
-
-      dst_info[ :biggest_pack_id ] = pack_id
-      dst_info[ :last_continue_at ] = now
+        @mutex.synchronize do
+          if destination_addr then
+            # puts "debug1 resolved #{ domain_port } #{ Addrinfo.new( destination_addr ).inspect }"
+            @resolv_caches[ domain_port ] = [ destination_addr, Time.new ]
 
-      # 写后超过上限，暂停读dst
-      if tund_info[ :wmems ].size >= WMEMS_LIMIT
-        puts "p#{ Process.pid } #{ Time.new } pause dst #{ dst_id } #{ dst_info[ :domain_port ] } #{ dst_info[ :biggest_pack_id ] }"
-        add_pause_dst( dst )
+            unless tund.closed? then
+              if deal_with_destination_addr( tund, src_id, destination_addr, domain_port ) then
+                next_tick
+              end
+            end
+          end
+        end
       end
     end
 
@@ -438,164 +474,68 @@ module Girl
     #
     def send_data( sock, data, to_addr )
       begin
-        sock.sendmsg( data, 0, to_addr )
+        written = sock.sendmsg_nonblock( data, 0, to_addr )
       rescue IO::WaitWritable, Errno::EINTR
-        return false
+        return :wait
       rescue Errno::EHOSTUNREACH, Errno::ENETUNREACH, Errno::ENETDOWN => e
-        if @roles[ sock ] == :tund
-          puts "#{ Time.new } #{ e.class }, close tund"
-          close_tund( sock )
-          return false
-        end
+        puts "p#{ Process.pid } #{ Time.new } sendmsg #{ e.class }"
+        return :fatal
       end
 
-      true
+      written
     end
 
     ##
-    # close dst
+    # set dst closing
     #
-    def close_dst( dst )
-      # puts "debug1 close dst"
-      close_sock( dst )
-      @pause_dsts.delete( dst )
+    def set_dst_closing( dst )
+      return if dst.closed?
       dst_info = @dst_infos[ dst ]
-      tund = dst_info[ :tund ]
-
-      if tund.closed?
-        @dst_infos.delete( dst )
-        return
-      end
-
-      tund_info = @tund_infos[ tund ]
-      dst_id = dst_info[ :id ]
-
-      if dst_info[ :is_src_closed ]
-        # puts "debug1 4-3. after close dst -> src closed ? yes -> del dst ext -> send fin2"
-        del_dst_ext( tund_info, dst_id )
-        data = [ 0, FIN2, dst_id ].pack( 'Q>Cn' )
-      else
-        # puts "debug1 3-1. after close dst -> src closed ? no -> send fin1"
-        data = [ 0, FIN1, dst_id, dst_info[ :biggest_pack_id ], dst_info[ :continue_src_pack_id ] ].pack( 'Q>CnQ>Q>' )
-      end
-
-      send_data( tund, data, tund_info[ :tun_addr ] )
-    end
-
-    ##
-    # close tun
-    #
-    def close_tund( tund )
-      # puts "debug1 close tund"
-      close_sock( tund )
-      tund_info = @tund_infos.delete( tund )
-      tund_info[ :dsts ].each{ | _, dst | set_is_closing( dst ) }
-      @tunneling_tunds.delete( tund_info[ :tun_addr ] )
-      @tunds.delete( tund_info[ :port ] )
-    end
-
-    ##
-    # close sock
-    #
-    def close_sock( sock )
-      sock.close
-      @reads.delete( sock )
-      @writes.delete( sock )
-      @roles.delete( sock )
+      dst_info[ :closing ] = true
+      @reads.delete( dst )
+      add_write( dst )
     end
 
     ##
-    # del dst ext
+    # set dst closing write
     #
-    def del_dst_ext( tund_info, dst_id )
-      tund_info[ :wmems ].delete_if { | port_and_pack_id, _ | port_and_pack_id.first == dst_id }
-      dst = tund_info[ :dsts ].delete( dst_id )
-
-      if dst
-        dst_info = @dst_infos.delete( dst )
-
-        if dst_info
-          tund_info[ :dst_ids ].delete( dst_info[ :src_id ] )
-        end
-      end
+    def set_dst_closing_write( dst )
+      return if dst.closed?
+      dst_info = @dst_infos[ dst ]
+      dst_info[ :closing_write ] = true
+      add_write( dst )
     end
 
     ##
-    # release wmems
+    # set streamd closing
     #
-    def release_wmems( tund_info, dst_info, completed_pack_id )
-      if completed_pack_id > dst_info[ :completed_pack_id ]
-        # puts "debug2 update completed pack #{ completed_pack_id }"
-
-        pack_ids = dst_info[ :send_ats ].keys.select { | pack_id | pack_id <= completed_pack_id }
-
-        pack_ids.each do | pack_id |
-          tund_info[ :wmems ].delete( [ dst_info[ :id ], pack_id ] )
-          dst_info[ :send_ats ].delete( pack_id )
-        end
-
-        dst_info[ :completed_pack_id ] = completed_pack_id
-      end
+    def set_streamd_closing( streamd )
+      return if streamd.closed?
+      streamd_info = @streamd_infos[ streamd ]
+      streamd_info[ :closing ] = true
+      @reads.delete( streamd )
+      add_write( streamd )
     end
 
     ##
-    # next tick
+    # set streamd closing write
     #
-    def next_tick
-      @dotw.write( '.' )
+    def set_streamd_closing_write( streamd )
+      return if streamd.closed?
+      streamd_info = @streamd_infos[ streamd ]
+      streamd_info[ :closing_write ] = true
+      add_write( streamd )
     end
 
     ##
-    # write dst
+    # set tund is closing
     #
-    def write_dst( dst )
-      dst_info = @dst_infos[ dst ]
-
-      if dst_info[ :is_closing ]
-        close_dst( dst )
-        return
-      end
-
-      data = dst_info[ :wbuff ]
-
-      if data.empty?
-        @writes.delete( dst )
-        return
-      end
-
-      begin
-        written = dst.write_nonblock( data )
-      rescue IO::WaitWritable, Errno::EINTR
-        return
-      rescue Exception => e
-        # puts "debug1 write dst #{ e.class }"
-        close_dst( dst )
-        return
-      end
-
-      # puts "debug2 write dst #{ written }"
-      data = data[ written..-1 ]
-      dst_info[ :wbuff ] = data
-      dst_info[ :last_continue_at ] = Time.new
-    end
-
-    ##
-    # write tund
-    #
-    def write_tund( tund )
+    def set_tund_closing( tund )
+      return if tund.closed?
       tund_info = @tund_infos[ tund ]
-
-      if tund_info[ :is_closing ]
-        if tund_info[ :changed_tun_addr ]
-          data = [ 0, IP_CHANGED ].pack( 'Q>C' )
-          send_data( tund, data, tund_info[ :changed_tun_addr ] )
-        end
-
-        close_tund( tund )
-        return
-      end
-
-      @writes.delete( tund )
+      tund_info[ :closing ] = true
+      @reads.delete( tund )
+      add_write( tund )
     end
 
     ##
@@ -612,302 +552,420 @@ module Girl
       data, addrinfo, rflags, *controls = proxyd.recvmsg
       from_addr = addrinfo.to_sockaddr
 
-      if @tunneling_tunds.include?( from_addr )
+      if @tunneling_tunds.include?( from_addr ) then
         tund = @tunneling_tunds[ from_addr ]
         tund_info = @tund_infos[ tund ]
-        port = tund_info[ :port ]
-        data = [ 0, TUND_PORT, port ].pack( 'Q>Cn' )
-        puts "p#{ Process.pid } #{ Time.new } resend tund port #{ port }"
-        send_data( proxyd, data, from_addr )
+        puts "p#{ Process.pid } #{ Time.new } resend tund port #{ tund_info[ :port ] }, #{ tund_info[ :stream_port ] }"
+        add_proxyd_ctlmsg_tund_port( tund_info )
         return
       end
 
       result = @custom.check( data, addrinfo )
 
-      if result != :success
+      if result != :success then
         puts "p#{ Process.pid } #{ Time.new } #{ result }"
         return
       end
 
       tund = Socket.new( Socket::AF_INET, Socket::SOCK_DGRAM, 0 )
       tund.bind( Socket.sockaddr_in( 0, '0.0.0.0' ) )
-      port = tund.local_address.ip_port
+      tund_port = tund.local_address.ip_port
+      add_read( tund, :tund )
 
-      @tunneling_tunds[ from_addr ] = tund
-      @tunds[ port ] = tund
-      @tund_infos[ tund ] = {
-        port: port,           # 端口
-        wbuffs: [],           # 写前 [ dst_id, pack_id, data ]
-        wmems: {},            # 写后 [ dst_id, pack_id ] => data
+      tcpd = Socket.new( Socket::AF_INET, Socket::SOCK_STREAM, 0 )
+      tcpd.setsockopt( Socket::SOL_TCP, Socket::TCP_NODELAY, 1 ) if RUBY_PLATFORM.include?( 'linux' )
+      tcpd.bind( Socket.sockaddr_in( 0, '0.0.0.0' ) )
+      tcpd_port = tcpd.local_address.ip_port
+      tcpd.listen( 127 )
+      add_read( tcpd, :tcpd )
+
+      tund_info = {
+        port: tund_port,      # 端口
+        tcpd: tcpd,           # 对应的tcpd
+        tcpd_port: tcpd_port, # tcpd端口
+        ctlmsgs: [],          # [ ctlmsg, to_addr ]
         tun_addr: from_addr,  # tun地址
-        dsts: {},             # dst额外信息 dst_id => dst
+        dsts: {},             # dst_id => dst
         dst_ids: {},          # src_id => dst_id
         created_at: Time.new, # 创建时间
-        last_recv_at: nil,    # 上一次收到流量的时间，过期关闭
-        is_closing: false,    # 是否准备关闭
+        last_recv_at: nil,    # 上一次收到流量的时间
+        last_sent_at: nil,    # 上一次发出流量的时间
+        closing: false,       # 准备关闭
+        closing_read: false,  # 准备关闭读
+        closing_write: false, # 准备关闭写
         changed_tun_addr: nil # 记录到和tun addr不符的来源地址
       }
 
-      add_read( tund, :tund )
+      @tunneling_tunds[ from_addr ] = tund
+      @tund_infos[ tund ] = tund_info
+      @tcpd_infos[ tcpd ] = {
+        tund: tund
+      }
 
-      data = [ 0, TUND_PORT, port ].pack( 'Q>Cn' )
-      puts "p#{ Process.pid } #{ Time.new } a new tunnel #{ addrinfo.ip_unpack.inspect } - #{ port }, #{ @tunds.size } tunds"
-      send_data( proxyd, data, from_addr )
+      puts "p#{ Process.pid } #{ Time.new } a new tunnel #{ addrinfo.ip_unpack.inspect } - #{ tund_port }, #{ tcpd_port }, #{ @tund_infos.size } tunds"
+      add_proxyd_ctlmsg_tund_port( tund_info )
     end
 
     ##
-    # read dst
+    # read tund
     #
-    def read_dst( dst )
+    def read_tund( tund )
       begin
-        data = dst.read_nonblock( READ_SIZE )
+        data, addrinfo, rflags, *controls = tund.recvmsg_nonblock
       rescue IO::WaitReadable, Errno::EINTR
-        return
-      rescue Exception => e
-        # puts "debug1 read dst #{ e.class }"
-        set_is_closing( dst )
+        print 'r'
         return
       end
 
-      tunnel_data( dst, data )
-    end
-
-    ##
-    # read tund
-    #
-    def read_tund( tund )
-      data, addrinfo, rflags, *controls = tund.recvmsg
       from_addr = addrinfo.to_sockaddr
-      now = Time.new
       tund_info = @tund_infos[ tund ]
 
-      if from_addr != tund_info[ :tun_addr ]
-        # 通常是光猫刷新ip（端口也会变），但万一不是，为了避免脏数据注入，关闭tund
+      if from_addr != tund_info[ :tun_addr ] then
+        # 通常是光猫刷新ip和端口，但万一不是，为了避免脏数据注入，关闭tund
         puts "p#{ Process.pid } #{ Time.new } from #{ addrinfo.inspect } not match tun addr #{ Addrinfo.new( tund_info[ :tun_addr ] ).inspect }"
         tund_info[ :changed_tun_addr ] = from_addr
-        set_is_closing( tund )
+        set_tund_closing( tund )
         return
       end
 
-      tund_info[ :last_recv_at ] = now
       pack_id = data[ 0, 8 ].unpack( 'Q>' ).first
+      return if pack_id != 0
 
-      if pack_id == 0
-        ctl_num = data[ 8 ].unpack( 'C' ).first
+      tund_info[ :last_recv_at ] = Time.new
+      ctl_num = data[ 8 ].unpack( 'C' ).first
 
-        case ctl_num
-        when A_NEW_SOURCE
-          src_id = data[ 9, 8 ].unpack( 'Q>' ).first
-          dst_id = tund_info[ :dst_ids ][ src_id ]
+      case ctl_num
+      when A_NEW_SOURCE then
+        src_id = data[ 9, 8 ].unpack( 'Q>' ).first
+        dst_id = tund_info[ :dst_ids ][ src_id ]
 
-          if dst_id
-            dst = tund_info[ :dsts ][ dst_id ]
-            return unless dst
-
-            if dst.closed?
-              dst_id = 0
-            end
+        if dst_id then
+          dst = tund_info[ :dsts ][ dst_id ]
+          return unless dst
 
-            # puts "debug1 resend paired #{ dst_id }"
-            data2 = [ 0, PAIRED, src_id, dst_id ].pack( 'Q>CQ>n' )
-            send_data( tund, data2, tund_info[ :tun_addr ] )
-            return
+          if dst.closed? then
+            dst_id = 0
           end
 
-          data = data[ 17..-1 ]
-          domain_port = @custom.decode( data )
-          puts "p#{ Process.pid } #{ Time.new } a new source #{ src_id } #{ domain_port }"
-          resolve_domain( tund, src_id, domain_port )
-        when SOURCE_STATUS
-          src_id, relay_src_pack_id, continue_dst_pack_id  = data[ 9, 24 ].unpack( 'Q>Q>Q>' )
-
-          dst_id = tund_info[ :dst_ids ][ src_id ]
-          return unless dst_id
+          # puts "debug1 resend paired #{ dst_id }"
+          data2 = [ 0, PAIRED, src_id, dst_id ].pack( 'Q>CQ>n' )
+          add_ctlmsg( tund, data2 )
+          return
+        end
 
-          dst = tund_info[ :dsts ][ dst_id ]
-          return unless dst
+        data = data[ 17..-1 ]
+        domain_port = @custom.decode( data )
+        # puts "debug1 a new source #{ src_id } #{ domain_port }"
+        resolve_domain( tund, src_id, domain_port )
+      when TUN_FIN then
+        puts "p#{ Process.pid } #{ Time.new } recv tun fin"
+        set_tund_closing( tund )
+      end
+    end
 
-          dst_info = @dst_infos[ dst ]
-          return unless dst_info
+    ##
+    # read tcpd
+    #
+    def read_tcpd( tcpd )
+      begin
+        streamd, addrinfo = tcpd.accept_nonblock
+      rescue IO::WaitReadable, Errno::EINTR
+        print 'r'
+        return
+      end
 
-          # puts "debug2 got source status #{ Time.new }"
+      # puts "debug1 accept a streamd"
+      tcpd_info = @tcpd_infos[ tcpd ]
+      tund = tcpd_info[ :tund ]
 
-          # 消写后
-          release_wmems( tund_info, dst_info, continue_dst_pack_id )
+      @streamd_infos[ streamd ] = {
+        tund: tund,       # 对应tund
+        dst: nil,         # 对应dst
+        domain_port: nil, # dst的目的地和端口
+        wbuff: '',        # 写前，写往近端stream
+        paused: false     # 是否已暂停读
+      }
 
-          # 发miss
-          if !dst.closed? && ( dst_info[ :continue_src_pack_id ] < relay_src_pack_id )
-            ranges = []
-            ignored = false
-            curr_pack_id = dst_info[ :continue_src_pack_id ] + 1
+      add_read( streamd, :streamd )
+    end
 
-            dst_info[ :pieces ].keys.sort.each do | pack_id |
-              if pack_id > curr_pack_id
-                ranges << [ curr_pack_id, pack_id - 1 ]
+    ##
+    # read dst
+    #
+    def read_dst( dst )
+      begin
+        data = dst.read_nonblock( READ_SIZE )
+      rescue IO::WaitReadable, Errno::EINTR
+        print 'r'
+        return
+      rescue Exception => e
+        # puts "debug1 read dst #{ e.class }"
+        dst_info = close_read_dst( dst )
+        streamd = dst_info[ :streamd ]
+        set_streamd_closing_write( streamd ) if streamd
+        return
+      end
 
-                if ranges.size >= MISS_RANGE_LIMIT
-                  puts "p#{ Process.pid } #{ Time.new } break add miss range at #{ pack_id }"
-                  ignored = true
-                  break
-                end
-              end
+      dst_info = @dst_infos[ dst ]
+      streamd = dst_info[ :streamd ]
+
+      if streamd then
+        unless streamd.closed? then
+          streamd_info = @streamd_infos[ streamd ]
+          data = @custom.encode( data )
+          # puts "debug2 add streamd.wbuff encoded #{ data.bytesize }"
+          streamd_info[ :wbuff ] << data
+          add_write( streamd )
+
+          if streamd_info[ :wbuff ].bytesize >= WBUFF_LIMIT then
+            puts "p#{ Process.pid } #{ Time.new } pause dst #{ dst_info[ :domain_port ] }"
+            dst_info[ :paused ] = true
+            @reads.delete( dst )
+          end
+        end
+      else
+        dst_info[ :rbuff ] << data
 
-              curr_pack_id = pack_id + 1
-            end
+        if dst_info[ :rbuff ].bytesize >= WBUFF_LIMIT then
+          # puts "debug1 dst.rbuff full"
+          set_dst_closing( dst )
+        end
+      end
+    end
 
-            if !ignored && ( curr_pack_id <= relay_src_pack_id )
-              ranges << [ curr_pack_id, relay_src_pack_id ]
-            end
+    ##
+    # read streamd
+    #
+    def read_streamd( streamd )
+      begin
+        data = streamd.read_nonblock( READ_SIZE )
+      rescue IO::WaitReadable, Errno::EINTR
+        print 'r'
+        return
+      rescue Exception => e
+        # puts "debug1 read streamd #{ e.class }"
+        streamd_info = close_read_streamd( streamd )
+        dst = streamd_info[ :dst ]
+        set_dst_closing_write( dst ) if dst
+        return
+      end
 
-            # puts "debug1 continue/relay #{ dst_info[ :continue_src_pack_id ] }/#{ relay_src_pack_id } send MISS #{ ranges.size }"
-            idx = 0
-            ranges = ranges.map{ | pack_id_begin, pack_id_end | [ pack_id_begin, pack_id_end ].pack( 'Q>Q>' ) }
+      streamd_info = @streamd_infos[ streamd ]
+      dst = streamd_info[ :dst ]
 
-            while idx < ranges.size
-              chunk = ranges[ idx, MULTI_MISS_SIZE ].join
-              data2 = [ [ 0, MULTI_MISS, src_id ].pack( 'Q>CQ>' ), chunk ].join
-              send_data( tund, data2, tund_info[ :tun_addr ] )
-              idx += MULTI_MISS_SIZE
-            end
-          end
-        when MULTI_MISS
-          dst_id, *ranges = data[ 9..-1 ].unpack( 'nQ>*' )
+      unless dst then
+        dst_id = data[ 0, 2 ].unpack( 'n' ).first
+        tund = streamd_info[ :tund ]
 
-          dst = tund_info[ :dsts ][ dst_id ]
-          return unless dst
+        if tund.closed? then
+          set_streamd_closing( streamd )
+          return
+        end
 
-          dst_info = @dst_infos[ dst ]
-          return unless dst_info
+        tund_info = @tund_infos[ tund ]
+        dst = tund_info[ :dsts ][ dst_id ]
 
-          return if ranges.empty? || ( ranges.size % 2 != 0 )
+        unless dst then
+          set_streamd_closing( streamd )
+          return
+        end
 
-          # puts "debug1 got multi miss #{ dst_id } #{ ranges.size }"
+        # puts "debug1 set streamd.dst #{ dst_id }"
+        streamd_info[ :dst ] = dst
+        dst_info = @dst_infos[ dst ]
+        streamd_info[ :domain_port ] = dst_info[ :domain_port ]
 
-          idx = 0
+        unless dst_info[ :rbuff ].empty? then
+          # puts "debug1 encode and move dst.rbuff to streamd.wbuff"
+          streamd_info[ :wbuff ] << @custom.encode( dst_info[ :rbuff ] )
+        end
 
-          while idx < ranges.size
-            pack_id_begin, pack_id_end = ranges[ idx ], ranges[ idx + 1 ]
+        dst_info[ :streamd ] = streamd
+        data = data[ 2..-1 ]
 
-            ( pack_id_begin..pack_id_end ).each do | pack_id |
-              send_at = dst_info[ :send_ats ][ pack_id ]
+        return if data.empty?
+      end
 
-              if send_at
-                break if now - send_at < CHECK_STATUS_INTERVAL
-                data2 = tund_info[ :wmems ][ [ dst_id, pack_id ] ]
+      unless dst.closed? then
+        dst_info = @dst_infos[ dst ]
+        data = @custom.decode( data )
+        # puts "debug2 add dst.wbuff decoded #{ data.bytesize }"
+        dst_info[ :wbuff ] << data
+        dst_info[ :last_recv_at ] = Time.new
+        add_write( dst )
 
-                if data2
-                  if send_data( tund, data2, tund_info[ :tun_addr ] )
-                    dst_info[ :last_continue_at ] = now
-                  end
-                end
-              end
-            end
+        if dst_info[ :wbuff ].bytesize >= WBUFF_LIMIT then
+          puts "p#{ Process.pid } #{ Time.new } pause streamd #{ streamd_info[ :domain_port ] }"
+          streamd_info[ :paused ] = true
+          @reads.delete( streamd )
+        end
+      end
+    end
 
-            idx += 2
-          end
-        when MISS
-          dst_id, pack_id_begin, pack_id_end = data[ 9, 18 ].unpack( 'nQ>Q>' )
+    ##
+    # write proxyd
+    #
+    def write_proxyd( proxyd )
+      # 发ctlmsg
+      while @proxyd_info[ :ctlmsgs ].any? do
+        data, to_addr = @proxyd_info[ :ctlmsgs ].first
+        sent = send_data( proxyd, data, to_addr )
+
+        if sent == :wait then
+          puts "p#{ Process.pid } #{ Time.new } wait proxyd send ctlmsg, left #{ @proxyd_info[ :ctlmsgs ].size }"
+          return
+        else
+          @proxyd_info[ :ctlmsgs ].shift
+        end
+      end
 
-          dst = tund_info[ :dsts ][ dst_id ]
-          return unless dst
+      @writes.delete( proxyd )
+    end
 
-          dst_info = @dst_infos[ dst ]
-          return unless dst_info
+    ##
+    # write tund
+    #
+    def write_tund( tund )
+      tund_info = @tund_infos[ tund ]
 
-          ( pack_id_begin..pack_id_end ).each do | pack_id |
-            send_at = dst_info[ :send_ats ][ pack_id ]
+      # 处理关闭
+      if tund_info[ :closing ] then
+        if tund_info[ :changed_tun_addr ] then
+          data = [ 0, IP_CHANGED ].pack( 'Q>C' )
+          send_data( tund, data, tund_info[ :changed_tun_addr ] )
+        end
 
-            if send_at
-              break if now - send_at < CHECK_STATUS_INTERVAL
-              data2 = tund_info[ :wmems ][ [ dst_id, pack_id ] ]
+        close_tund( tund )
+        return
+      end
 
-              if data2
-                if send_data( tund, data2, tund_info[ :tun_addr ] )
-                  dst_info[ :last_continue_at ] = now
-                end
-              end
-            end
-          end
-        when FIN1
-          src_id, biggest_src_pack_id, continue_dst_pack_id = data[ 9, 24 ].unpack( 'Q>Q>Q>' )
+      now = Time.new
 
-          dst_id = tund_info[ :dst_ids ][ src_id ]
-          return unless dst_id
+      # 发ctlmsg
+      while tund_info[ :ctlmsgs ].any? do
+        data = tund_info[ :ctlmsgs ].first
+        sent = send_data( tund, data, tund_info[ :tun_addr ] )
 
-          dst = tund_info[ :dsts ][ dst_id ]
-          return unless dst
+        if sent == :fatal then
+          close_tund( tund )
+          return
+        elsif sent == :wait then
+          puts "p#{ Process.pid } #{ Time.new } wait tund #{ tund_info[ :port ] } send ctlmsg, left #{ tund_info[ :ctlmsgs ].size }"
+          tund_info[ :last_sent_at ] = now
+          return
+        end
 
-          dst_info = @dst_infos[ dst ]
-          return unless dst_info
+        tund_info[ :ctlmsgs ].shift
+      end
 
-          # puts "debug1 got fin1 #{ src_id } biggest src pack #{ biggest_src_pack_id } completed dst pack #{ continue_dst_pack_id }"
-          dst_info[ :is_src_closed ] = true
-          dst_info[ :biggest_src_pack_id ] = biggest_src_pack_id
-          release_wmems( tund_info, dst_info, continue_dst_pack_id )
+      tund_info[ :last_sent_at ] = now
+      @writes.delete( tund )
+    end
 
-          if biggest_src_pack_id == dst_info[ :continue_src_pack_id ]
-            # puts "debug1 4-1. tund recv fin1 -> all traffic received ? -> close dst after write"
-            set_is_closing( dst )
-          end
-        when FIN2
-          src_id = data[ 9, 8 ].unpack( 'Q>' ).first
+    ##
+    # write dst
+    #
+    def write_dst( dst )
+      return if dst.closed?
+      dst_info = @dst_infos[ dst ]
+      streamd = dst_info[ :streamd ]
 
-          dst_id = tund_info[ :dst_ids ][ src_id ]
-          return unless dst_id
+      # 处理关闭
+      if dst_info[ :closing ] then
+        close_dst( dst )
 
-          # puts "debug1 3-2. tund recv fin2 -> del dst ext"
-          del_dst_ext( tund_info, dst_id )
-        when TUN_FIN
-          puts "p#{ Process.pid } #{ Time.new } recv tun fin"
-          set_is_closing( tund )
+        if streamd then
+          close_read_streamd( streamd )
+          set_streamd_closing_write( streamd )
         end
 
         return
       end
 
-      src_id = data[ 8, 8 ].unpack( 'Q>' ).first
+      data = dst_info[ :wbuff ]
 
-      dst_id = tund_info[ :dst_ids ][ src_id ]
-      return unless dst_id
+      # 写前为空，处理关闭写
+      if data.empty? then
+        if dst_info[ :closing_write ] then
+          close_write_dst( dst )
+        else
+          @writes.delete( dst )
+        end
 
-      dst = tund_info[ :dsts ][ dst_id ]
-      return unless dst
+        return
+      end
 
-      dst_info = @dst_infos[ dst ]
-      return unless dst_info
+      # 写入
+      begin
+        written = dst.write_nonblock( data )
+      rescue IO::WaitWritable, Errno::EINTR
+        print 'w'
+        return
+      rescue Exception => e
+        # puts "debug1 write dst #{ e.class }"
+        close_write_dst( dst )
+        close_read_streamd( streamd ) if streamd
+        return
+      end
 
-      return if ( pack_id <= dst_info[ :continue_src_pack_id ] ) || dst_info[ :pieces ].include?( pack_id )
+      # puts "debug2 written dst #{ written }"
+      data = data[ written..-1 ]
+      dst_info[ :wbuff ] = data
+    end
 
-      data = data[ 16..-1 ]
-      # puts "debug2 got pack #{ pack_id }"
+    ##
+    # write streamd
+    #
+    def write_streamd( streamd )
+      return if streamd.closed?
+      streamd_info = @streamd_infos[ streamd ]
+      dst = streamd_info[ :dst ]
+
+      # 处理关闭
+      if streamd_info[ :closing ] then
+        close_streamd( streamd )
+
+        if dst then
+          close_read_dst( dst )
+          set_dst_closing_write( dst )
+        end
 
-      if pack_id <= CONFUSE_UNTIL
-        # puts "debug2 #{ data.inspect }"
-        data = @custom.decode( data )
-        # puts "debug1 decoded pack #{ pack_id }"
+        return
       end
 
-      # 放进写前，跳号放碎片缓存
-      if pack_id - dst_info[ :continue_src_pack_id ] == 1
-        while dst_info[ :pieces ].include?( pack_id + 1 )
-          data << dst_info[ :pieces ].delete( pack_id + 1 )
-          pack_id += 1
+      data = streamd_info[ :wbuff ]
+
+      # 写前为空，处理关闭写
+      if data.empty? then
+        if streamd_info[ :closing_write ] then
+          close_write_streamd( streamd )
+        else
+          @writes.delete( streamd )
         end
 
-        dst_info[ :continue_src_pack_id ] = pack_id
-        dst_info[ :last_continue_at ] = now
-        dst_info[ :wbuff ] << data
-        add_write( dst )
-        # puts "debug2 update continue src pack #{ pack_id }"
+        return
+      end
 
-        # 若对面已关闭，且流量正好收全，关闭dst
-        if dst_info[ :is_src_closed ] && ( pack_id == dst_info[ :biggest_src_pack_id ] )
-          # puts "debug1 4-2. tund recv traffic -> src closed and all traffic received ? -> close dst after write"
-          set_is_closing( dst )
-        end
-      elsif !dst_info[ :pieces ].include?( pack_id )
-        dst_info[ :pieces ][ pack_id ] = data
-        dst_info[ :last_continue_at ] = now
+      # 写入
+      begin
+        written = streamd.write_nonblock( data )
+      rescue IO::WaitWritable, Errno::EINTR
+        print 'w'
+        return
+      rescue Exception => e
+        # puts "debug1 write streamd #{ e.class }"
+        close_write_streamd( streamd )
+        close_read_dst( dst ) if dst
+        return
+      end
+
+      # puts "debug2 written streamd #{ written }"
+      data = data[ written..-1 ]
+      streamd_info[ :wbuff ] = data
+
+      if dst && !dst.closed? then
+        dst_info = @dst_infos[ dst ]
+        dst_info[ :last_sent_at ] = Time.new
       end
     end