dango 0.4.8 → 0.5.0

data/lib/dango/server_framework.rb

@@ -10,11 +10,15 @@ require "dango/framework_base"
 require "dango/socket_list"
 require "dango/version"
 require "dango/session_manager"
-require "dango/monitor/server_monitor_action"
+#require "dango/monitor/server_monitor_action"
+require "dango/monitor/server_monitor_module"
+require "dango/default_dango_receive"
 
 # フレームワーククラス
 class DangoServerFramework
   require "dango/dango_g_server"
+  include DefaultDangoReceive
+  include ServerMonitorModule
   
   include DangoFrameworkModule
   
@@ -107,14 +111,15 @@ class DangoServerFramework
     
     # スレッドグループの初期化
     @thr_gr_gserver_serve = ThreadGroup.new
-    @thr_gr_send_notice = ThreadGroup.new
-    @thr_gr_send_data = ThreadGroup.new
     
     # データ送信用スレッドとQueue
     @queue_send_notice = Queue.new
-    @mutex_send_notice = Mutex.new
+    @mutex_send_notice = DangoMutex.new(:mutex_send_notice)
     Thread.start{ thread_send_notice_queue() }
     
+    # メインスレッドの名前
+    Thread.current[:_name] = "main"
+    
     # サーバー開始
     server_start()
     
@@ -146,31 +151,42 @@ class DangoServerFramework
       logger.warn("max_connections=#{@server_max_connections}")
       logger.warn("RAILS_ENV=#{RAILS_ENV}")
       logger.warn("Process.pid=#{Process.pid}")
-      logger.warn("Dango Version=#{Dango::VERSION::STRING}")
-#    logger.debug("$LOADED_FEATURES=#{$LOADED_FEATURES.pretty_inspect}")
+      logger.warn("Dango Version=#{Dango::VERSION::STRING} p11")
+#      logger.debug("$LOADED_FEATURES=#{$LOADED_FEATURES.pretty_inspect}")
       
       shared_init()                                             # 共有メモリを初期化
       socket_list_init()                                        # ソケット一覧を初期化
       @mutex_sock = {}                                          # ソケットの排他処理ハッシュ
       @session_manager = SessionManager.new(shared, RAILS_ENV)  # セッション情報の初期化
       
-      @mutex_proc_thread = Mutex.new                     # スレッドが同時起動しないようにするためのMutex
+      @mutex_proc_thread = DangoMutex.new(:mutex_proc_thread)   # スレッドが同時起動しないようにするためのMutex
       heart_beat_thread_start()                          # ハートビートスレッドの開始
       gc_thread_start()                                  # GCスレッドの開始
       
       dango_server_init()                                # 初期設定読み込み
       
-      @mutex_send_response = Mutex.new                   # 送信レスポンス用のMutex
-      @arr_send_response = []                            # 送信レスポンス用のキャッシュ
+      @mutex_send_response = DangoMutex.new(:mutex_send_response) # 送信レスポンス用のMutex
+      @arr_send_response = []                                     # 送信レスポンス用のキャッシュ
       
       
       TCPSocket.do_not_reverse_lookup = true # 逆引きを行わない
 #     TCPSocket.do_not_reverse_lookup = false # 逆引きを行う
       
       if @backdoor_run_drb_server                    # backdoor用のdrbを起動
-        uri = "druby://#{@network_host}:#{@backdoor_run_drb_port}"
-        DRb.start_service(uri, self)
-        logger.warn("start drb server: uri=#{DRb.uri}")
+        require "drb"
+        require "drb/acl"
+#        uri = "druby://#{@network_host}:#{@backdoor_run_drb_port}"
+        uri = @backdoor_run_drb_url
+        list = ["deny", "all", "allow", @backdoor_host]
+        acl = ACL.new(list, ACL::DENY_ALLOW)
+        @drb = DRb.start_service(uri, self, acl)
+#        @drb = DRb.start_service(uri, self)
+        DRb.thread[:_name] = "thread_drb_main"
+        message = "start drb server: uri=#{DRb.uri}"
+        logger.warn(message)
+        puts message
+      else
+        @drb = nil
       end
       
       set_loop_setting(@loop_setting) if @loop_setting # ループ処理を定義
@@ -193,9 +209,9 @@ class DangoServerFramework
       @gserver.start(@server_max_connections, 20) # backlogを20に設定
       
     rescue Exception
-      puts error_message($!)
+      puts "gserver.start:" + error_message($!)
 #      error_print(error_message($!, 'u'))
-      exit
+      exit!
     end
   end
   
@@ -207,9 +223,14 @@ class DangoServerFramework
         
         debug_print("-- stopping server")
         stop_gserver() # gserverの停止
-        
         @gserver = nil
         
+        if @drb # drbを起動していればそれの停止
+          DRb.stop_service()
+          puts "stop drb server"
+          @drb = nil
+        end
+        
         debug_print("-- stopped server")
         
         # GCを行う
@@ -221,14 +242,14 @@ class DangoServerFramework
         debug_print("-- reload classes")
         tmp_verbose = $VERBOSE
         $VERBOSE = nil
-
+        
         # serverファイル名一覧を取得
         load_files = []
         glob_str = 'dango/server/*.rb'
         Dir.glob(glob_str) do |srv_file|
           load_files.push({:file=>srv_file, :mtime=>File.mtime(srv_file)})
         end
-
+        
         # ファイル名順にソート
         load_files = load_files.sort_by{|f| f[:file] }
         load_files.each do |f|
@@ -238,7 +259,7 @@ class DangoServerFramework
             debug_print("#{error_message($!, 'u')}")
           end
         end
-
+        
         $VERBOSE = tmp_verbose
         
         # サーバーを再度起動
@@ -252,7 +273,7 @@ class DangoServerFramework
   
   # gserver接続開始処理
   def connecting(sock)
-    logger.info "connecting:sock=#{sock} peeraddr=#{sock.peeraddr.inspect} addr=#{sock.addr.inspect}"
+    logger.info "connecting:sock=#{sock.object_id} peeraddr=#{sock.peeraddr.inspect} addr=#{sock.addr.inspect}"
   end
   
   # gserver接続開始処理
@@ -276,7 +297,8 @@ class DangoServerFramework
   
   # gserverのserveメソッド：スレッド開始処理
   def thread_main(sock)
-    logger.info "thread_main:connect. sock=#{sock} thread.current=#{Thread.current.object_id}"
+    logger.info "thread_main:connect. sock=#{sock.object_id} thread.current=#{Thread.current.object_id}"
+    Thread.current[:_name] = "thread_main #{sock.object_id}"
     
     @thr_gr_gserver_serve.add(Thread.current) # スレッドグループに登録
     
@@ -285,9 +307,9 @@ class DangoServerFramework
       sid = session[:sid]
       
       # sockに対しての=nil write close などの書き換えに対する排他処理
-      @mutex_sock[sock.object_id] = Mutex.new 
+      @mutex_sock[sock.object_id] = DangoMutex.new("mutex_sock_#{sock.object_id}")
       
-      @mutex_sock[sock.object_id].synchronize do
+      @mutex_sock[sock.object_id].timeout_sync(2, "start #{sock.object_id}") do
         sock.binmode
         sock.sync = true
         
@@ -298,7 +320,7 @@ class DangoServerFramework
       
       session[:encode_type] = DefaultEncodeType
       
-      logger.debug "thread_main:start_session. sid=#{sid} sock=#{sock} thread.current=#{Thread.current.object_id}"
+      logger.debug "thread_main:start_session. sid=#{sid} sock=#{sock.object_id} thread.current=#{Thread.current.object_id}"
       
       socket_list.add(sid, sock)
       
@@ -306,28 +328,32 @@ class DangoServerFramework
       
       is_flash_policy_file = false
       
-      # 誰かが接続したときに60秒以上経った@arr_send_responseを削除
-      @mutex_send_response.synchronize do
-        @arr_send_response.delete_if{|r| r[:time] < Time.now - 60}
-      end
-      
       logger.debug "thread_main:do main loop."
       
       # メインループ入り
+      count = 0
       loop do
         if sock.closed?
-          logger.info "#{sock.inspect}:sock is closed..."
+          logger.info "#{sock.object_id}:sock is closed... #{count}"
           break
         end
         
         ret_objs = nil
         begin
+          logger.debug "dango_receive_data:start:sock=#{sock.object_id} #{count}" if count % 100 == 0
           ret_objs = dango_receive_data(sock) # データ受信処理
+          logger.debug "dango_receive_data:finish:sock=#{sock.object_id} #{count}" if count % 100 == 0
           is_flash_policy_file = false
         rescue DangoFrameworkFlashPolicyException
           is_flash_policy_file = true
         end
         
+        # メンテナンス機能
+        if shared[:_is_server_sleep]
+          logger.info "thread_main SLEEP !!!"
+          sleep 10000 
+        end
+        
         if is_flash_policy_file # Flashのポリシーファイルが来たら
           logger.debug "is_flash_policy_file"
           do_flash_policy_file(sid, sock)
@@ -340,6 +366,8 @@ class DangoServerFramework
           logger.debug "ret_objs is empty. "
           
         else # 受信データがあれば
+          logger.debug "dango_receive_data:ret_objs=#{ret_objs.class} #{count}" if count % 100 == 0
+          
           ret_objs.each do |ret_obj| # 受信データループ
             if !(ret_obj && ret_obj["_action_name"]) # action_nameが無い場合はエラー
               logger.warn "no action_name error:#{ret_obj.inspect}"
@@ -351,37 +379,27 @@ class DangoServerFramework
             session[:_action_name] = action_name
             
             # 自動返信用データ
-            session[:_return] = {}
+            return_data = {}
             ret_obj.each do |key, value|
               if key =~ /^_return_(.*)$/
-                session[:_return][$1.to_sym] = value
+                return_data[$1.to_sym] = value
               end
             end
+            session[:_return] = return_data
             
-            @mutex_proc_thread.synchronize do
-              
-              if action_name[0, 9] == '_monitor_' # メンテナンスアクション名なら
-                peerhost = session[:peer_host]
-                peeraddr = session[:peer_ip_address]
-                ServerMonitorAction.new(self, @config, peerhost, peeraddr, ret_obj)
-                
-              elsif action_name == '_response' # 送信パケットの返事なら
-                action_name_is_response(ret_obj)
-                
-              elsif action_name == '_change_encode_type' # エンコードタイプの変更なら
-                action_name_is_change_encode_type(ret_obj)
-                
-              else # メンテナンスコマンド以外のユーザーアクション名なら
-                logger.debug "receive_action:#{sid}:#{action_name}:"
-                begin
-                  __send__("dango_receive_#{action_name}", ret_obj)
-                rescue NoMethodError
-                  @recv_fail_count += 1 if @recv_fail_count # 受信失敗回数カウント
-                  logger.error "not find action #{action_name}:#{sid}:#{error_message($!, 'u')}"
-                end
+            logger.debug "mpth:1:receive_action:session=#{sid} #{count}"
+            @mutex_proc_thread.timeout_sync(4, "receive_action_#{action_name}") do
+              logger.debug "mpth:2:receive_action:#{sid}:#{action_name}:"
+              begin
+                __send__("dango_receive_#{action_name}", ret_obj)
+              rescue NoMethodError
+                @recv_fail_count += 1 if @recv_fail_count # 受信失敗回数カウント
+                logger.error "not find action #{action_name}:#{sid}:#{error_message($!, 'u')}"
+              rescue DangoFrameworkTimeoutException
+                logger.error "action is timeout:#{action_name}:#{sid}:#{error_message($!, 'u')}"
               end
               
-            end # @mutex_proc_thread.synchronize
+            end # @mutex_proc_thread
             
             # 不要になったセッション内のデータを削除（sidは再利用されるので消さない）
             session.delete(:_action_name)
@@ -389,7 +407,9 @@ class DangoServerFramework
           end
           
           if ! session[:connected] # まだdango_connectを呼び出していないなら
-            @mutex_proc_thread.synchronize do
+            logger.debug "mpth:1:dango_connect"
+            @mutex_proc_thread.timeout_sync(4, :dango_connect) do
+              logger.debug "mpth:2:dango_connect"
               # 接続直後のsid通知
               send_obj = {"_sid"=>sid}
               send_notice("_notice_sid", sid, send_obj)
@@ -397,10 +417,12 @@ class DangoServerFramework
               
               dango_connect() # 接続時メソッド呼び出し
               session[:connected] = true
+              
             end # mutex_proc_thread
           end
         end
         
+        count += 1
       end # loop
       
     rescue DangoFrameworkFlashPolicyException
@@ -424,6 +446,9 @@ class DangoServerFramework
     rescue DangoFrameworkDisconnectException
       logger.debug "DangoFrameworkDisconnectException. #{sid} \n#{error_message($!, 'u')}"
       
+    rescue DangoFrameworkMutexTimeoutException
+      logger.warn "DangoFrameworkMutexTimeoutException. #{sid} \n#{error_message($!, 'u')}"
+      
     rescue DangoFrameworkException
       logger.debug "DangoFrameworkException. #{sid} \n#{error_message($!, 'u')}"
       
@@ -434,37 +459,62 @@ class DangoServerFramework
       # 接続切断の処理
       
       if !is_flash_policy_file # Flash Policy Fileなら切断処理不要
-        logger.info "#{sock.inspect} #{sid} is closing"
+        logger.info "#{sock.object_id} #{sid} is closing"
         if session[:connected] # dango_connect()が実行済みなら
           begin
-            @mutex_proc_thread.synchronize do
+            logger.debug "mpth:1:dango_close"
+            @mutex_proc_thread.timeout_sync(4, :dango_close) do
+              logger.debug "mpth:2:dango_close"
               dango_close() # 接続解除時に呼び出されるメソッド
+              
+              # ハートビートのごみを削除
+              shared.transaction(:_heart_beat_time_hash) do |hbt_hash|
+                hbt_hash.delete(sid)
+                shared.commit(hbt_hash)
+              end
+              
             end # mutex_proc_thread
+            logger.debug "dango_close end"
+            
+          rescue DangoFrameworkMutexTimeoutException
+            logger.warn "DangoFrameworkMutexTimeoutException " + 
+                         "#{sock.object_id} #{sid} \n#{error_message($!, 'u')}"
           rescue Exception
-            logger.error "#{sock.inspect} #{sid} ERROR\n#{error_message($!, 'u')}"
+            logger.error "#{sock.object_id} #{sid} ERROR\n#{error_message($!, 'u')}"
           end
         end
       end
       
       begin
-        @mutex_sock[sock.object_id].synchronize do
+        @mutex_sock[sock.object_id].timeout_sync(2, "close #{sock.object_id}") do
           sock.close            # ソケットを閉じる
         end
       rescue IOError
-        logger.info "IOError #{sock.inspect} #{sid}"
+        logger.info "IOError #{sock.object_id} #{sid}"
+      rescue DangoFrameworkMutexTimeoutException
+        logger.warn "DangoFrameworkMutexTimeoutException #{sock.object_id} #{sid}\n" + 
+                    "#{error_message($!, 'u')}"
       rescue Exception
-        logger.error "#{sock.inspect} #{sid} Exception\n#{error_message($!, 'u')}"
+        logger.error "#{sock.object_id} #{sid} Exception\n#{error_message($!, 'u')}"
       end
       
-      logger.info "#{sock.inspect} #{sid} is closed"
+      logger.info "#{sock.object_id} #{sid} is closed"
       
       socket_list.delete(sid)     # ソケットリストから削除
       
-      @mutex_sock[sock.object_id].synchronize do
-        sock = nil   # ソケット削除
+      sock_object_id = sock.object_id
+      begin
+        @mutex_sock[sock_object_id].timeout_sync(2, "delete #{sock.object_id}") do
+          sock = nil   # ソケット削除
+        end
+      rescue DangoFrameworkMutexTimeoutException
+        logger.warn "DangoFrameworkMutexTimeoutException #{sock.object_id} #{sid}\n" + 
+                    "#{error_message($!, 'u')}"
+      rescue Exception
+        logger.error "#{sock.object_id} #{sid} Exception\n#{error_message($!, 'u')}"
       end
       
-      @mutex_sock.delete(sock.object_id) # ソケット排他処理から削除
+      @mutex_sock.delete(sock_object_id) # ソケット排他処理から削除
       
       @session_manager.close_session(sid) # sessionの終了処理
     end
@@ -492,7 +542,7 @@ class DangoServerFramework
 EOF
 
       logger.debug policy_data
-      @mutex_sock[sock.object_id].synchronize do
+      @mutex_sock[sock.object_id].timeout_sync(2, "policy_send #{sock.object_id}") do
         sock.write policy_data
         sock.flush
       end
@@ -501,20 +551,6 @@ EOF
     end
   end
   
-  # action_nameが送信パケットの返事なら
-  def action_name_is_response(ret_obj)
-    logger.debug "action_name_is_response: #{ret_obj['_id'].inspect} #{session[:sid].inspect}"
-    @mutex_send_response.synchronize do
-      @arr_send_response.push({:_id => ret_obj["_id"], :_sid => session[:sid], :time => Time.now})
-    end
-  end
-  
-  # action_nameが送信パケットの返事なら
-  def action_name_is_change_encode_type(ret_obj)
-    logger.debug "action_name_is_change_encode_type: #{ret_obj['encode_type'].inspect}"
-    session[:encode_type] = ret_obj['encode_type']
-  end
-  
   # 各種サーバー仕様の変数設定
   def set_server_variables() 
     # 変数の初期設定
@@ -552,7 +588,7 @@ EOF
     @statistics_process_memory = @config['server']['statistics_process_memory'] || false # プロセスのメモリ使用量統計取る
     
     @backdoor_run_drb_server   = @config['server']['backdoor_run_drb_server'] || false # drbサーバーを起動するか
-    @backdoor_run_drb_port     = @config['server']['backdoor_run_drb_port'] || 12345   # drbサーバーのポート番号
+    @backdoor_run_drb_url      = @config['server']['backdoor_run_drb_url'] || 12345    # drbサーバーのポート番号
     @backdoor_host             = @config['server']['backdoor_host'] || '127.0.0.1'     # バックドアのアクセス可能ホスト名
     
     @loop_setting              = @config['server']['loop_setting']                     # メインループ用の定義
@@ -612,6 +648,10 @@ EOF
     @session_manager.session
   end
   
+  def session_list
+    @session_manager.session_list
+  end
+  
   
   # ソケットとIDの対応
   def socket_list_init
@@ -633,7 +673,7 @@ EOF
   def dango_server_send_encrypt(str) # 継承用
     str
   end
-  def dango_send_encrypt(str) # フレームワークから呼ばれる部分
+  def send_encrypt(str) # フレームワークから呼ばれる部分
     dango_server_send_encrypt(str)
   end
   
@@ -641,32 +681,38 @@ EOF
   def dango_server_receive_decrypt(str) # 継承用
     str
   end
-  def dango_receive_decrypt(str) # フレームワークから呼ばれる部分
+  def receive_decrypt(str) # フレームワークから呼ばれる部分
     dango_server_receive_decrypt(str)
   end
   
   # GCスレッドの開始(メモリ使用量統計も含む)
   def gc_thread_start
     th = Thread.start do
+      Thread.current[:_name] = "thread_gc"
       loop do
         begin
           sleep @gc_interval_sec
           
-          @mutex_proc_thread.synchronize do
-            logger.debug "GC start "
+          logger.debug "mpth:1:GC start mutex start"
+          @mutex_proc_thread.timeout_sync(4, :gc) do
+            logger.debug "mpth:2:GC start #{Time.now_to_s}"
             gc_start_time = Time.now
             GC.enable
             GC.start 
             GC.disable
             logger.debug "GC end #{Time.now - gc_start_time}sec "
           end
+          logger.debug "GC start mutex end"
           
           output_memory_statistics() if @statistics_process_memory # メモリ使用量統計
           
+        rescue DangoFrameworkMutexTimeoutException
+          logger.warn "DangoFrameworkMutexTimeoutException gc_thread_start" + 
+                       "#{error_message($!, 'u')}"
         rescue Exception
           logger.error "Exception gc_thread_start  #{error_message($!, 'u')}"
         end
-      end
+      end # loop end
     end
     th.priority = 3
   end
@@ -699,16 +745,16 @@ EOF
     shared[:_heart_beat_time_hash] = {}
     
     th = Thread.start do
-      count = 0
+      Thread.current[:_name] = "thread_heart_beat"
       loop do
         begin
           sleep @herat_beat_interval_sec
           
           next if @gserver.stopped? # gserverが起動していればスキップ
           
-          logger.debug "heart_beat_thread_start "
-          
-          @mutex_proc_thread.synchronize do
+          logger.debug "mpth:1:heart_beat_thread_start "
+          @mutex_proc_thread.timeout_sync(4, :heart_beat_thread) do
+            logger.debug "mpth:2:heart_beat_thread_start "
             # クライアントから来るハートビートのチェック
             heart_beat_time_hash = shared[:_heart_beat_time_hash]
             
@@ -717,62 +763,53 @@ EOF
               sock = socket_list[sid]
               
               if ! sock # すでにsocketが無い場合は、スキップ
-                logger.warn "!!!! heart_beat_thread_start:sock is not found. #{sid}:#{sock}"
+                logger.warn "!!!! heart_beat_thread_start:sock is not found. #{sid}:#{sock.object_id}"
+                shared.transaction(:_heart_beat_time_hash) do |hbt_hash|
+                  hbt_hash.delete(sid)
+                  shared.commit(hbt_hash)
+                end
                 next 
               end
               
               # クライアントから来たハートビートの時間チェックし、古ければsocketを閉じる
               if heart_beat_time_hash[sid] && 
                  heart_beat_time_hash[sid] < Time.now - @heart_beat_receive_wait_sec
+                
                 logger.warn "!!!! heart_beat_thread_start:heart beat old:#{sid}:" + 
-                            "#{heart_beat_time_hash[sid]}:#{sock}:#{sock.closed?}"
+                            "#{heart_beat_time_hash[sid]}:#{sock.object_id}:#{sock.closed?}"
+                
+                shared.transaction(:_heart_beat_time_hash) do |hbt_hash|
+                  hbt_hash.delete(sid)
+                  shared.commit(hbt_hash)
+                end
+                
                 begin
-                  @mutex_sock[sock.object_id].synchronize do
+                  @mutex_sock[sock.object_id].timeout_sync(2, "hb_close #{sock.object_id}") do
                     sock.close
-                  end # @mutex_sock[sock.object_id].synchronize
+                  end 
                 rescue IOError
-                  logger.debug "heart_beat_thread_start:maybe already closed. IOError:#{sid}:#{sock}"
-                rescue
+                  logger.debug "heart_beat_thread_start:maybe already closed. IOError:#{sid}:#{sock.object_id}"
+                rescue Exception
                   logger.warn "heart_beat_thread_start:failed close socket:#{error_message($!, 'u')}"
                 end
                 next
               end
             end # socket_list
+            
           end # @mutex_proc_thread
+          logger.debug "heart_beat_thread_start end"
           
-#          # スレッド情報をログに出力
-#          if logger.debug?
-#            count += 1
-#            if count % 100 == 0
-#              sids = socket_list.all_sid
-#              thr_list = @thr_gr_gserver_serve.list.collect{|th| th.object_id }
-#              
-#              logger.debug "heart_beat_thread_start:sids=#{sids.inspect}"
-#              logger.debug "heart_beat_thread_start:thr_list=#{thr_list.inspect}"
-#              logger.debug "heart_beat_thread_start:sids-thr_list=#{(sids-thr_list).inspect}"
-#              logger.debug "heart_beat_thread_start:thr_list-sids=#{(thr_list-sids).inspect}"
-#            end
-#          end
-          
+        rescue DangoFrameworkMutexTimeoutException
+          logger.warn "DangoFrameworkMutexTimeoutException heart_beat_thread_start" + 
+                       "#{error_message($!, 'u')}"
         rescue Exception
           logger.error "Exception heart_beat_thread_start  #{error_message($!, 'u')}"
         end
-      end
+      end # loop end
     end
     th.priority = 1
   end
   
-  # クライアントから来たheart_beatなら
-  def dango_receive__notice_heart_beat(ret_obj)
-    logger.debug "dango_receive__notice_heart_beat:#{session[:sid]}:#{ret_obj['_hb_id']} " 
-    
-    shared.transaction(:_heart_beat_time_hash) do |heart_beat_time_hash|
-      heart_beat_time_hash[session[:sid]] = Time.now
-      heart_beat_time_hash
-      shared.commit(heart_beat_time_hash)
-    end
-  end
-  
   # サーバーからクライアントへのデータ返信
   def send_return(send_obj, options = {})
     logger.info "send_return(#{session[:sid].inspect}) "
@@ -805,8 +842,15 @@ EOF
       send_id = notice_name.to_s + Time.now.to_f.to_s + rand().to_s
       send_obj_dup["_id"] = send_id
       
-      @mutex_send_notice.synchronize do # queue_send_noticeに排他処理
-        @queue_send_notice.push([sid, send_obj_dup, encode_type].deep_dup)
+      # queue_send_noticeに排他処理
+      begin
+        @mutex_send_notice.timeout_sync(10, :queue_push) do 
+          @queue_send_notice.push([sid, send_obj_dup, encode_type].deep_dup)
+        end
+      rescue DangoFrameworkMutexTimeoutException
+        logger.warn "mutex_send_notice #{error_message($!, 'u')}"
+      rescue Exception
+        logger.error "mutex_send_notice #{error_message($!, 'u')}"
       end
     end # end sids.each
     
@@ -815,6 +859,7 @@ EOF
   
   # thread for send_notice_queue
   def thread_send_notice_queue()
+    Thread.current[:_name] = "thread_send_notice_queue"
     loop do
       begin
         before_time = Time.now
@@ -824,7 +869,8 @@ EOF
         pop_data = @queue_send_notice.pop
         send_data_list.push(pop_data) # Queueデータ待ち
         
-        @mutex_send_notice.synchronize do # queue_send_noticeに排他処理(sizeとpopの実行の間にpushが起きないように)
+        # queue_send_noticeに排他処理(sizeとpopの実行の間にpushが起きないように)
+        @mutex_send_notice.timeout_sync(10, :queue_pop) do 
           @queue_send_notice.size.times do 
 #            while(! @queue_send_notice.empty?) do       # 残りのデータもQueueに入れていく
             pop_data = @queue_send_notice.pop
@@ -848,25 +894,32 @@ EOF
           sock = socket_list[sid]
           
           if !sock
-            logger.info "thread_send_notice_queue:not found sid=#{sid} sock=#{sock.inspect}"
+            logger.info "thread_send_notice_queue:not found sid=#{sid} sock=#{sock.object_id}"
             next
           end
           
           if sock.closed?
-            logger.info "thread_send_notice_queue:socket closed. sid=#{sid} sock=#{sock.inspect}"
+            logger.info "thread_send_notice_queue:socket closed. sid=#{sid} sock=#{sock.object_id}"
             next
           end
           
           begin
-            logger.debug "thread_send_notice_queue:dango_send_data:sid=#{sid} sock=#{sock.inspect} " + 
+            logger.debug "thread_send_notice_queue:dango_send_data:sid=#{sid} sock=#{sock.object_id} " + 
                          "#{one_send_data_list.collect{|d| d['_notice_name'] + ','}}"
             
-            @mutex_sock[sock.object_id].synchronize do
+            @mutex_sock[sock.object_id].timeout_sync(2, "send #{sock.object_id}") do
               dango_send_data(sock, one_send_data_list, :type=>send_data_encode_type_list[sid]) # データ送信
-            end # mutex_sock[sock.object_id].synchronize
+            end
             
           rescue DangoFrameworkConnectionException
-            logger.info "thread_send_notice_queue:failed send. sid=#{sid} sock=#{sock.inspect} #{$!.class}"
+            logger.info "thread_send_notice_queue:failed send. sid=#{sid} " + 
+                        "sock=#{sock.object_id} #{$!.class}"
+          rescue DangoFrameworkMutexTimeoutException
+            logger.warn "thread_send_notice_queue:failed send. sid=#{sid} " + 
+                        "sock=#{sock.object_id} #{$!.class}"
+          rescue Exception
+            logger.error "thread_send_notice_queue:unknown failed. sid=#{sid} " + 
+                         "sock=#{sock.object_id} #{error_message($!, 'u')}"
           end
         end
         
@@ -877,10 +930,13 @@ EOF
         
         sleep @send_receive_sleep_interval_sec # スリープ
         
+      rescue DangoFrameworkMutexTimeoutException
+        logger.warn "ERROR:thread_send_notice_queue:DangoFrameworkMutexTimeoutException" + 
+                    "#{error_message($!, 'u')}"
       rescue Exception
-        logger.warn "ERROR:thread_send_notice_queue:Exception:#{error_message($!, 'u')}"
+        logger.error "ERROR:thread_send_notice_queue:Exception:#{error_message($!, 'u')}"
       end
-    end
+    end # loop end
   end
   
   # 必要に応じて追加するメソッド
@@ -903,9 +959,10 @@ EOF
   
   # メインループ用の定義をするためのメソッド
   def set_loop_setting(options={})
-    logger.debug "loop_setting options:#{options.inspect}"
     options = options.deep_dup
     
+    shared[:_is_server_sleep] = nil # メンテナンス停止用
+    
     if options["method_name"] # メソッド名が無ければ処理しない
       method_name = options["method_name"]
       
@@ -914,23 +971,36 @@ EOF
       logger.warn "set_loop_setting wait_sec:#{wait_sec.inspect} method_name=#{method_name}"
       
       th = Thread.new do 
-        begin
-          loop do
+        logger.debug "set_loop_setting start"
+        count = 0
+        
+        loop do
+          begin
             before_time = Time.now
-            @mutex_proc_thread.synchronize do
-              begin
-                __send__(method_name)
-              rescue Exception
-                logger.error "set_loop_setting error:#{error_message($!, 'u')}"
-              end
-            end
             
-            real_wait_sec = wait_sec - (Time.now - before_time)
+            logger.debug "mpth:1:sls start"
+            @mutex_proc_thread.timeout_sync(4, "set_loop_setting_send_#{method_name}") do
+              __send__(method_name)
+            end # mutex_proc_thread
+            logger.debug "mpth:2:sls end"
+            # if count % 1000 == 0
             
+            real_wait_sec = wait_sec - (Time.now - before_time)
             sleep real_wait_sec if real_wait_sec > 0
+            
+            count += 1
+            count = 0 if count >= 10000000
+            
+            # メンテナンス機能
+            if shared[:_is_server_sleep]
+              logger.info "set_loop_setting SLEEP !!!"
+              sleep 10000 
+            end
+          rescue DangoFrameworkMutexTimeoutException
+            logger.warn "set_loop_setting error:#{error_message($!, 'u')}"
+          rescue Exception
+            logger.error "set_loop_setting error:#{error_message($!, 'u')}"
           end
-        rescue Exception
-          logger.error "set_loop_setting error:#{error_message($!, 'u')}"
         end
       end # th.end
     end