bayserver-core 2.3.2 → 3.0.1

data/lib/baykit/bayserver/agent/grand_agent.rb

@@ -2,12 +2,18 @@ require 'socket'
 require 'objspace'
 
 require 'baykit/bayserver/sink'
-require 'baykit/bayserver/agent/accept_handler'
 require 'baykit/bayserver/agent/command_receiver'
-require 'baykit/bayserver/agent/grand_agent_monitor'
-require 'baykit/bayserver/agent/spin_handler'
+require 'baykit/bayserver/agent/letter'
+require 'baykit/bayserver/agent/multiplexer/spider_multiplexer'
+require 'baykit/bayserver/agent/multiplexer/spin_multiplexer'
+require 'baykit/bayserver/agent/multiplexer/job_multiplexer'
+require 'baykit/bayserver/agent/multiplexer/taxi_multiplexer'
+require 'baykit/bayserver/agent/multiplexer/rudder_state'
+require 'baykit/bayserver/agent/monitor/grand_agent_monitor'
 require 'baykit/bayserver/agent/signal/signal_agent'
 
+require 'baykit/bayserver/docker/harbor'
+
 require 'baykit/bayserver/train/train_runner'
 require 'baykit/bayserver/taxi/taxi_runner'
 
@@ -22,6 +28,8 @@ module Baykit
         include Baykit::BayServer::Train
         include Baykit::BayServer::Taxi
         include Baykit::BayServer::Agent::Signal
+        include Baykit::BayServer::Agent::Multiplexer
+        include Baykit::BayServer::Docker
         include Baykit::BayServer::Util
 
         SELECT_TIMEOUT_SEC = 10
@@ -32,66 +40,88 @@ module Baykit
         CMD_MEM_USAGE = 3
         CMD_SHUTDOWN = 4
         CMD_ABORT = 5
+        CMD_CATCHUP = 6
 
         attr :agent_id
         attr :anchorable
-        attr :non_blocking_handler
-        attr :spin_handler
-        attr :accept_handler
+        attr :net_multiplexer
+        attr :job_multiplexer
+        attr :taxi_multiplexer
+        attr :spin_multiplexer
+        attr :spider_multiplexer
+        attr :job_multiplexer
+        attr :taxi_multiplexer
+        attr :recipient
+
         attr :send_wakeup_pipe
-        attr :select_wakeup_pipe
-        attr :select_timeout_sec
         attr :max_inbound_ships
-        attr :selector
         attr :unanchorable_transporters
         attr :aborted
         attr :command_receiver
         attr :timer_handlers
+        attr :last_timeout_check
+        attr :letter_queue
+        attr :letter_queue_lock
+        attr :postpone_queue
+        attr :postpone_queue_lock
 
         class << self
           attr :agents
           attr :listeners
           attr :agent_count
-          attr :anchorable_port_map
-          attr :unanchorable_port_map
           attr :max_ships
           attr :max_agent_id
-          attr :multi_core
         end
 
         # Class variables
         @agent_count = 0
         @max_agent_id = 0
         @max_ships = 0
-        @multi_core = false
 
         @agents = []
         @listeners = []
 
-        @anchorable_port_map = {}
-        @unanchorable_port_map = {}
         @finale = false
 
         def initialize (agent_id, max_ships, anchorable)
           @agent_id = agent_id
+          @max_inbound_ships = max_ships
           @anchorable = anchorable
-
           @timer_handlers = []
-          if @anchorable
-            @accept_handler = AcceptHandler.new(self, GrandAgent.anchorable_port_map)
-          else
-            @accept_handler = nil
+          @select_timeout_sec = SELECT_TIMEOUT_SEC
+          @aborted = false
+          @letter_queue = []
+          @letter_queue_lock = Mutex.new
+          @postpone_queue = []
+          @postpone_queue_lock = Mutex.new
+
+          @spider_multiplexer = SpiderMultiplexer.new(self, anchorable)
+          @spin_multiplexer = SpinMultiplexer.new(self)
+          @job_multiplexer = JobMultiplexer.new(self, anchorable)
+          @taxi_multiplexer = TaxiMultiplexer.new(self)
+
+          case BayServer.harbor.recipient
+          when Harbor::RECIPIENT_TYPE_SPIDER
+            @recipient = @spider_multiplexer
+
+          when Harbor::RECIPIENT_TYPE_PIPE
+            raise NotImplementedError.new
           end
 
-          @spin_handler = SpinHandler.new(self)
-          @non_blocking_handler = NonBlockingHandler.new(self)
+          case BayServer.harbor.net_multiplexer
+          when Harbor::MULTIPLEXER_TYPE_SPIDER
+            @net_multiplexer = @spider_multiplexer
 
-          @select_timeout_sec = SELECT_TIMEOUT_SEC
-          @max_inbound_ships = max_ships
-          @selector = Selector.new()
-          @aborted = false
-          @unanchorable_transporters = {}
+          when Harbor::MULTIPLEXER_TYPE_JOB
+            @net_multiplexer = @job_multiplexer
+
+          when Harbor::MULTIPLEXER_TYPE_PIGEON, Harbor::MULTIPLEXER_TYPE_SPIN,
+               Harbor::MULTIPLEXER_TYPE_TAXI, Harbor::MULTIPLEXER_TYPE_TRAIN
 
+            raise Sink.new("Multiplexer not supported: %s", Harbor.get_multiplexer_type_name(BayServer.harbor.net_multiplexer))
+          end
+
+          @last_timeout_check = 0
         end
 
 
@@ -100,17 +130,41 @@ module Baykit
         end
 
 
-        def inspect()
+        def inspect
           return to_s
         end
 
+        #########################################
+        # Custom methods
+        #########################################
+        def start
+          Thread.new do
+            run
+          end
+        end
+
         def run
           BayLog.info(BayMessage.get(:MSG_RUNNING_GRAND_AGENT, self))
-          @select_wakeup_pipe = IO.pipe
-          @selector.register(@select_wakeup_pipe[0], Selector::OP_READ)
-          @selector.register(@command_receiver.communication_channel, Selector::OP_READ)
+
+          if @net_multiplexer.is_non_blocking
+            BayLog.info("rec=%s", @command_receiver)
+            @command_receiver.rudder.set_non_blocking
+          end
+
+          @net_multiplexer.req_read(@command_receiver.rudder)
+
+          if @anchorable
+            # Adds server socket channel of anchorable ports
+            BayServer.anchorable_port_map.keys.each do |rd|
+              if @net_multiplexer.is_non_blocking
+                rd.set_non_blocking
+              end
+              @net_multiplexer.add_rudder_state(rd, RudderState.new(rd))
+            end
+          end
 
           # Set up unanchorable channel
+=begin
           for ch in GrandAgent.unanchorable_port_map.keys() do
             port_dkr = GrandAgent.unanchorable_port_map[ch]
             tp = port_dkr.new_transporter(self, ch)
@@ -121,124 +175,166 @@ module Baykit
               @non_blocking_handler.ask_to_read(ch)
             end
           end
+=end
 
           busy = true
           begin
-            while not @aborted
-
-              count = -1
-
-              if @accept_handler
-                test_busy = @accept_handler.ch_count >= @max_inbound_ships
-                if test_busy != busy
-                  busy = test_busy
-                  if busy
-                    @accept_handler.on_busy()
-                  else
-                    @accept_handler.on_free()
-                  end
-                end
-              end
+            while true
 
-              if !busy && @selector.count() == 2
-                # agent finished
-                BayLog.debug("%s Selector has no key", self)
-                break
+              test_busy = @net_multiplexer.is_busy
+              if test_busy != busy
+                busy = test_busy
+                if busy
+                  @net_multiplexer.on_busy
+                else
+                  @net_multiplexer.on_free
+                end
               end
 
-              if !@spin_handler.empty?
-                timeout = 0
+              if not @spin_multiplexer.is_empty
+                # If "SpinHandler" is running, the select function does not block.
+                received = @recipient.receive(false)
+                @spin_multiplexer.process_data
               else
-                timeout = @select_timeout_sec
+                received = @recipient.receive(true)
               end
 
               if @aborted
                 BayLog.info("%s aborted by another thread", self)
-                break
+                break;
               end
-              #@BayServer.debug("Selecting... read=" + read_list.to_s)
-              selected_map = @selector.select(timeout)
-              #BayLog.debug("%s selected: %s", self, selected_map)
 
-              processed = @non_blocking_handler.register_channel_ops() > 0
-
-              if @aborted
-                BayLog.info("%s aborted by another thread", self)
-                break
-              end
-
-              if selected_map.length == 0
-                # No channel is selected
-                processed |= @spin_handler.process_data()
+              if @spin_multiplexer.is_empty && @letter_queue.empty?
+                # timed out
+                # check per 10 seconds
+                if Time.now.tv_sec - @last_timeout_check >= 10
+                  ring
+                end
               end
 
-              selected_map.keys().each do |ch|
-                if ch == @select_wakeup_pipe[0]
-                  # Waked up by ask_to_*
-                  on_waked_up(ch)
-                elsif ch == @command_receiver.communication_channel
-                  @command_receiver.on_pipe_readable()
-                elsif @accept_handler && @accept_handler.server_socket?(ch)
-                  @accept_handler.on_acceptable(ch)
-                else
-                  @non_blocking_handler.handle_channel(ch, selected_map[ch])
+              while !@letter_queue.empty?
+                let = nil
+                @letter_queue_lock.synchronize do
+                  let = @letter_queue.shift
                 end
-                processed = true
-              end
 
-              if not processed
-                # timeout check
-                @timer_handlers.each do |h|
-                  h.on_timer()
+                case let.type
+                when Letter::ACCEPTED
+                  on_accept(let)
+                when Letter::CONNECTED
+                  on_connect(let)
+                when Letter::READ
+                  on_read(let)
+                when Letter::WROTE
+                  on_wrote(let)
+                when Letter::CLOSEREQ
+                  on_close_req(let)
                 end
               end
-
             end # while
 
-          rescue => e
+          rescue Exception => e
             BayLog.fatal_e(e)
           ensure
             BayLog.info("%s end", self)
-            shutdown()
+            shutdown
           end
         end
 
-        def shutdown()
-          BayLog.info("%s shutdown", self)
 
-          if @accept_handler != nil
-            @accept_handler.shutdown()
-          end
 
-          @command_receiver.end()
-          @non_blocking_handler.close_all()
+        def abort_agent
+          BayLog.info("%s abort", self)
 
-          GrandAgent.listeners.each do |lis|
-            lis.remove(self)
+          if BayServer.harbor.multi_core
+            exit(1)
           end
+        end
 
-          GrandAgent.agents.delete(@agent_id)
+        def req_shutdown
+          @aborted = true
+          @recipient.wakeup
+        end
 
-          @agent_id = -1
-          if BayServer.harbor.multi_core
-            exit(1)
+
+        def print_usage
+          # print memory usage
+          BayLog.info("%s MemUsage", self)
+          BayLog.info("  Ruby version: %s", RUBY_VERSION)
+          memsize = ObjectSpace.memsize_of_all
+          # formatted by comma
+          msize_comma = memsize.to_s.reverse.gsub(/(\d{3})(?=\d)/, '\\1,').reverse.then do |str|
+            str[0] == ',' ? str[1..-1] : str
           end
+          BayLog.info("  Total object size: %s bytes", msize_comma)
+          MemUsage.get(@agent_id).print_usage(1)
         end
 
-        def abort_agent()
-          BayLog.info("%s abort", self)
+
+        def add_timer_handler(handler)
+          @timer_handlers << handler
+        end
+
+        def remove_timer_handler(handler)
+          @timer_handlers.delete(handler)
+        end
+
+        def add_command_receiver(rd)
+          @command_receiver = CommandReceiver.new()
+          com_transporter = PlainTransporter.new(@net_multiplexer, @command_receiver, true, 8, false)
+          @command_receiver.init(@agent_id, rd, com_transporter)
+          @net_multiplexer.add_rudder_state(@command_receiver.rudder, RudderState.new(@command_receiver.rudder, com_transporter))
+          BayLog.info("ComRec=%s", @command_receiver)
+        end
+
+        def send_accepted_letter(st, client_rd, e, wakeup)
+          send_letter(Letter.new(Letter::ACCEPTED, st, client_rd, -1, nil, e), wakeup)
+        end
+
+        def send_connected_letter(st, e, wakeup)
+          send_letter(Letter.new(Letter::CONNECTED, st, nil, -1, nil, e), wakeup)
+        end
+        def send_read_letter(st, n, adr, e, wakeup)
+          send_letter(Letter.new(Letter::READ, st, nil, n, adr, e), wakeup)
+        end
+
+        def send_wrote_letter(st, n, e, wakeup)
+          send_letter(Letter.new(Letter::WROTE, st, nil, n, nil, e), wakeup)
+        end
+
+        def send_close_req_letter(st, wakeup)
+          send_letter(Letter.new(Letter::CLOSEREQ, st, nil, -1, nil, nil), wakeup)
+        end
+
+        def shutdown
+          BayLog.info("%s shutdown", self)
+          if @aborted
+            return
+          end
+
+          @aborted = true
+          BayLog.debug("%s shutdown netMultiplexer", self)
+          @net_multiplexer.shutdown()
+
+          GrandAgent.listeners.each do |lis|
+            lis.remove(@agent_id)
+          end
+          @command_receiver.end()
+          GrandAgent.agents.delete(@agent_id)
 
           if BayServer.harbor.multi_core
+            BayLog.debug("%s exit", self)
             exit(1)
           end
+          @agent_id = -1
         end
 
-        def req_shutdown()
-          @aborted = true
-          wakeup()
+        def abort
+          BayLog.fatal("%s abort", self)
         end
 
-        def reload_cert()
+
+        def reload_cert
           GrandAgent.anchorable_port_map.values().each do |port|
             if port.secure()
               begin
@@ -250,53 +346,265 @@ module Baykit
           end
         end
 
-        def print_usage()
-          # print memory usage
-          BayLog.info("%s MemUsage", self)
-          BayLog.info("  Ruby version: %s", RUBY_VERSION)
-          memsize = ObjectSpace.memsize_of_all
-          # formatted by comma
-          msize_comma = memsize.to_s.reverse.gsub(/(\d{3})(?=\d)/, '\\1,').reverse.then do |str|
-            str[0] == ',' ? str[1..-1] : str
+        def add_postpone(p)
+          @postpone_queue_lock.synchronize do
+            @postpone_queue << p
           end
-          BayLog.info("  Total object size: %s bytes", msize_comma)
-          MemUsage.get(@agent_id).print_usage(1)
         end
 
-        def wakeup
-          #BayLog.debug("%s wakeup", self)
-          IOUtil.write_int32(@select_wakeup_pipe[1], 0)
+        def count_postpone
+          return @postpone_queue.length
         end
 
-        def run_command_receiver(com_channel)
-          @command_receiver = CommandReceiver.new(self, com_channel)
+        def req_catch_up
+          BayLog.debug("%s Req catchUp", self)
+          if count_postpone > 0
+            catch_up
+          else
+            begin
+              @command_receiver.send_command_to_monitor(self, CMD_CATCHUP, false)
+            rescue IOError => e
+              BayLog.error_e(e)
+              abort
+            end
+          end
         end
 
-        def add_timer_handler(handler)
-          @timer_handlers << handler
+        def catch_up
+          BayLog.debug("%s catchUp", self)
+          @postpone_queue_lock.synchronize do
+            if not @postpone_queue.empty?
+              r = @postpone_queue.shift
+              r.run()
+            end
+          end
         end
 
-        def remove_timer_handler(handler)
-          @timer_handlers.delete(handler)
+        #########################################
+        # Private methods
+        #########################################
+        private
+        def ring
+          BayLog.trace("%s Ring", self)
+          # timeout check
+          @timer_handlers.each do |h|
+            h.on_timer
+          end
+          @last_timeout_check = Time.now.tv_sec
         end
 
-        private
-        def on_waked_up(pipe_fd)
-          #BayLog.debug("%s waked up", self)
-          val = IOUtil.read_int32(pipe_fd)
+        def send_letter(let, wakeup)
+          @letter_queue_lock.synchronize do
+            @letter_queue << let
+          end
+
+          if wakeup
+            @recipient.wakeup
+          end
+        end
+
+        def on_accept(let)
+          begin
+            if let.err != nil
+              raise let.err
+            end
+
+            p = BayServer::anchorable_port_map[let.state.rudder]
+            p.on_connected(@agent_id, let.client_rudder)
+          rescue IOError => e
+            BayLog.error_e(e)
+            next_action(let.state, NextSocketAction::CLOSE, false)
+          rescue HttpException => e
+            BayLog.error_e(e)
+            let.client_rudder.close
+          end
+
+          if !@net_multiplexer.is_busy
+            let.state.multiplexer.next_accept(let.state)
+          end
         end
 
+        def on_connect(let)
+          st = let.state
+          if st.closed
+            BayLog.debug("%s Rudder is already closed: rd=%s", self, st.rudder);
+            return;
+          end
+
+          BayLog.debug("%s connected rd=%s", self, st.rudder)
+          next_act = nil
+          begin
+            if let.err != nil
+              raise let.err
+            end
+
+            next_act = st.transporter.on_connect(st.rudder)
+            BayLog.debug("%s nextAct=%s", self, next_act)
+          rescue IOError => e
+            st.transporter.on_error(st.rudder, e)
+            next_act = NextSocketAction::CLOSE
+          end
+
+          if next_act == NextSocketAction::READ
+            # Read more
+            st.multiplexer.cancel_write(st)
+          end
+
+          next_action(st, next_act, false)
+        end
+
+        def on_read(let)
+          st = let.state
+          if st.closed
+            BayLog.debug("%s Rudder is already closed: rd=%s", self, st.rudder)
+            return
+          end
+
+          begin
+            if let.err != nil
+              BayLog.debug("%s error on OS read %s", self, let.err)
+              raise let.err
+            end
+
+            BayLog.debug("%s read %d bytes (rd=%s)", self, let.n_bytes, st.rudder)
+            st.bytes_read += let.n_bytes
+
+            if let.n_bytes <= 0
+              st.read_buf.clear
+              next_act = st.transporter.on_read(st.rudder, "", let.address)
+            else
+              next_act = st.transporter.on_read(st.rudder, st.read_buf, let.address)
+            end
+
+          rescue => e
+            st.transporter.on_error(st.rudder, e)
+            next_act = NextSocketAction::CLOSE
+          end
+
+          next_action(st, next_act, true)
+        end
+
+        def on_wrote(let)
+          st = let.state
+          if st.closed
+            BayLog.debug("%s Rudder is already closed: rd=%s", self, st.rudder)
+            return
+          end
+
+          begin
+            if let.err != nil
+              BayLog.debug("%s error on OS write %s", self, let.err)
+              raise let.err
+            end
+
+            BayLog.debug("%s wrote %d bytes rd=%s qlen=%d", self, let.n_bytes, st.rudder, st.write_queue.length)
+            st.bytes_wrote += let.n_bytes
+
+            if st.write_queue.empty?
+              raise Sink("%s Write queue is empty: rd=%s", self, st.rudder)
+            end
+
+            unit = st.write_queue[0]
+            if unit.buf.length > 0
+              BayLog.debug("Could not write enough data buf_len=%d", unit.buf.length)
+            else
+              st.multiplexer.consume_oldest_unit(st)
+            end
+
+            write_more = true
+
+            st.writing_lock.synchronize do
+              if st.write_queue.empty?
+                write_more = false
+                st.writing = false
+              end
+            end
+
+            if write_more
+              st.multiplexer.next_write(st)
+            else
+              if st.finale
+                # close
+                BayLog.debug("%s finale return Close", self)
+                next_action(st, NextSocketAction::CLOSE, false)
+              else
+                # Write off
+                st.multiplexer.cancel_write(st)
+              end
+            end
+          rescue SystemCallError, IOError => e
+            BayLog.debug("%s IO error on wrote", self)
+            st.transporter.on_error(st.rudder, e)
+            next_action(st, NextSocketAction::CLOSE, false)
+          end
+        end
+
+        def on_close_req(let)
+          st = let.state
+          BayLog.debug("%s reqClose rd=%s", self, st.rudder)
+          if st.closed
+            BayLog.debug("%s Rudder is already closed: rd=%s", self, st.rudder)
+            return
+          end
+
+          st.multiplexer.close_rudder(st)
+          st.access
+        end
+
+        def next_action(st, act, reading)
+          BayLog.debug("%s next action: %s (reading=%s)", self, act, reading)
+          cancel = false
+
+          case(act)
+          when NextSocketAction::CONTINUE
+            if reading
+              st.multiplexer.next_read(st)
+            end
+
+          when NextSocketAction::READ
+            st.multiplexer.next_read(st)
+
+          when NextSocketAction::WRITE
+            if reading
+              cancel = true
+            end
+
+          when NextSocketAction::CLOSE
+            if reading
+              cancel = true
+            end
+            st.multiplexer.close_rudder(st)
+
+          when NextSocketAction::SUSPEND
+            if reading
+              cancel = true
+            end
+
+          else
+            raise ArgumentError.new("Invalid action: #{act}")
+
+          end
+
+          if cancel
+            st.multiplexer.cancel_read(st)
+            st.reading_lock.synchronize do
+              BayLog.debug("%s Reading off %s", self, st.rudder)
+              st.reading = false
+            end
+          end
+
+          st.access
+        end
+
+
         ######################################################
         # class methods
         ######################################################
-        def GrandAgent.init(agt_ids, anchorable_port_map, unanchorable_port_map, max_ships, multi_core)
+        def GrandAgent.init(agt_ids, max_ships)
           @agent_count = agt_ids.length
-          @anchorable_port_map = anchorable_port_map
-          @unanchorable_port_map = unanchorable_port_map != nil ? unanchorable_port_map : {}
           @max_ships = max_ships
-          @multi_core = multi_core
 
-          if(BayServer.harbor.multi_core?)
+          if BayServer.harbor.multi_core
             agt_ids.each do | agt_id |
               add(agt_id, true)
             end
@@ -320,8 +628,10 @@ module Baykit
           @agents[agt_id] = agt
 
           @listeners.each do |lis|
-            lis.add(agt)
+            lis.add(agt.agent_id)
           end
+
+          return agt
         end
 
         def GrandAgent.add_lifecycle_listener(lis)