bayserver-core 3.0.2 → 3.3.1

data/lib/baykit/bayserver/docker/built_in/built_in_log_docker.rb

@@ -1,12 +1,11 @@
 require 'baykit/bayserver/agent/grand_agent'
 require 'baykit/bayserver/agent/lifecycle_listener'
 require 'baykit/bayserver/agent/multiplexer/plain_transporter'
-require 'baykit/bayserver/agent/multiplexer/rudder_state'
+require 'baykit/bayserver/common/rudder_state'
 require 'baykit/bayserver/rudders/io_rudder'
 require 'baykit/bayserver/docker/built_in/write_file_taxi'
 require 'baykit/bayserver/docker/log'
 require 'baykit/bayserver/docker/built_in/log_items'
-require 'baykit/bayserver/docker/built_in/log_boat'
 require 'baykit/bayserver/util/string_util'
 
 module Baykit
@@ -20,11 +19,20 @@ module Baykit
             include Baykit::BayServer::Util
             include Baykit::BayServer::Bcf
             include Baykit::BayServer::Agent
+            include Baykit::BayServer::Common
+
+            class LoggerInfo
+              attr_accessor :file_name
+              attr_accessor :file_size
+              attr_accessor :rudder
+              attr_accessor :multiplexer
+              attr_accessor :rudder_state
+              attr_accessor :registered
+            end
 
             class AgentListener
               include Baykit::BayServer::Agent::Multiplexer
               include Baykit::BayServer::Agent::LifecycleListener  # implements
-              include Baykit::BayServer::Agent::Transporter
               include Baykit::BayServer::Agent
               include Baykit::BayServer::Rudders
 
@@ -35,17 +43,19 @@ module Baykit
               end
 
               def add(agt_id)
-                file_name = "#{@log_docker.file_prefix}_#{agt_id}.#{@log_docker.file_ext}"
-                size = 0
-                if ::File.exist?(file_name)
-                  size = ::File.size(file_name)
+                info = LoggerInfo.new()
+                info.file_name = "#{@log_docker.file_prefix}_#{agt_id}.#{@log_docker.file_ext}"
+                info.file_size = 0
+
+                if ::File.exist?(info.file_name)
+                  info.file_size = ::File.size(info.file_name)
                 end
                 agt = GrandAgent.get(agt_id)
 
                 begin
-                  f = File.open(file_name, "a")
+                  f = File.open(info.file_name, "a")
                 rescue => e
-                  BayLog.fatal(BayMessage.get(:INT_CANNOT_OPEN_LOG_FILE, file_name))
+                  BayLog.fatal(BayMessage.get(:INT_CANNOT_OPEN_LOG_FILE, info.file_name))
                   BayLog.fatal_e(e);
                 end
 
@@ -68,20 +78,21 @@ module Baykit
                   raise Sink.new
                 end
 
-                st = RudderState.new(rd)
-                st.bytes_wrote = size
-                mpx.add_rudder_state(rd, st)
+                info.multiplexer = mpx
+                info.rudder = rd
+                info.registered = false
 
-                @log_docker.multiplexers[agt_id] = mpx
-                @log_docker.rudders[agt_id] = rd
+                @log_docker.loggers[agt_id] = info
               end
 
 
               def remove(agt_id)
-                rd = @log_docker.rudders[agt_id]
-                @log_docker.multiplexers[agt_id].req_close(rd)
-                @log_docker.multiplexers[agt_id] = nil
-                @log_docker.rudders[agt_id] = nil
+                info = @log_docker.loggers[agt_id]
+                rd = info.rudder
+                if info.registered
+                  info.multiplexer.req_close(rd)
+                end
+                @log_docker.loggers[agt_id] = nil
               end
             end
 
@@ -95,27 +106,18 @@ module Baykit
             attr :file_prefix
             attr :file_ext
 
-            # Logger for each agent.
-            #    Map of Agent ID => LogBoat
-            attr :loggers
-
             # Log format
             attr :format
 
             # Log items
             attr :log_items
 
-            attr :rudders
-
-            # Multiplexer to write to file
-            attr :multiplexers
+            attr :loggers
 
             def initialize
-              @loggers = {}
               @format = nil
               @log_items = []
-              @rudders = {}
-              @multiplexers = {}
+              @loggers = {}
             end
 
             def init(elm, parent)
@@ -137,8 +139,6 @@ module Baykit
                 @file_prefix = BayServer.get_location @file_prefix
               end
 
-              @loggers = Array.new(BayServer.harbor.grand_agents)
-
               log_dir = File.dirname(@file_prefix)
               if !File.directory?(log_dir)
                 Dir.mkdir(log_dir)
@@ -174,12 +174,20 @@ module Baykit
 
               # If threre are message to write, write it
               if sb.length > 0
-                @multiplexers[tour.ship.agent_id].req_write(
-                  @rudders[tour.ship.agent_id],
+                info = @loggers[tour.ship.agent_id]
+                if info.rudder_state == nil
+                  info.rudder_state = RudderStateStore.get_store(tour.ship.agent_id).rent()
+                  info.rudder_state.init(info.rudder)
+                  info.rudder_state.bytes_wrote = info.file_size
+                  info.multiplexer.add_rudder_state(info.rudder, info.rudder_state)
+                  info.registered = true
+                end
+
+                info.multiplexer.req_write(
+                  info.rudder,
                   sb,
                   nil,
-                  "log",
-                  nil
+                  "log"
                 )
               end
             end

data/lib/baykit/bayserver/docker/harbor.rb

@@ -121,6 +121,20 @@ module Baykit
           raise NotImplementedError
         end
 
+        # True if cache is enabled
+        def enable_cache
+          raise NotImplementedError
+        end
+
+        # Lifespan seconds of cache
+        def cache_lifespan_sec
+          raise NotImplementedError
+        end
+
+        # Cache size
+        def cache_size_mb
+          raise NotImplementedError
+        end
 
         def self.get_multiplexer_type_name(type)
           case type

data/lib/baykit/bayserver/docker/send_file/file_content.rb

@@ -0,0 +1,66 @@
+require 'baykit/bayserver/rudders/rudder'
+require 'baykit/bayserver/util/simple_buffer'
+
+module Baykit
+  module BayServer
+    module Docker
+      module SendFile
+        class FileContent
+
+          attr :path
+          attr :content
+          attr :content_length
+          attr_accessor :bytes_loaded
+          attr :loaded_time
+          attr :waiters
+          attr :lock
+
+          include Baykit::BayServer::Rudders
+          include Baykit::BayServer::Util
+
+
+          def initialize(path, length)
+            @path = path
+            @content = ""
+            @content_length = length
+            @bytes_loaded = 0
+            @loaded_time = Time.now.to_i
+            @waiters = []
+            @lock = Mutex.new
+          end
+
+          def is_loaded()
+            return @bytes_loaded == @content_length
+          end
+
+          def add_waiter(waiter)
+            @lock.synchronize do
+              if is_loaded
+                wakeup_waiter(waiter)
+              else
+                waiters << waiter
+              end
+            end
+          end
+
+          def complete()
+            @lock.synchronize do
+              @waiters.each do |waiter|
+                wakeup_waiter(waiter)
+              end
+              waiters.clear
+            end
+          end
+
+          def wakeup_waiter(waiter)
+            begin
+              waiter.write(" ")
+            rescue IOError => e
+              BayLog.error_e(e, "Write error: %s", e)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/baykit/bayserver/docker/send_file/file_content_handler.rb

@@ -4,6 +4,7 @@ require 'baykit/bayserver/rudders/io_rudder'
 require 'baykit/bayserver/agent/multiplexer/plain_transporter'
 require 'baykit/bayserver/util/mimes'
 require 'baykit/bayserver/docker/send_file/send_file_ship'
+require 'baykit/bayserver/docker/send_file/wait_file_ship'
 
 module Baykit
   module BayServer
@@ -16,13 +17,39 @@ module Baykit
           include Baykit::BayServer::Tours
           include Baykit::BayServer::Rudders
           include Baykit::BayServer::Util
+          include Baykit::BayServer::Common
 
+          attr :tour
           attr :path
+          attr :charset
+          attr :mime_type
           attr :abortable
+          attr :store
+          attr :file_content
+          attr :lock
 
-          def initialize(path)
+          def initialize(tur, store, path, charset)
+            @tour = tur
+            @store = store
             @path = path
+            @charset = charset
             @abortable = true
+            @lock = Mutex.new
+
+            rname = File.basename(path)
+            pos = rname.rindex('.')
+            if pos
+              ext = rname[pos + 1 .. -1].downcase
+              @mime_type = Mimes.type(ext)
+            end
+
+            if @mime_type == nil
+              @mime_type = "application/octet-stream"
+            end
+
+            if @mime_type.start_with?("text/") && charset != nil
+              @mime_type = @mime_type + "; charset=" + charset
+            end
           end
 
           ######################################################
@@ -36,7 +63,7 @@ module Baykit
 
           def on_end_req_content(tur)
             BayLog.debug("%s endReqContent", tur)
-            send_file_async(tur, path, tur.res.charset)
+            req_start_tour()
             @abortable = false
           end
 
@@ -49,43 +76,82 @@ module Baykit
           # Sending file methods
           ######################################################
 
-          def send_file_async(tur, file, charset)
+          def req_start_tour
 
-            if File.directory?(file)
-              raise HttpException.new HttpStatus::FORBIDDEN, file
-            elsif !File.exist?(file)
-              raise HttpException.new HttpStatus::NOT_FOUND, file
-            end
+            @lock.synchronize do
+              if @store == nil
+                status = FileStore::FileContentStatus.new(nil, FileStore::FileContentStatus::EXCEEDED)
+              else
+                status = @store.get(path)
+              end
+              @file_content = status.file_content
+
+              BayLog.debug("%s file content status: %d", @tour, status.status)
+              case status.status
+              when FileStore::FileContentStatus::STARTED, FileStore::FileContentStatus::EXCEEDED
+                send_file_async
+
+              when FileStore::FileContentStatus::READING
+                # Wait file loaded
+                BayLog.debug("%s Cannot start tour (file reading)", @tour)
+
+                agt = GrandAgent.get(@tour.ship.agent_id)
+                wait_file_ship = WaitFileShip.new()
+                tp = PlainTransporter.new(
+                  agt.spider_multiplexer,
+                  wait_file_ship,
+                  true,
+                  8192,
+                  false)
+
+                begin
+                  pipe = IO::pipe
+                  source_rd = IORudder.new(pipe[0])
+                  source_rd.set_non_blocking()
+                  wait_rd = IORudder.new(pipe[1])
+                rescue IOError => e
+                  raise Sink.new("Fatal error: %s", e)
+                end
 
-            mime_type = nil
+                wait_file_ship.init(source_rd, tp, @tour, @file_content, self)
+                @tour.res.set_consume_listener(&ContentConsumeListener::DEV_NULL)
 
-            rname = File.basename(file)
-            pos = rname.rindex('.')
-            if pos
-              ext = rname[pos + 1 .. -1].downcase
-              mime_type = Mimes.type(ext)
-            end
+                st = RudderStateStore.get_store(agt.agent_id).rent()
+                st.init(source_rd, tp)
+                agt.spider_multiplexer.add_rudder_state(source_rd, st)
+                agt.spider_multiplexer.req_read(source_rd)
+
+                @file_content.add_waiter(wait_rd)
+
+                when FileStore::FileContentStatus::COMPLETED
+                  send_file_from_cache
 
-            if !mime_type
-              mime_type = "application/octet-stream"
+              else
+                raise Sink.new("Unknown file content status: %d", status.status)
+              end
             end
+          end
 
-            if mime_type.start_with?("text/") && charset != nil
-              mime_type = mime_type + "; charset=" + charset
+          def send_file_async()
+
+            if File.directory?(@path)
+              raise HttpException.new(HttpStatus::FORBIDDEN, @path)
+            elsif !File.exist?(@path)
+              raise HttpException.new(HttpStatus::NOT_FOUND, @path)
             end
 
-            file_len = ::File.size(file)
+            file_len = ::File.size(@path)
 
-            tur.res.headers.set_content_type(mime_type)
-            tur.res.headers.set_content_length(file_len)
+            @tour.res.headers.set_content_type(@mime_type)
+            @tour.res.headers.set_content_length(File.size(@path))
 
             begin
-              tur.res.send_headers(Tour::TOUR_ID_NOCHECK)
+              @tour.res.send_headers(Tour::TOUR_ID_NOCHECK)
 
-              bufsize = tur.ship.protocol_handler.max_res_packet_data_size
-              agt = GrandAgent.get(tur.ship.agent_id)
+              bufsize = @tour.ship.protocol_handler.max_res_packet_data_size
+              agt = GrandAgent.get(@tour.ship.agent_id)
 
-              f = File.open(file, "rb")
+              f = File.open(@path, "rb")
               rd = IORudder.new(f)
 
               case(BayServer.harbor.file_multiplexer)
@@ -93,6 +159,9 @@ module Baykit
               when Harbor::MULTIPLEXER_TYPE_SPIDER
                 mpx = agt.spider_multiplexer
 
+              when Harbor::MULTIPLEXER_TYPE_JOB
+                mpx = agt.job_multiplexer
+
               when Harbor::MULTIPLEXER_TYPE_SPIN
                 mpx = agt.spin_multiplexer
 
@@ -111,16 +180,18 @@ module Baykit
                 8195,
                 false)
 
-              send_file_ship.init(rd, tp, tur)
+              send_file_ship.init(rd, tp, @tour, @file_content)
               sid = send_file_ship.ship_id
 
-              tur.res.set_consume_listener do |len, resume|
+              @tour.res.set_consume_listener do |len, resume|
                 if resume
                   send_file_ship.resume_read(sid)
                 end
               end
 
-              mpx.add_rudder_state(rd, RudderState.new(rd, tp))
+              st = RudderStateStore.get_store(agt.agent_id).rent()
+              st.init(rd, tp)
+              mpx.add_rudder_state(rd, st)
               mpx.req_read(rd)
 
             rescue IOError => e
@@ -130,6 +201,19 @@ module Baykit
 
           end
 
+          def send_file_from_cache
+            @tour.res.set_consume_listener(&ContentConsumeListener::DEV_NULL)
+            @tour.res.headers.set_content_type(@mime_type)
+            @tour.res.headers.set_content_length(File.size(@path))
+            begin
+              @tour.res.send_headers(Tour::TOUR_ID_NOCHECK)
+              @tour.res.send_res_content(Tour::TOUR_ID_NOCHECK, @file_content.content, 0, @file_content.content_length)
+              @tour.res.end_res_content(Tour::TOUR_ID_NOCHECK)
+            rescue IOError => e
+              BayLog.error_e(e)
+              raise HttpException.new(HttpStatus::INTERNAL_SERVER_ERROR, @file_content.path)
+            end
+          end
 
         end
       end

data/lib/baykit/bayserver/docker/send_file/file_store.rb

@@ -0,0 +1,117 @@
+require 'baykit/bayserver/rudders/rudder'
+require 'baykit/bayserver/util/simple_buffer'
+
+require 'baykit/bayserver/docker/send_file/file_content'
+
+module Baykit
+  module BayServer
+    module Docker
+      module SendFile
+        class FileStore
+
+          class FileContentStatus
+            STARTED = 1
+            READING = 2
+            COMPLETED = 3
+            EXCEEDED = 4
+
+            attr :file_content
+            attr :status
+
+            def initialize(file_content, status)
+              @file_content = file_content
+              @status = status
+            end
+          end
+
+          attr :contents
+          attr :limit_bytes
+          attr :total_bytes
+          attr :lifespan_seconds
+          attr :lock
+
+          def initialize(timeout_sec, limit_bytes)
+            @lifespan_seconds = timeout_sec
+            @limit_bytes = limit_bytes
+            @total_bytes = 0
+            @contents = {}
+            @lock = Mutex.new
+          end
+
+          def get(path)
+            @lock.synchronize do
+              status = 0
+              file_content = @contents[path]
+
+              if file_content != nil
+                now =  Time.now.to_i
+
+                if file_content.loaded_time + @lifespan_seconds < Time.now.to_i
+                  @total_bytes -= file_content.length
+                  BayLog.debug("Remove expired content: %s", path)
+                  @contents.delete(path)
+                  file_content = nil
+                else
+                  if file_content.is_loaded
+                    status = FileContentStatus::COMPLETED
+                  else
+                    status = FileContentStatus::READING
+                  end
+                end
+              end
+
+              if file_content == nil
+                length = File.size(path)
+                exceeded = false
+                if length <= @limit_bytes
+                  if @total_bytes + length > @limit_bytes
+                    if !evict()
+                      exceeded = true
+                    end
+                  end
+                else
+                  exceeded = true
+                end
+
+                if exceeded
+                  status = FileContentStatus::EXCEEDED
+                else
+                  file_content = FileContent.new(path, length)
+                  @contents[path] = file_content
+                  @total_bytes += length
+                  status = FileContentStatus::STARTED
+                end
+              end
+              return FileContentStatus.new(file_content, status)
+
+            end
+          end
+
+          def evict()
+            evict_list = []
+            @contents.each do | path, content |
+              if content.is_loaded
+                next
+              end
+
+              if content.loaded_time + @lifespan_seconds < Time.now.to_i
+                # Timed out content
+                BayLog.debug("Remove expired content: %s", path)
+                @total_bytes -= content.length
+                evict_list << path
+              else
+                break
+              end
+            end
+
+            evict_list.each do | path |
+              @contents.delete(path)
+            end
+
+            return !evict_list.empty?
+          end
+        end
+      end
+    end
+  end
+end

data/lib/baykit/bayserver/docker/send_file/send_file_docker.rb

@@ -3,6 +3,7 @@ require 'baykit/bayserver/tours/package'
 require 'baykit/bayserver/docker/base/club_base'
 require 'baykit/bayserver/docker/send_file/file_content_handler'
 require 'baykit/bayserver/docker/send_file/directory_train'
+require 'baykit/bayserver/docker/send_file/file_store'
 
 require 'baykit/bayserver/util/string_util'
 
@@ -17,6 +18,7 @@ module Baykit
           include Baykit::BayServer::Tours
 
           attr :list_files
+          attr :file_store
 
           ######################################################
           # Implements DockerBase
@@ -57,14 +59,20 @@ module Baykit
 
               real = "#{tur.town.location}/#{rel_path}"
 
-              if File.directory?(real) && @list_files
-                train = DirectoryTrain.new(tur, real)
-                train.start_tour()
+              if File.directory?(real)
+                if @list_files
+                  train = DirectoryTrain.new(tur, real)
+                  train.start_tour()
+                else
+                  raise HttpException.new(HttpStatus::FORBIDDEN, "Directory scan is prohibited")
+                end
               else
-                handler = FileContentHandler.new(real)
+                if BayServer.harbor.enable_cache() && @file_store == nil
+                  @file_store = FileStore.new(BayServer.harbor.cache_lifespan_sec, BayServer.harbor.cache_size_mb * 1024 * 1024)
+                end
+                handler = FileContentHandler.new(tur, @file_store, real, tur.res.charset)
                 tur.req.set_content_handler(handler)
               end
-
             end
 
           end

data/lib/baykit/bayserver/docker/send_file/send_file_ship.rb

@@ -8,6 +8,8 @@ module Baykit
           include Baykit::BayServer::Tours::ReqContentHandler   # implements
 
           attr :file_wrote_len
+
+          attr :file_content
           attr :tour
           attr :tour_id
 
@@ -19,11 +21,12 @@ module Baykit
           attr :path
           attr :abortable
 
-          def init(rd, tp, tur)
+          def init(rd, tp, tur, file_content)
             super(tur.ship.agent_id, rd, tp)
             @file_wrote_len = 0
             @tour = tur
             @tour_id = tur.tour_id
+            @file_content = file_content
           end
 
           ######################################################
@@ -48,6 +51,12 @@ module Baykit
             begin
               available = @tour.res.send_res_content(@tour_id, buf, 0, buf.length)
 
+              if @file_content != nil
+                #BayLog.debug("buf=%s target=%s", buf, @file_content.content)
+                @file_content.content << buf
+                @file_content.bytes_loaded += buf.length
+              end
+
               if available
                 return NextSocketAction::CONTINUE
               else
@@ -73,7 +82,7 @@ module Baykit
             begin
               @tour.res.end_res_content(@tour_id)
             rescue IOError => e
-              BayLog.debug_e(ex)
+              BayLog.debug_e(e)
             end
             return NextSocketAction::CLOSE
           end