content_server 0.0.10 → 1.0.0

data/lib/content_server/content_server.rb

@@ -0,0 +1,89 @@
+require 'fileutils'
+require 'set'
+require 'thread'
+
+require 'content_data'
+require 'content_server/content_receiver'
+require 'content_server/queue_indexer'
+require 'content_server/queue_copy'
+require 'content_server/remote_content'
+require 'file_indexing'
+require 'file_monitoring'
+require 'log'
+require 'networking/tcp'
+require 'params'
+require 'process_monitoring/thread_safe_hash'
+require 'process_monitoring/monitoring'
+require 'process_monitoring/monitoring_info'
+
+# Content server. Monitors files, index local files, listen to backup server content,
+# copy changes and new files to backup server.
+module ContentServer
+  # Content server specific flags.
+  Params.integer('local_files_port', 4444, 'Remote port in backup server to copy files.')
+  Params.integer('local_content_data_port', 3333, 'Listen to incoming content data requests.')
+
+  def run_content_server
+    all_threads = []
+
+    @process_variables = ThreadSafeHash::ThreadSafeHash.new
+    @process_variables.set('server_name', 'content_server')
+
+    # # # # # # # # # # # #
+    # Initialize/Start monitoring
+    monitoring_events = Queue.new
+    fm = FileMonitoring::FileMonitoring.new
+    fm.set_event_queue(monitoring_events)
+    # Start monitoring and writing changes to queue
+    all_threads << Thread.new do
+      fm.monitor_files
+    end
+
+    # # # # # # # # # # # # # #
+    # Initialize/Start local indexer
+    local_server_content_data_queue = Queue.new
+    queue_indexer = QueueIndexer.new(monitoring_events,
+                                     local_server_content_data_queue,
+                                     Params['local_content_data_path'])
+    # Start indexing on demand and write changes to queue
+    all_threads << queue_indexer.run
+
+    # # # # # # # # # # # # # # # # # # # # # #
+    # Initialize/Start content data comparator
+    copy_files_events = Queue.new
+    local_dynamic_content_data = ContentData::DynamicContentData.new
+    all_threads << Thread.new do
+      while true do
+        # Note: This thread should be the only consumer of local_server_content_data_queue
+        Log.info 'Waiting on local server content data.'
+        local_server_content_data = local_server_content_data_queue.pop
+        local_dynamic_content_data.update(local_server_content_data)
+      end
+    end
+
+    remote_content_client = RemoteContentServer.new(local_dynamic_content_data,
+                                                    Params['local_content_data_port'])
+    all_threads << remote_content_client.tcp_thread
+
+    # # # # # # # # # # # # # # # #
+    # Start copying files on demand
+    copy_server = FileCopyServer.new(copy_files_events, Params['local_files_port'])
+    all_threads.concat(copy_server.run())
+
+    if Params['enable_monitoring']
+      mon = Monitoring::Monitoring.new(@process_variables)
+      Log.add_consumer(mon)
+      all_threads << mon.thread
+      monitoring_info = MonitoringInfo::MonitoringInfo.new(@process_variables)
+    end
+
+    # Finalize server threads.
+    all_threads.each { |t| t.abort_on_exception = true }
+    all_threads.each { |t| t.join }
+    # Should never reach this line.
+  end
+  module_function :run_content_server
+
+end # module ContentServer
+
+

data/lib/content_server/file_streamer.rb

@@ -3,280 +3,285 @@ require 'thread'
 
 require 'file_indexing/index_agent'
 require 'log'
+require 'params'
 
-module BBFS
-  module ContentServer
-
-    Params.integer('streaming_chunk_size', 2*1024*1024,
-                   'Max number of content bytes to send in one chunk.')
-    Params.integer('file_streaming_timeout', 5*60,
-                   'If no action is taken on a file streamer, abort copy.')
-    Params.string('backup_destination_folder', '',
-                  'Backup server destination folder, default is the relative local folder.')
-
-    class Stream
-      attr_reader :checksum, :path, :tmp_path, :file, :size
-      def initialize(checksum, path, file, size)
-        @checksum = checksum
-        @path = path
-        @file = file
-        @size = size
-      end
 
-      def self.close_delete_stream(checksum, streams_hash)
-        if streams_hash.key?(checksum)
-          Log.info("close_delete_stream #{streams_hash[checksum].file}")
-          begin
-            streams_hash[checksum].file.close()
-          rescue IOError => e
-            Log.warning("While closing stream, could not close file #{streams_hash[checksum].path}." \
+module ContentServer
+
+  Params.integer('streaming_chunk_size', 2*1024*1024,
+                 'Max number of content bytes to send in one chunk.')
+  Params.integer('file_streaming_timeout', 5*60,
+                 'If no action is taken on a file streamer, abort copy.')
+  Params.path('backup_destination_folder', '',
+              'Backup server destination folder, default is the relative local folder.')
+
+  class Stream
+    attr_reader :checksum, :path, :tmp_path, :file, :size
+    def initialize(checksum, path, file, size)
+      @checksum = checksum
+      @path = path
+      @file = file
+      @size = size
+    end
+
+    def self.close_delete_stream(checksum, streams_hash)
+      if streams_hash.key?(checksum)
+        Log.debug1("close_delete_stream #{streams_hash[checksum].file}")
+        begin
+          streams_hash[checksum].file.close()
+        rescue IOError => e
+          Log.warning("While closing stream, could not close file #{streams_hash[checksum].path}." \
                         " #{e.to_s}")
-          end
-          streams_hash.delete(checksum)
         end
+        streams_hash.delete(checksum)
       end
-
     end
 
-    class FileStreamer
-      attr_reader :thread
+  end
 
-      :NEW_STREAM
-      :ABORT_STREAM
-      :RESET_STREAM
-      :COPY_CHUNK
+  class FileStreamer
+    attr_reader :thread
 
-      def initialize(send_chunk_clb, abort_streaming_clb=nil)
-        @send_chunk_clb = send_chunk_clb
-        @abort_streaming_clb = abort_streaming_clb
-        @stream_queue = Queue.new
+    :NEW_STREAM
+    :ABORT_STREAM
+    :RESET_STREAM
+    :COPY_CHUNK
 
-        # Used from internal thread only.
-        @streams = {}
-        @thread = run
-      end
+    def initialize(send_chunk_clb, abort_streaming_clb=nil)
+      @send_chunk_clb = send_chunk_clb
+      @abort_streaming_clb = abort_streaming_clb
+      @stream_queue = Queue.new
 
-      def copy_another_chuck(checksum)
-        @stream_queue << [:COPY_CHUNK, checksum]
-      end
+      # Used from internal thread only.
+      @streams = {}
+      @thread = run
+    end
 
-      def start_streaming(checksum, path)
-        @stream_queue << [:NEW_STREAM, [checksum, path]]
-      end
+    def copy_another_chuck(checksum)
+      @stream_queue << [:COPY_CHUNK, checksum]
+    end
 
-      def abort_streaming(checksum)
-        @stream_queue << [:ABORT_STREAM, checksum]
-      end
+    def start_streaming(checksum, path)
+      @stream_queue << [:NEW_STREAM, [checksum, path]]
+    end
 
-      def reset_streaming(checksum, new_offset)
-        @stream_queue << [:RESET_STREAM, [checksum, new_offset]]
-      end
+    def abort_streaming(checksum)
+      @stream_queue << [:ABORT_STREAM, checksum]
+    end
 
-      def run
-        return Thread.new do
-          loop {
-            checksum = handle(@stream_queue.pop)
-          }
-        end
+    def reset_streaming(checksum, new_offset)
+      @stream_queue << [:RESET_STREAM, [checksum, new_offset]]
+    end
+
+    def run
+      return Thread.new do
+        loop {
+          checksum = handle(@stream_queue.pop)
+        }
       end
+    end
 
-      def handle(message)
-        type, content = message
-        if type == :NEW_STREAM
-          checksum, path = content
-          reset_stream(checksum, path, 0)
-          @stream_queue << [:COPY_CHUNK, checksum] if @streams.key?(checksum)
-        elsif type == :ABORT_STREAM
-          checksum = content
-          Stream.close_delete_stream(checksum, @streams)
-        elsif type == :RESET_STREAM
-          checksum, new_offset = content
-          reset_stream(checksum, nil, new_offset)
-          @stream_queue << [:COPY_CHUNK, checksum] if @streams.key?(checksum)
-        elsif type == :COPY_CHUNK
-          checksum = content
-          if @streams.key?(checksum)
-            offset = @streams[checksum].file.pos
-            Log.debug1("Sending chunk for #{checksum}, offset #{offset}.")
-            chunk = @streams[checksum].file.read(Params['streaming_chunk_size'])
-            if chunk.nil?
-              # No more to read, send end of file.
-              @send_chunk_clb.call(checksum, offset, @streams[checksum].size, nil, nil)
-              Stream.close_delete_stream(checksum, @streams)
-            else
-              chunk_checksum = FileIndexing::IndexAgent.get_content_checksum(chunk)
-              @send_chunk_clb.call(checksum, offset, @streams[checksum].size, chunk, chunk_checksum)
-            end
+    def handle(message)
+      type, content = message
+      if type == :NEW_STREAM
+        checksum, path = content
+        reset_stream(checksum, path, 0)
+        @stream_queue << [:COPY_CHUNK, checksum] if @streams.key?(checksum)
+      elsif type == :ABORT_STREAM
+        checksum = content
+        Stream.close_delete_stream(checksum, @streams)
+      elsif type == :RESET_STREAM
+        checksum, new_offset = content
+        reset_stream(checksum, nil, new_offset)
+        @stream_queue << [:COPY_CHUNK, checksum] if @streams.key?(checksum)
+      elsif type == :COPY_CHUNK
+        checksum = content
+        if @streams.key?(checksum)
+          offset = @streams[checksum].file.pos
+          Log.debug1("Sending chunk for #{checksum}, offset #{offset}.")
+          chunk = @streams[checksum].file.read(Params['streaming_chunk_size'])
+          if chunk.nil?
+            # No more to read, send end of file.
+            @send_chunk_clb.call(checksum, offset, @streams[checksum].size, nil, nil)
+            Stream.close_delete_stream(checksum, @streams)
           else
-            Log.info("No checksum found to copy chunk. #{checksum}.")
+            chunk_checksum = FileIndexing::IndexAgent.get_content_checksum(chunk)
+            @send_chunk_clb.call(checksum, offset, @streams[checksum].size, chunk, chunk_checksum)
           end
+        else
+          Log.info("No checksum found to copy chunk. #{checksum}.")
         end
-
       end
 
-      def reset_stream(checksum, path, offset)
-        if !@streams.key? checksum
-          begin
-            file = File.new(path, 'rb')
-            if offset > 0
-              file.seek(offset)
-            end
-            Log.info("File streamer: #{file.to_s}.")
-          rescue IOError => e
-            Log.warning("Could not stream local file #{path}. #{e.to_s}")
+    end
+
+    def reset_stream(checksum, path, offset)
+      if !@streams.key? checksum
+        begin
+          file = File.new(path, 'rb')
+          if offset > 0
+            file.seek(offset)
           end
-          @streams[checksum] = Stream.new(checksum, path, file, file.size)
-        else
-          @streams[checksum].file.seek(offset)
+          Log.debug1("File streamer: #{file.to_s}.")
+        rescue IOError => e
+          Log.warning("Could not stream local file #{path}. #{e.to_s}")
         end
+        @streams[checksum] = Stream.new(checksum, path, file, file.size)
+      else
+        @streams[checksum].file.seek(offset)
       end
     end
+  end
 
-    # Start implementing as dummy, no self thread for now.
-    # Later when we need it to response and send aborts, timeouts, ect, it will
-    # need self thread.
-    class FileReceiver
+  # Start implementing as dummy, no self thread for now.
+  # Later when we need it to response and send aborts, timeouts, ect, it will
+  # need self thread.
+  class FileReceiver
 
-      def initialize(file_done_clb=nil, file_abort_clb=nil, reset_copy=nil)
-        @file_done_clb = file_done_clb
-        @file_abort_clb = file_abort_clb
-        @reset_copy = reset_copy
-        @streams = {}
-      end
+    def initialize(file_done_clb=nil, file_abort_clb=nil, reset_copy=nil)
+      @file_done_clb = file_done_clb
+      @file_abort_clb = file_abort_clb
+      @reset_copy = reset_copy
+      @streams = {}
+    end
 
-      def receive_chunk(file_checksum, offset, file_size, content, content_checksum)
-        # If standard chunk copy.
-        if !content.nil? && !content_checksum.nil?
-          received_content_checksum = FileIndexing::IndexAgent.get_content_checksum(content)
-          comment = "Calculated received chunk with content checksum #{received_content_checksum}" \
+    def receive_chunk(file_checksum, offset, file_size, content, content_checksum)
+      # If standard chunk copy.
+      if !content.nil? && !content_checksum.nil?
+        received_content_checksum = FileIndexing::IndexAgent.get_content_checksum(content)
+        comment = "Calculated received chunk with content checksum #{received_content_checksum}" \
                     " vs message content checksum #{content_checksum}, " \
                     "file checksum #{file_checksum}"
-          Log.debug1(comment) if content_checksum == received_content_checksum
-          # TODO should be here a kind of abort?
-          if content_checksum != received_content_checksum
-            Log.warning(comment)
-            new_offset = 0
-            if @streams.key?(file_checksum)
-              new_offset = @streams[file_checksum].file.pos
-            end
-            @reset_copy.call(file_checksum, new_offset) unless @reset_copy.nil?
-            return false
-          end
-
-          if !@streams.key?(file_checksum)
-            handle_new_stream(file_checksum, file_size)
-          end
-          # We still check @streams has the key, because file can fail to open.
+        Log.debug1(comment) if content_checksum == received_content_checksum
+        # TODO should be here a kind of abort?
+        if content_checksum != received_content_checksum
+          Log.warning(comment)
+          new_offset = 0
           if @streams.key?(file_checksum)
-            return handle_new_chunk(file_checksum, offset, content)
-          else
-            Log.warning('Cannot handle chunk, stream does not exists, sending abort.')
-            @file_abort_clb.call(file_checksum) unless @file_abort_clb.nil?
-            return false
+            new_offset = @streams[file_checksum].file.pos
           end
-        # If last chunk copy.
-        elsif content.nil? && content_checksum.nil?
-          handle_last_chunk(file_checksum)
+          @reset_copy.call(file_checksum, new_offset) unless @reset_copy.nil?
           return false
+        end
+
+        if !@streams.key?(file_checksum)
+          handle_new_stream(file_checksum, file_size)
+        end
+        # We still check @streams has the key, because file can fail to open.
+        if @streams.key?(file_checksum)
+          return handle_new_chunk(file_checksum, offset, content)
         else
-          Log.warning("Unexpected receive chuck message. file_checksum:#{file_checksum}, " \
-                      "content.nil?:#{content.nil?}, content_checksum:#{content_checksum}")
+          Log.warning('Cannot handle chunk, stream does not exists, sending abort.')
+          @file_abort_clb.call(file_checksum) unless @file_abort_clb.nil?
           return false
         end
+        # If last chunk copy.
+      elsif content.nil? && content_checksum.nil?
+        # Handle the case of backup empty file.
+        handle_new_stream(file_checksum, 0) if !@streams.key?(file_checksum)
+        # Finalize the file copy.
+        handle_last_chunk(file_checksum)
+        return false
+      else
+        Log.warning("Unexpected receive chuck message. file_checksum:#{file_checksum}, " \
+                      "content.nil?:#{content.nil?}, content_checksum:#{content_checksum}")
+        return false
       end
+    end
 
-      # open new stream
-      def handle_new_stream(file_checksum, file_size)
-        # final destination path
-        tmp_path = FileReceiver.destination_filename(
-                       File.join(Params['backup_destination_folder'], 'tmp'),
-                       file_checksum)
-        path = FileReceiver.destination_filename(Params['backup_destination_folder'],
-                                                 file_checksum)
-        if File.exists?(path)
-          Log.warning("File already exists (#{path}) not writing.")
-          @file_abort_clb.call(file_checksum) unless @file_abort_clb.nil?
-        else
-          # the file will be moved from tmp location once the transfer will be done
-          # system will use the checksum and some more unique key for tmp file name
-          FileUtils.makedirs(File.dirname(tmp_path)) unless File.directory?(File.dirname(tmp_path))
-          tmp_file = file = File.new(tmp_path, 'wb')
-          @streams[file_checksum] = Stream.new(file_checksum, tmp_path, tmp_file, file_size)
-        end
+    # open new stream
+    def handle_new_stream(file_checksum, file_size)
+      # final destination path
+      tmp_path = FileReceiver.destination_filename(
+          File.join(Params['backup_destination_folder'], 'tmp'),
+          file_checksum)
+      path = FileReceiver.destination_filename(Params['backup_destination_folder'],
+                                               file_checksum)
+      if File.exists?(path)
+        Log.warning("File already exists (#{path}) not writing.")
+        @file_abort_clb.call(file_checksum) unless @file_abort_clb.nil?
+      else
+        # The file will be moved from tmp location once the transfer will be done
+        # system will use the checksum and some more unique key for tmp file name
+        FileUtils.makedirs(File.dirname(tmp_path)) unless File.directory?(File.dirname(tmp_path))
+        tmp_file = file = File.new(tmp_path, 'wb')
+        @streams[file_checksum] = Stream.new(file_checksum, tmp_path, tmp_file, file_size)
       end
+    end
 
-      # write chunk to temp file
-      def handle_new_chunk(file_checksum, offset, content)
-        if offset == @streams[file_checksum].file.pos
-          FileReceiver.write_string_to_file(content, @streams[file_checksum].file)
-          Log.info("Written already #{@streams[file_checksum].file.pos} bytes, " \
+    # write chunk to temp file
+    def handle_new_chunk(file_checksum, offset, content)
+      if offset == @streams[file_checksum].file.pos
+        FileReceiver.write_string_to_file(content, @streams[file_checksum].file)
+        Log.info("Written already #{@streams[file_checksum].file.pos} bytes, " \
                  "out of #{@streams[file_checksum].size} " \
                  "(#{100.0*@streams[file_checksum].file.size/@streams[file_checksum].size}%)")
-          return true
-        else
-          # Offset is wrong, send reset/resume copy from correct offset.
-          Log.warning("Received chunk with incorrect offset #{offset}, should " \
+        return true
+      else
+        # Offset is wrong, send reset/resume copy from correct offset.
+        Log.warning("Received chunk with incorrect offset #{offset}, should " \
                       "be #{@streams[file_checksum].file.pos}, file_checksum:#{file_checksum}")
-          @reset_copy.call(file_checksum, @streams[file_checksum].file.pos) unless @reset_copy.nil?
-          return false
-        end
+        @reset_copy.call(file_checksum, @streams[file_checksum].file.pos) unless @reset_copy.nil?
+        return false
       end
+    end
 
-      # copy file to permanent location
-      # close stream
-      # remove temp file
-      # check written file
-      def handle_last_chunk(file_checksum)
-        if @streams.key?(file_checksum)
-          # Make the directory if does not exists.
-          path = FileReceiver.destination_filename(Params['backup_destination_folder'],
-                                                   file_checksum)
-          Log.debug1("Moving tmp file #{@streams[file_checksum].path} to #{path}")
-          Log.debug1("Creating directory: #{path}")
-          file_dir = File.dirname(path)
-          FileUtils.makedirs(file_dir) unless File.directory?(file_dir)
-          # Move tmp file to permanent location.
-          tmp_file_path = @streams[file_checksum].path
-          Stream.close_delete_stream(file_checksum, @streams)  # temp file will be closed here
-
-          local_file_checksum = FileIndexing::IndexAgent.get_checksum(tmp_file_path)
-          message = "Local checksum (#{local_file_checksum}) received checksum (#{file_checksum})."
-          if local_file_checksum == file_checksum
-            Log.info(message)
-            begin
-              File.rename(tmp_file_path, path)
-              Log.info("End move tmp file to permanent location #{path}.")
-              @file_done_clb.call(local_file_checksum, path) unless @file_done_clb.nil?
-            rescue IOError => e
-              Log.warning("Could not move tmp file to permanent file #{path}. #{e.to_s}")
-            end
-          else
-            Log.error(message)
-            Log.debug1("Deleting tmp file: #{tmp_file_path}")
-            File.delete(tmp_file_path)
+    # copy file to permanent location
+    # close stream
+    # remove temp file
+    # check written file
+    def handle_last_chunk(file_checksum)
+      # Should always be true, unless file creation failed.
+      if @streams.key?(file_checksum)
+        # Make the directory if does not exists.
+        path = FileReceiver.destination_filename(Params['backup_destination_folder'],
+                                                 file_checksum)
+        Log.debug1("Moving tmp file #{@streams[file_checksum].path} to #{path}")
+        Log.debug1("Creating directory: #{path}")
+        file_dir = File.dirname(path)
+        FileUtils.makedirs(file_dir) unless File.directory?(file_dir)
+        # Move tmp file to permanent location.
+        tmp_file_path = @streams[file_checksum].path
+        Stream.close_delete_stream(file_checksum, @streams)  # temp file will be closed here
+
+        local_file_checksum = FileIndexing::IndexAgent.get_checksum(tmp_file_path)
+        message = "Local checksum (#{local_file_checksum}) received checksum (#{file_checksum})."
+        if local_file_checksum == file_checksum
+          Log.info(message)
+          begin
+            File.rename(tmp_file_path, path)
+            Log.info("End move tmp file to permanent location #{path}.")
+            @file_done_clb.call(local_file_checksum, path) unless @file_done_clb.nil?
+          rescue IOError => e
+            Log.warning("Could not move tmp file to permanent file #{path}. #{e.to_s}")
           end
         else
-          Log.error("Handling last chunk and tmp stream does not exists.")
-        end
-      end
-
-      def self.write_string_to_file(str, file)
-        bytes_to_write = str.bytesize
-        Log.info("writing to file: #{file.to_s}, #{bytes_to_write} bytes.")
-        while bytes_to_write > 0
-          bytes_to_write -= file.write(str)
+          Log.error(message)
+          Log.debug1("Deleting tmp file: #{tmp_file_path}")
+          File.delete(tmp_file_path)
         end
+      else
+        Log.error("Handling last chunk and tmp stream does not exists.")
       end
+    end
 
-      # Creates destination filename for backup server, input is base folder and sha1.
-      # for example: folder:/mnt/hd1/bbbackup, sha1:d0be2dc421be4fcd0172e5afceea3970e2f3d940
-      # dest filename: /mnt/hd1/bbbackup/d0/be/2d/d0be2dc421be4fcd0172e5afceea3970e2f3d940
-      def self.destination_filename(folder, sha1)
-        File.join(folder, sha1[0,2], sha1[2,2], sha1)
+    def self.write_string_to_file(str, file)
+      bytes_to_write = str.bytesize
+      Log.info("writing to file: #{file.to_s}, #{bytes_to_write} bytes.")
+      while bytes_to_write > 0
+        bytes_to_write -= file.write(str)
       end
+    end
 
-      private :handle_new_stream, :handle_new_chunk, :handle_last_chunk
+    # Creates destination filename for backup server, input is base folder and sha1.
+    # for example: folder:/mnt/hd1/bbbackup, sha1:d0be2dc421be4fcd0172e5afceea3970e2f3d940
+    # dest filename: /mnt/hd1/bbbackup/d0/be/2d/d0be2dc421be4fcd0172e5afceea3970e2f3d940
+    def self.destination_filename(folder, sha1)
+      File.join(folder, sha1[0,2], sha1[2,2], sha1)
     end
 
+    private :handle_new_stream, :handle_new_chunk, :handle_last_chunk
   end
+
 end
+