content_server 0.0.2 → 0.0.8

data/bin/backup_server

@@ -5,11 +5,23 @@
 # A remote server copies new/changed files in the backup server. The backup server
 # index those new files too and send the content data (their SHA1 to the original server).
 
+# NOTE this file mainly is a copy of content_server
+
 begin
+  require 'yaml'
+  require 'params'
+  require 'run_in_background'
   require 'content_server'
 rescue LoadError
   require 'rubygems'
+  require 'yaml'
+  require 'params'
+  require 'run_in_background'
   require 'content_server'
 end
+include BBFS
+
+Params.init ARGV
+Log.init
 
-BBFS::ContentServer.run_backup_server
+RunInBackground.run { ContentServer.run_backup_server }

data/bin/content_server

@@ -5,10 +5,21 @@
 # Each unique content is backed up to the remote (backup) server.
 
 begin
+  require 'yaml'
+  require 'params'
+  require 'run_in_background'
   require 'content_server'
-rescue LoadError
+rescue LoadError => e
+  print e.to_s
   require 'rubygems'
+  require 'yaml'
+  require 'params'
+  require 'run_in_background'
   require 'content_server'
 end
+include BBFS
 
-BBFS::ContentServer.run
+Params.init ARGV
+Log.init
+
+RunInBackground.run { ContentServer.run }

data/lib/content_server.rb

@@ -1,30 +1,34 @@
+require 'fileutils'
+require 'set'
+require 'thread'
+
 require 'content_data'
-require 'file_copy'
+require 'content_server/content_receiver'
+require 'content_server/queue_indexer'
+require 'content_server/queue_copy'
+require 'content_server/remote_content'
 require 'file_indexing'
 require 'file_monitoring'
+require 'log'
+require 'networking/tcp'
 require 'params'
-require 'set'
-require 'thread'
 
-require_relative 'content_server/content_receiver'
-require_relative 'content_server/queue_indexer'
+
 
 # Content server. Monitors files, index local files, listen to backup server content,
 # copy changes and new files to backup server.
 module BBFS
   module ContentServer
-    VERSION = '0.0.1'
-
-    PARAMS.parameter('remote_server', 'localhost', 'IP or DNS of backup server.')
-    PARAMS.parameter('remote_listening_port', 3333, 'Listening port for backup server content data.')
-    PARAMS.parameter('backup_username', nil, 'Backup server username.')
-    PARAMS.parameter('backup_password', nil, 'Backup server password.')
-    PARAMS.parameter('backup_destination_folder', File.expand_path('~/backup_data'),
-                     'Backup server destination folder.')
-    PARAMS.parameter('content_data_path', File.expand_path('~/.bbfs/var/content.data'),
+    Params.string('remote_server', 'localhost', 'IP or DNS of backup server.')
+    Params.string('backup_username', nil, 'Backup server username.')
+    Params.string('backup_password', nil, 'Backup server password.')
+    Params.integer('backup_file_listening_port', 4444, 'Listening port in backup server for files')
+    Params.string('content_data_path', File.expand_path('~/.bbfs/var/content.data'),
                      'ContentData file path.')
-    PARAMS.parameter('monitoring_config_path', File.expand_path('~/.bbfs/etc/file_monitoring.yml'),
+    Params.string('monitoring_config_path', File.expand_path('~/.bbfs/etc/file_monitoring.yml'),
                      'Configuration file for monitoring.')
+    Params.integer('remote_content_port', 3333, 'Default port for remote content copy.')
+    Params.integer('backup_check_delay', 5, 'Time between two content vs backup checks.')
 
     def run
       all_threads = []
@@ -33,7 +37,7 @@ module BBFS
       # Initialize/Start monitoring
       monitoring_events = Queue.new
       fm = FileMonitoring::FileMonitoring.new
-      fm.set_config_path(PARAMS.monitoring_config_path)
+      fm.set_config_path(Params['monitoring_config_path'])
       fm.set_event_queue(monitoring_events)
       # Start monitoring and writing changes to queue
       all_threads << Thread.new do
@@ -42,103 +46,74 @@ module BBFS
 
       # # # # # # # # # # # # # # # # # # # # # # # # #
       # Initialize/Start backup server content data listener
-      backup_server_content_data = nil
-      backup_server_content_data_queue = Queue.new
-      content_data_receiver = ContentDataReceiver.new(
-          backup_server_content_data_queue,
-          PARAMS.remote_listening_port)
+      #backup_server_content_data = nil
+      #backup_server_content_data_queue = Queue.new
+      #content_data_receiver = ContentDataReceiver.new(
+      #    backup_server_content_data_queue,
+      #    Params['remote_listening_port'])
       # Start listening to backup server
-      all_threads << Thread.new do
-        content_data_receiver.run
-      end
+      #all_threads << Thread.new do
+      #  content_data_receiver.run
+      #end
 
       # # # # # # # # # # # # # #
       # Initialize/Start local indexer
       local_server_content_data_queue = Queue.new
       queue_indexer = QueueIndexer.new(monitoring_events,
                                        local_server_content_data_queue,
-                                       PARAMS.content_data_path)
+                                       Params['content_data_path'])
       # Start indexing on demand and write changes to queue
       all_threads << queue_indexer.run
 
       # # # # # # # # # # # # # # # # # # # # # #
       # Initialize/Start content data comparator
       copy_files_events = Queue.new
+      local_dynamic_content_data = ContentData::DynamicContentData.new
       all_threads << Thread.new do
-        backup_server_content_data = ContentData::ContentData.new
-        local_server_content_data = nil
+      #  backup_server_content_data = ContentData::ContentData.new
+      #  local_server_content_data = nil
         while true do
 
           # Note: This thread should be the only consumer of local_server_content_data_queue
-          p 'Waiting on local server content data.'
+          Log.info 'Waiting on local server content data.'
           local_server_content_data = local_server_content_data_queue.pop
-
-          # Note: This thread should be the only consumer of backup_server_content_data_queue
-          # Note: The server will wait in the first time on pop until backup sends it's content data
-          while backup_server_content_data_queue.size > 0
-            p 'Waiting on backup server content data.'
-            backup_server_content_data = backup_server_content_data_queue.pop
-          end
-
-          p 'Updating file copy queue.'
-          p "local_server_content_data #{local_server_content_data}."
-          p "backup_server_content_data #{backup_server_content_data}."
-          # Remove backup content data from local server
-          content_to_copy = ContentData::ContentData.remove(backup_server_content_data, local_server_content_data)
-          # Add copy instruction in case content is not empty
-          p "Content to copy: #{content_to_copy}"
-          copy_files_events.push(content_to_copy) unless content_to_copy.empty?
+          local_dynamic_content_data.update(local_server_content_data)
+      #
+      #    # Note: This thread should be the only consumer of backup_server_content_data_queue
+      #    # Note: The server will wait in the first time on pop until backup sends it's content data
+      #    while backup_server_content_data_queue.size > 0
+      #      Log.info 'Waiting on backup server content data.'
+      #      backup_server_content_data = backup_server_content_data_queue.pop
+      #    end
+
+      #    Log.info 'Updating file copy queue.'
+      #    Log.debug1 "local_server_content_data #{local_server_content_data}."
+      #    Log.debug1 "backup_server_content_data #{backup_server_content_data}."
+      #    # Remove backup content data from local server
+      #    content_to_copy = ContentData::ContentData.remove(backup_server_content_data, local_server_content_data)
+      #    content_to_copy = local_server_content_data
+      #    # Add copy instruction in case content is not empty
+      #    Log.debug1 "Content to copy: #{content_to_copy}"
+      #    copy_files_events.push([:COPY_MESSAGE, content_to_copy]) unless content_to_copy.empty?
         end
       end
 
+      remote_content_client = RemoteContentClient.new(local_dynamic_content_data,
+                                                      Params['remote_content_port'])
+      all_threads << remote_content_client.tcp_thread
 
       # # # # # # # # # # # # # # # #
       # Start copying files on demand
-      all_threads << Thread.new do
-        while true do
-          p 'Waiting on copy files events.'
-          copy_event = copy_files_events.pop
-
-          p "Copy file event: #{copy_event}"
-
-          # Prepare source,dest map for copy.
-          used_contents = Set.new
-          files_map = Hash.new
-          p "Instances: #{copy_event.instances}"
-          copy_event.instances.each { |key, instance|
-            p "Instance: #{instance}"
-            # Add instance only if such content has not added yet.
-            if !used_contents.member?(instance.checksum)
-              files_map[instance.full_path] = destination_filename(
-                  PARAMS.backup_destination_folder,
-                  instance.checksum)
-              used_contents.add(instance.checksum)
-            end
-          }
-
-          p "Copying files: #{files_map}."
-          # Copy files, waits until files are finished copying.
-          FileCopy::sftp_copy(PARAMS.backup_username,
-                              PARAMS.backup_password,
-                              PARAMS.remote_server,
-                              files_map)
-        end
-      end
+      copy_server = FileCopyServer.new(copy_files_events, Params['backup_file_listening_port'])
+      all_threads.concat(copy_server.run())
 
+      # Finalize server threads.
       all_threads.each { |t| t.abort_on_exception = true }
       all_threads.each { |t| t.join }
       # Should never reach this line.
     end
     module_function :run
 
-    # Creates destination filename for backup server, input is base folder and sha1.
-    # for example: folder:/mnt/hd1/bbbackup, sha1:d0be2dc421be4fcd0172e5afceea3970e2f3d940
-    # dest filename: /mnt/hd1/bbbackup/d0/be/2d/d0be2dc421be4fcd0172e5afceea3970e2f3d940
-    def destination_filename(folder, sha1)
-      File.join(folder, sha1[0,2], sha1[2,2], sha1[4,2], sha1)
-    end
-    module_function :destination_filename
-
     def run_backup_server
       all_threads = []
 
@@ -146,7 +121,7 @@ module BBFS
       # Initialize/Start monitoring
       monitoring_events = Queue.new
       fm = FileMonitoring::FileMonitoring.new
-      fm.set_config_path(PARAMS.monitoring_config_path)
+      fm.set_config_path(Params['monitoring_config_path'])
       fm.set_event_queue(monitoring_events)
       # Start monitoring and writing changes to queue
       all_threads << Thread.new do
@@ -158,23 +133,51 @@ module BBFS
       local_server_content_data_queue = Queue.new
       queue_indexer = QueueIndexer.new(monitoring_events,
                                        local_server_content_data_queue,
-                                       PARAMS.content_data_path)
+                                       Params['content_data_path'])
       # Start indexing on demand and write changes to queue
       all_threads << queue_indexer.run
 
       # # # # # # # # # # # # # # # # # # # # # # # # # # #
       # Initialize/Start backup server content data sender
-      content_data_sender = ContentDataSender.new(
-          PARAMS.remote_server,
-          PARAMS.remote_listening_port)
+      dynamic_content_data = ContentData::DynamicContentData.new
+      #content_data_sender = ContentDataSender.new(
+      #    Params['remote_server'],
+      #    Params['remote_listening_port'])
       # Start sending to backup server
       all_threads << Thread.new do
         while true do
-          p 'Waiting on local server content data queue.'
-          content_data_sender.send_content_data(local_server_content_data_queue.pop)
+          Log.info 'Waiting on local server content data queue.'
+          cd = local_server_content_data_queue.pop
+      #    content_data_sender.send_content_data(cd)
+          dynamic_content_data.update(cd)
         end
       end
 
+      content_server_dynamic_content_data = ContentData::DynamicContentData.new
+      remote_content = ContentServer::RemoteContent.new(content_server_dynamic_content_data,
+                                                        Params['remote_server'],
+                                                        Params['remote_content_port'],
+                                                        Params['backup_destination_folder'])
+      all_threads.concat(remote_content.run())
+
+      file_copy_client = FileCopyClient.new(Params['remote_server'],
+                                               Params['backup_file_listening_port'],
+                                               dynamic_content_data)
+      all_threads.concat(file_copy_client.threads)
+
+      # Each
+      all_threads << Thread.new do
+        loop do
+          sleep(Params['backup_check_delay'])
+          local_cd = dynamic_content_data.last_content_data()
+          remote_cd = content_server_dynamic_content_data.last_content_data()
+          diff = ContentData::ContentData.remove(local_cd, remote_cd)
+          Log.debug2("Files to send? #{!diff.empty?}")
+          file_copy_client.request_copy(diff) unless diff.empty?
+        end
+      end
+
+
       all_threads.each { |t| t.abort_on_exception = true }
       all_threads.each { |t| t.join }
       # Should never reach this line.
@@ -183,3 +186,4 @@ module BBFS
 
   end # module ContentServer
 end # module BBFS
+

data/lib/content_server/content_receiver.rb

@@ -1,3 +1,5 @@
+require 'log'
+require 'params'
 require 'socket'
 
 module BBFS
@@ -11,32 +13,46 @@ module BBFS
 
       def run
         Socket.tcp_server_loop(@port) do |sock, client_addrinfo|
-          p 'Waiting on sock.gets.'
-          size_of_data = sock.read(4).unpack("l")[0]
-          p "Size of data: #{size_of_data}"
-          size_of_data = size_of_data.to_i
-          p "Size of data: #{size_of_data}"
-          data = sock.read(size_of_data)
-          p "Data received: #{data}"
-          unmarshaled_data = Marshal.load(data)
-          p "Unmarshaled data: #{unmarshaled_data}"
-          @queue.push unmarshaled_data
+          while size_of_data = sock.read(4)
+            size_of_data = size_of_data.unpack("l")[0]
+            Log.debug3 "Size of data: #{size_of_data}"
+            data = sock.read(size_of_data)
+            #Log.debug3 "Data received: #{data}"
+            unmarshaled_data = Marshal.load(data)
+            #Log.debug3 "Unmarshaled data: #{unmarshaled_data}"
+            @queue.push unmarshaled_data
+            Log.debug3 "Socket closed? #{sock.closed?}."
+            break if sock.closed?
+            Log.debug1 'Waiting on sock.read'
+          end
+          Log.debug1 'Exited, socket closed or read returned nil.'
         end
       end
     end
 
     class ContentDataSender
+
       def initialize host, port
-        p "Connecting to content server #{host}:#{port}."
-        @tcp_socket = TCPSocket.open(host, port)
+        @host = host
+        @port = port
+        open_socket
+      end
+
+      def open_socket
+        Log.debug1 "Connecting to content server #{@host}:#{@port}."
+        @tcp_socket = TCPSocket.new(@host, @port)
       end
 
       def send_content_data content_data
-        p "Data to send: #{content_data}"
+        open_socket if @tcp_socket.closed?
+        #Log.debug3 "Data to send: #{content_data}"
         marshal_data = Marshal.dump(content_data)
-        p "Marshaled size: #{marshal_data.length}."
+        Log.debug3 "Marshaled size: #{marshal_data.length}."
         data_size = [marshal_data.length].pack("l")
-        p "Marshaled data: #{marshal_data}."
+        #Log.debug3 "Marshaled data: #{marshal_data}."
+        if data_size.nil? || marshal_data.nil?
+          Log.debug3 'Send data is nil!!!!!!!!'
+        end
         @tcp_socket.write data_size
         @tcp_socket.write marshal_data
       end

data/lib/content_server/file_streamer.rb

@@ -0,0 +1,191 @@
+require 'thread'
+
+require 'file_indexing/index_agent'
+require 'log'
+
+module BBFS
+  module ContentServer
+
+    Params.integer('streaming_chunk_size', 64*1024,
+                   'Max number of content bytes to send in one chunk.')
+    Params.integer('file_streaming_timeout', 5*60,
+                   'If no action is taken on a file streamer, abort copy.')
+    Params.string('backup_destination_folder', '',
+                  'Backup server destination folder, default is the relative local folder.')
+
+    class Stream
+      attr_reader :checksum, :path, :file, :size
+      def initialize(checksum, path, file, size)
+        @checksum = checksum
+        @path = path
+        @file = file
+        @size = size
+      end
+
+      def self.close_delete_stream(checksum, streams_hash)
+        if streams_hash.key?(checksum)
+          Log.info("close_delete_stream #{streams_hash[checksum].file}")
+          begin
+            streams_hash[checksum].file.close()
+          rescue IOError => e
+            Log.warning("While closing stream, could not close file #{streams_hash[checksum].path}." \
+                        " #{e.to_s}")
+          end
+          streams_hash.delete(checksum)
+        end
+      end
+
+    end
+
+    class FileStreamer
+      attr_reader :thread
+
+      :NEW_STREAM
+      :ABORT_STREAM
+      :COPY_CHUNK
+
+      def initialize(send_chunk_clb, abort_streaming_clb=nil)
+        @send_chunk_clb = send_chunk_clb
+        @abort_streaming_clb = abort_streaming_clb
+        @stream_queue = Queue.new
+
+        # Used from internal thread only.
+        @streams = {}
+        @thread = run
+      end
+
+      def start_streaming(checksum, path)
+        @stream_queue << [:NEW_STREAM, [checksum, path]]
+      end
+
+      def abort_streaming(checksum)
+        @stream_queue << [:ABORT_STREAM, checksum]
+      end
+
+      def run
+        return Thread.new do
+          loop {
+            checksum = handle(@stream_queue.pop)
+          }
+        end
+      end
+
+      def handle(message)
+        type, content = message
+        if type == :NEW_STREAM
+          checksum, path = content
+          if !@streams.key? checksum
+            begin
+              file = File.new(path, 'rb')
+              Log.info("File streamer: #{file.to_s}.")
+            rescue IOError => e
+              Log.warning("Could not stream local file #{path}. #{e.to_s}")
+            end
+            @streams[checksum] = Stream.new(checksum, path, file, file.size)
+            @stream_queue << [:COPY_CHUNK, checksum]
+          end
+        elsif type == :ABORT_STREAM
+          Stream.close_delete_stream(content, @streams)
+        elsif type == :COPY_CHUNK
+          checksum = content
+          if @streams.key?(checksum)
+            chunk = @streams[checksum].file.read(Params['streaming_chunk_size'])
+            if chunk.nil?
+              # No more to read, send end of file.
+              @send_chunk_clb.call(checksum, @streams[checksum].size, nil, nil)
+              Stream.close_delete_stream(checksum, @streams)
+            else
+              chunk_checksum = FileIndexing::IndexAgent.get_content_checksum(chunk)
+              @send_chunk_clb.call(checksum, @streams[checksum].size, chunk, chunk_checksum)
+              @stream_queue << [:COPY_CHUNK, checksum]
+            end
+          else
+            Log.info("No checksum found to copy chunk. #{checksum}.")
+          end
+        end
+
+      end
+    end
+
+    # Start implementing as dummy, no self thread for now.
+    # Later when we need it to response and send aborts, timeouts, ect, it will
+    # need self thread.
+    class FileReceiver
+
+      def initialize(file_done_clb=nil, file_abort_clb=nil)
+        @file_done_clb = file_done_clb
+        @file_abort_clb = file_abort_clb
+        @streams = {}
+      end
+
+      def receive_chunk(file_checksum, file_size, content, content_checksum)
+        if !content.nil? && !content_checksum.nil?
+          received_content_checksum = FileIndexing::IndexAgent.get_content_checksum(content)
+          comment = "Calculated received chunk with content checksum #{received_content_checksum}" \
+                    " vs message content checksum #{content_checksum}, " \
+                    "file checksum #{file_checksum}"
+          Log.debug1(comment) if content_checksum == received_content_checksum
+          Log.error(comment) if content_checksum != received_content_checksum
+
+          if !@streams.key?(file_checksum)
+            path = FileReceiver.destination_filename(Params['backup_destination_folder'],
+                                                     file_checksum)
+            if File.exists?(path)
+              Log.warning("File already exists (#{path}) not writing.")
+              @file_abort_clb.call(file_checksum) unless @file_abort_clb.nil?
+            else
+              begin
+                # Make the directory if does not exists.
+                Log.debug1("Writing to: #{path}")
+                Log.debug1("Creating directory: #{File.dirname(path)}")
+                FileUtils.makedirs(File.dirname(path))
+                file = File.new(path, 'wb')
+                @streams[file_checksum] = Stream.new(file_checksum, path, file, file_size)
+              rescue IOError => e
+                Log.warning("Could not stream write to local file #{path}. #{e.to_s}")
+              end
+            end
+          end
+          # We still check @streams has the key, because file can fail to open.
+          if @streams.key?(file_checksum)
+            FileReceiver.write_string_to_file(content, @streams[file_checksum].file)
+            Log.info("Written already #{@streams[file_checksum].file.size} bytes, " \
+                     "out of #{file_size} (#{100.0*@streams[file_checksum].file.size/file_size}%)")
+          end
+        elsif content.nil? && content_checksum.nil?
+          if @streams.key?(file_checksum)
+            # Check written file checksum!
+            local_path = @streams[file_checksum].path
+            Stream.close_delete_stream(file_checksum, @streams)
+
+            local_file_checksum = FileIndexing::IndexAgent.get_checksum(local_path)
+            message = "Local checksum (#{local_file_checksum}) received checksum (#{file_checksum})."
+            Log.info(message) ? local_file_checksum == file_checksum : Log.error(message)
+            Log.info("File fully received #{local_path}")
+            @file_done_clb.call(local_file_checksum, local_path) unless @file_done_clb.nil?
+          end
+        else
+          Log.warning("Unexpected receive chuck message. file_checksum:#{file_checksum}, " \
+                      "content.nil?:#{content.nil?}, content_checksum:#{content_checksum}")
+        end
+      end
+
+      def self.write_string_to_file(str, file)
+        Log.info("writing to file: #{file.to_s}.")
+        bytes_to_write = str.bytesize
+        while bytes_to_write > 0
+          bytes_to_write -= file.write(str)
+        end
+      end
+
+      # Creates destination filename for backup server, input is base folder and sha1.
+      # for example: folder:/mnt/hd1/bbbackup, sha1:d0be2dc421be4fcd0172e5afceea3970e2f3d940
+      # dest filename: /mnt/hd1/bbbackup/d0/be/2d/d0be2dc421be4fcd0172e5afceea3970e2f3d940
+      def self.destination_filename(folder, sha1)
+        File.join(folder, sha1[0,2], sha1[2,2], sha1)
+      end
+
+    end
+
+  end
+end

data/lib/content_server/queue_copy.rb

@@ -0,0 +1,172 @@
+require 'thread'
+
+require 'content_server/file_streamer'
+require 'file_indexing/index_agent'
+require 'log'
+require 'networking/tcp'
+
+module BBFS
+  module ContentServer
+    Params.integer('ack_timeout', 5, 'Timeout of ack from backup server in seconds.')
+
+    # Copy message types.
+    :ACK_MESSAGE
+    :COPY_MESSAGE
+    :SEND_COPY_MESSAGE
+    :COPY_CHUNK
+    :ABORT_COPY
+
+    # Simple copier, gets inputs events (files to copy), requests ack from backup to copy
+    # then copies one file.
+    class FileCopyServer
+      def initialize(copy_input_queue, port)
+        # Local simple tcp connection.
+        @backup_tcp = Networking::TCPServer.new(port, method(:receive_message))
+        @copy_input_queue = copy_input_queue
+        # Stores for each checksum, the file source path.
+        # TODO(kolman): If there are items in copy_prepare which timeout (don't get ack),
+        # resend the ack request.
+        @copy_prepare = {}
+        @file_streamer = FileStreamer.new(method(:send_chunk))
+      end
+
+      def send_chunk(*arg)
+        @copy_input_queue.push([:COPY_CHUNK, arg])
+      end
+
+      def receive_message(addr_info, message)
+        # Add ack message to copy queue.
+        Log.info("message received: #{message}")
+        @copy_input_queue.push(message)
+      end
+
+      def run()
+        threads = []
+        threads << @backup_tcp.tcp_thread if @backup_tcp != nil
+        threads << Thread.new do
+          while true do
+            Log.info 'Waiting on copy files events.'
+            message_type, message_content = @copy_input_queue.pop
+
+            if message_type == :COPY_MESSAGE
+              Log.info "Copy file event: #{message_content}"
+              # Prepare source,dest map for copy.
+              message_content.instances.each { |key, instance|
+                @copy_prepare[instance.checksum] = instance.full_path
+                Log.info("Sending ack for: #{instance.checksum}")
+                @backup_tcp.send_obj([:ACK_MESSAGE, [instance.checksum, Time.now.to_i]])
+              }
+            elsif message_type == :ACK_MESSAGE
+              # Received ack from backup, copy file if all is good.
+              # The timestamp is of local content server! not backup server!
+              timestamp, ack, checksum = message_content
+
+              Log.info("Ack (#{ack}) received for: #{checksum}, timestamp: #{timestamp} " \
+                       "now: #{Time.now.to_i}")
+
+              # Copy file if ack (does not exists on backup and not too much time passed)
+              if ack && (Time.now.to_i - timestamp < Params['ack_timeout'])
+                if !@copy_prepare.key?(checksum)
+                  Log.warning("Ack was already received:#{checksum}")
+                else
+                  path = @copy_prepare[checksum]
+                  Log.info "Streaming file: #{checksum} #{path}."
+                  @file_streamer.start_streaming(checksum, path)
+                end
+              else
+                Log.debug1("Ack timed out span: #{Time.now.to_i - timestamp} > " \
+                           "timeout: #{Params['ack_timeout']}")
+              end
+            elsif message_type == :COPY_CHUNK
+              file_checksum, file_size, content, content_checksum = message_content
+              Log.info "Send chunk for file #{file_checksum}."
+              @backup_tcp.send_obj([:COPY_CHUNK, message_content])
+              if content.nil? and content_checksum.nil?
+                @copy_prepare.delete(file_checksum)
+              end
+            elsif message_type == :ABORT_COPY
+              Log.info("Aborting file copy: #{message_content}")
+              if @copy_prepare.key?(message_content)
+                Log.info("Aborting: #{@copy_prepare[message_content]}")
+                @copy_prepare.delete(message_content)
+              end
+              @file_streamer.abort_streaming(message_content)
+            else
+              Log.error("Copy event not supported: #{message_type}")
+            end # handle messages here
+          end
+        end
+      end
+    end  # class QueueCopy
+
+    class FileCopyClient
+      def initialize(host, port, dynamic_content_data)
+        @local_queue = Queue.new
+        @dynamic_content_data = dynamic_content_data
+        @tcp_server = Networking::TCPClient.new(host, port, method(:handle_message))
+        @file_receiver = FileReceiver.new(method(:done_copy), method(:abort_copy))
+        @local_thread = Thread.new do
+          loop do
+            handle(@local_queue.pop)
+          end
+        end
+      end
+
+      def threads
+        ret = [@local_thread]
+        ret << @tcp_server.tcp_thread if @tcp_server != nil
+        return ret
+      end
+
+      def request_copy(content_data)
+        handle_message([:SEND_COPY_MESSAGE, content_data])
+      end
+
+      def abort_copy(checksum)
+        handle_message([:ABORT_COPY, checksum])
+      end
+
+      def done_copy(local_file_checksum, local_path)
+        Log.info("Done copy file: #{local_path}, #{local_file_checksum}")
+      end
+
+      def handle_message(message)
+        Log.debug2('QueueFileReceiver handle message')
+        @local_queue.push(message)
+      end
+
+      # This is a function which receives the messages (file or ack) and return answer in case
+      # of ack. Note that it is being executed from the class thread only!
+      def handle(message)
+        message_type, message_content = message
+        if message_type == :SEND_COPY_MESSAGE
+          Log.debug1("Requesting file (content data) to copy.")
+          Log.debug3("File requested: #{message_content.to_s}")
+          bytes_written = @tcp_server.send_obj([:COPY_MESSAGE, message_content])
+          Log.debug1("Sending copy message succeeded? bytes_written: #{bytes_written}.")
+        elsif message_type == :COPY_CHUNK
+          @file_receiver.receive_chunk(*message_content)
+        elsif message_type == :ACK_MESSAGE
+          checksum, timestamp = message_content
+          # Here we should check file existence
+          Log.debug1("Returning ack for: #{checksum}, timestamp: #{timestamp}")
+          Log.debug1("Ack: #{!@dynamic_content_data.exists?(checksum)}")
+          @tcp_server.send_obj([:ACK_MESSAGE, [timestamp,
+                                               !@dynamic_content_data.exists?(checksum),
+                                               checksum]])
+        elsif message_type == :ABORT_COPY
+          @tcp_server.send_obj([:ABORT_COPY, message_content])
+        else
+          Log.error("Unexpected message type: #{message_type}")
+        end
+      end
+
+      # Creates destination filename for backup server, input is base folder and sha1.
+      # for example: folder:/mnt/hd1/bbbackup, sha1:d0be2dc421be4fcd0172e5afceea3970e2f3d940
+      # dest filename: /mnt/hd1/bbbackup/d0/be/2d/d0be2dc421be4fcd0172e5afceea3970e2f3d940
+      def self.destination_filename(folder, sha1)
+        File.join(folder, sha1[0,2], sha1[2,2], sha1)
+      end
+    end # class QueueFileReceiver
+  end
+end

data/lib/content_server/queue_indexer.rb

@@ -1,5 +1,6 @@
 require 'file_indexing/index_agent'
 require 'file_indexing/indexer_patterns'
+require 'log'
 
 module BBFS
   module ContentServer
@@ -8,7 +9,7 @@ module BBFS
     # content data updates into output queue.
     class QueueIndexer
 
-      def initialize input_queue, output_queue, content_data_path
+      def initialize(input_queue, output_queue, content_data_path)
         @input_queue = input_queue
         @output_queue = output_queue
         @content_data_path = content_data_path
@@ -16,33 +17,94 @@ module BBFS
 
       def run
         server_content_data = ContentData::ContentData.new
-        server_content_data.from_file(@content_data_path) rescue Errno::ENOENT
+        # Shallow check content data files.
+        tmp_content_data = ContentData::ContentData.new
+        tmp_content_data.from_file(@content_data_path) if File.exists?(@content_data_path)
+        tmp_content_data.instances.each_value do |instance|
+          # Skipp instances (files) which did not pass the shallow check.
+          Log.info('Shallow checking content data:')
+          if shallow_check(instance)
+            Log.info("exists: #{instance.full_path}")
+            server_content_data.add_content(tmp_content_data.contents[instance.checksum])
+            server_content_data.add_instance(instance)
+          else
+            Log.info("changed: #{instance.full_path}")
+            # Add non existing and changed files to index queue.
+            @input_queue.push([FileMonitoring::FileStatEnum::STABLE, instance.full_path])
+          end
+        end
+
         # Start indexing on demand and write changes to queue
         thread = Thread.new do
           while true do
-            p 'Waiting on index input queue.'
-            event = @input_queue.pop
-            p "event: #{event}"
+            Log.info 'Waiting on index input queue.'
+            state, is_dir, path = @input_queue.pop
+            Log.info "event: #{state}, #{is_dir}, #{path}."
+
             # index files and add to copy queue
-            if event[0] == FileMonitoring::FileStatEnum::CHANGED || event[0] == FileMonitoring::FileStatEnum::NEW
-              p "Indexing content #{event[1]}."
+            # delete directory with it's sub files
+            # delete file
+            if state == FileMonitoring::FileStatEnum::STABLE && !is_dir
+              Log.info "Indexing content #{path}."
               index_agent = FileIndexing::IndexAgent.new
               indexer_patterns = FileIndexing::IndexerPatterns.new
-              indexer_patterns.add_pattern(event[1])
+              indexer_patterns.add_pattern(path)
               index_agent.index(indexer_patterns, server_content_data)
+              Log.info("Failed files: #{index_agent.failed_files.to_a.join(',')}.") \
+                  if !index_agent.failed_files.empty?
+              Log.info("indexed content #{index_agent.indexed_content}.")
               server_content_data.merge index_agent.indexed_content
-              # TODO(kolman): Don't write to file each change?
-              p "Writing server content data to #{@content_data_path}."
-              server_content_data.to_file(@content_data_path)
-              p 'Adding server content data to queue.'
-              @output_queue.push(server_content_data)
+            elsif ((state == FileMonitoring::FileStatEnum::NON_EXISTING ||
+              state == FileMonitoring::FileStatEnum::CHANGED) && !is_dir)
+              # If file content changed, we should remove old instance.
+              key = FileIndexing::IndexAgent.global_path(path)
+              # Check if deleted file exists at content data.
+              Log.info("Instance to remove: #{key}")
+              if server_content_data.instances.key?(key)
+                instance_to_remove = server_content_data.instances[key]
+                # Remove file from content data only if it does not pass the shallow check, i.e.,
+                # content has changed/removed.
+                if !shallow_check(instance_to_remove)
+                  content_to_remove = server_content_data.contents[instance_to_remove.checksum]
+                  # Remove the deleted instance.
+                  content_data_to_remove = ContentData::ContentData.new
+                  content_data_to_remove.add_content(content_to_remove)
+                  content_data_to_remove.add_instance(instance_to_remove)
+                  # Remove the file.
+                  server_content_data = ContentData::ContentData.remove_instances(
+                      content_data_to_remove, server_content_data)
+                end
+              end
+            elsif state == FileMonitoring::FileStatEnum::NON_EXISTING && is_dir
+              Log.info("NonExisting/Changed: #{path}")
+              # Remove directory but only when non-existing.
+              Log.info("Directory to remove: #{path}")
+              global_dir = FileIndexing::IndexAgent.global_path(path)
+              server_content_data = ContentData::ContentData.remove_directory(
+              server_content_data, global_dir)
+            else
+              Log.info("This case should not be handled: #{state}, #{is_dir}, #{path}.")
             end
-            #p 'End of if.'
+            # TODO(kolman): Don't write to file each change?
+            Log.info "Writing server content data to #{@content_data_path}."
+            server_content_data.to_file(@content_data_path)
+
+            Log.info 'Adding server content data to queue.'
+            @output_queue.push(ContentData::ContentData.new(server_content_data))
           end  # while true do
         end  # Thread.new do
         thread
       end  # def run
 
+      # Check file existence, check it's size and modification date.
+      # If something wrong reindex the file and update content data.
+      def shallow_check(instance)
+        shallow_instance = FileIndexing::IndexAgent.create_shallow_instance(instance.full_path)
+        return false unless shallow_instance
+        return (shallow_instance.size == instance.size &&
+                shallow_instance.modification_time == instance.modification_time)
+      end
+
     end  # class QueueIndexer
   end
 end

data/lib/content_server/remote_content.rb

@@ -0,0 +1,96 @@
+require 'thread'
+
+require 'content_data/dynamic_content_data'
+require 'log'
+require 'networking/tcp'
+require 'params'
+
+module BBFS
+  module ContentServer
+
+    Params.integer('remote_content_timeout', 10, 'Remote content desired freshness in seconds.')
+    Params.integer('max_content_timeout', 60*60, 'Remote content force refresh in seconds.')
+
+    # TODO(kolman): Use only one tcp/ip socket by utilizing one NQueue for many queues!
+    class RemoteContent
+      def initialize(dynamic_content_data, host, port, local_backup_folder)
+        @dynamic_content_data = dynamic_content_data
+        @remote_tcp = Networking::TCPClient.new(host, port, method(:receive_content))
+        @last_update_timestamp = nil
+        @content_server_content_data_path = File.join(local_backup_folder, 'remote',
+                                                      host + '_' + port.to_s)
+        FileUtils.makedirs(@content_server_content_data_path)
+      end
+
+      def receive_content(message)
+        Log.debug1("Remote content data received: #{message.to_s}")
+        ref = @dynamic_content_data.last_content_data
+        @dynamic_content_data.update(message)
+
+        max_time_span = Params['max_content_timeout']
+        if !@last_update_timestamp.nil?
+          max_time_span = Time.now.to_i - @last_update_timestamp
+        end
+
+        @last_update_timestamp = Time.now.to_i
+
+        if ref != message || max_time_span >= Params['max_content_timeout']
+          Log.debug2("Remote content data changed or max time span is large, writing.")
+          Log.debug3("max_time_span: #{max_time_span}")
+          write_to = File.join(@content_server_content_data_path,
+                               @last_update_timestamp.to_s + '.cd')
+          count = File.open(write_to, 'wb') { |f| f.write(message.to_s) }
+        else
+          Log.debug2("No need to write remote content data, it has not changed.")
+        end
+      end
+
+      def run()
+        threads = []
+        threads << @remote_tcp.tcp_thread if @remote_tcp != nil
+        threads << Thread.new do
+          loop do
+            # if need content data
+            if @last_update_timestamp.nil?
+              sleep_time_span = Params['remote_content_timeout']
+            else
+              sleep_time_span = Time.now.to_i - @last_update_timestamp
+            end
+
+            if sleep_time_span >= Params['remote_content_timeout']
+              # Send ping!
+              Log.debug2('Sending remote contend request.')
+              bytes_written = @remote_tcp.send_obj(nil)
+              Log.debug3("Bytes written #{bytes_written}.")
+            end
+
+            sleep_time_span = Time.now.to_i - @last_update_timestamp \
+                unless @last_update_timestamp.nil?
+            Log.debug2("sleep_time_span: #{sleep_time_span}")
+            sleep(sleep_time_span) if sleep_time_span > 0
+          end
+        end
+      end
+    end
+
+    class RemoteContentClient
+      def initialize(dynamic_content_data, port)
+        @dynamic_content_data = dynamic_content_data
+        @tcp_server = Networking::TCPServer.new(port, method(:content_requested))
+      end
+
+      def content_requested(addr_info, message)
+        # Send response.
+        Log.debug1('Local content data requested.')
+        @tcp_server.send_obj(@dynamic_content_data.last_content_data)
+      end
+
+      def tcp_thread
+        return @tcp_server.tcp_thread if @tcp_server != nil
+        nil
+      end
+
+    end
+
+  end
+end

data/lib/content_server/version.rb

@@ -0,0 +1,5 @@
+module BBFS
+  module ContentServer
+    VERSION = "0.0.8"
+  end
+end

data/{test → spec}/content_server/content_server_spec.rb

@@ -1,3 +1,5 @@
+require 'rspec'
+
 require_relative '../../lib/file_copy/copy.rb'
 
 module BBFS

data/spec/content_server/file_streamer_spec.rb

@@ -0,0 +1,53 @@
+require 'log'
+require 'rspec'
+require 'stringio'
+
+require_relative '../../lib/content_server/file_streamer'
+
+# Uncomment to debug spec.
+#BBFS::Params['log_write_to_console'] = true
+BBFS::Params['log_write_to_file'] = false
+BBFS::Params['log_debug_level'] = 0
+BBFS::Params['streaming_chunk_size'] = 5
+BBFS::Params.init ARGV
+BBFS::Log.init
+# Have to be set to test chunking mechanism.
+
+module BBFS
+  module ContentServer
+    module Spec
+      describe 'FileStreamer' do
+        it 'should copy one file chunk by chunks and validate content' do
+          Log.info('#0 start')
+          orig_file = StringIO.new('Some content. Some content. Some content. Some content.')
+          Log.info("orig_file #{orig_file.to_s}.")
+          dest_file = StringIO.new
+          Log.info("dest_file #{dest_file.to_s}.")
+          streamer = nil
+          done = lambda{ |checksum, filename|
+            Log.info('#4 streaming done, check content ok.')
+            dest_file.string().should eq(orig_file.string())
+
+            Log.info('#5 exiting streamer thread.')
+            streamer.thread.exit
+          }
+          receiver = BBFS::ContentServer::FileReceiver.new(done)
+          send_chunk = lambda { |*args|
+            receiver.receive_chunk(*args)
+          }
+          Log.info('#2 start streaming.')
+          # This is for 1) FileStreamer :NEW_STREAM and 2) FileReceiver receive_chunk.
+          ::File.stub(:new).and_return(orig_file, dest_file)
+          # This is for Index agent 'get_checksum' which opens file, read content and validates
+          # checksum.
+          ::File.stub(:open).and_return(dest_file)
+
+          streamer = BBFS::ContentServer::FileStreamer.new(send_chunk)
+          Log.info('#3 start streaming.')
+          streamer.start_streaming('da39a3ee5e6b4b0d3255bfef95601890afd80709', 'dummy')
+          streamer.thread.join()
+        end
+      end
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: content_server
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.8
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-04-11 00:00:00.000000000Z
+date: 2012-09-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: content_data
-  requirement: &70119917644580 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,15 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70119917644580
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: eventmachine
-  requirement: &70119917644120 !ruby/object:Gem::Requirement
+  name: file_indexing
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +37,15 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70119917644120
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: file_copy
-  requirement: &70119917643700 !ruby/object:Gem::Requirement
+  name: file_monitoring
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +53,15 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70119917643700
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: file_indexing
-  requirement: &70119917643240 !ruby/object:Gem::Requirement
+  name: log
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +69,15 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70119917643240
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: file_monitoring
-  requirement: &70119917642820 !ruby/object:Gem::Requirement
+  name: networking
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +85,15 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70119917642820
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: params
-  requirement: &70119917574700 !ruby/object:Gem::Requirement
+  requirement: !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -76,7 +101,28 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70119917574700
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: run_in_background
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Monitor and Index a directory and back it up to backup server.
 email: kolmanv@gmail.com
 executables:
@@ -87,8 +133,13 @@ extra_rdoc_files: []
 files:
 - lib/content_server.rb
 - lib/content_server/content_receiver.rb
+- lib/content_server/file_streamer.rb
+- lib/content_server/queue_copy.rb
 - lib/content_server/queue_indexer.rb
-- test/content_server/content_server_spec.rb
+- lib/content_server/remote_content.rb
+- lib/content_server/version.rb
+- spec/content_server/content_server_spec.rb
+- spec/content_server/file_streamer_spec.rb
 - bin/content_server
 - bin/backup_server
 homepage: http://github.com/kolmanv/bbfs
@@ -111,9 +162,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.15
+rubygems_version: 1.8.23
 signing_key: 
 specification_version: 3
 summary: Servers for backing up content.
 test_files:
-- test/content_server/content_server_spec.rb
+- spec/content_server/content_server_spec.rb
+- spec/content_server/file_streamer_spec.rb