cs_fix 0.0.1

data/bin/backup_server

@@ -0,0 +1,27 @@
+#!/usr/bin/env ruby
+
+# Runs backup server. This server monitors a set of directories (blobs/patterns).
+# The files in those directories are indexed (calculating their SHA1).
+# A remote server copies new/changed files in the backup server. The backup server
+# index those new files too and send the content data (their SHA1 to the original server).
+
+# NOTE this file mainly is a copy of content_server
+
+begin
+  require 'yaml'
+  require 'params'
+  require 'run_in_background'
+  require 'content_server'
+rescue LoadError
+  require 'rubygems'
+  require 'yaml'
+  require 'params'
+  require 'run_in_background'
+  require 'content_server'
+end
+include BBFS
+
+Params.init ARGV
+Log.init
+
+RunInBackground.run { ContentServer.run_backup_server }

data/bin/content_server

@@ -0,0 +1,34 @@
+#!/usr/bin/env ruby
+
+# Runs content server. This server monitors a set of directories (blobs/patterns).
+# The files in those directories are indexed (calculating their SHA1).
+# Each unique content is backed up to the remote (backup) server.
+
+begin
+  print "1"
+  require 'yaml'
+  print "2"
+  require 'params'
+  print "3"
+  require 'run_in_background'
+  print "4"
+  require 'content_server'
+  print "5"
+rescue LoadError
+  print "6"
+  require 'rubygems'
+  print "7"
+  require 'yaml'
+  print "8"
+  require 'params'
+  print "9"
+  require 'run_in_background'
+  print "0"
+  require 'content_server'
+end
+include BBFS
+
+Params.init ARGV
+Log.init
+
+RunInBackground.run { ContentServer.run }

data/lib/content_server/content_receiver.rb

@@ -0,0 +1,62 @@
+require 'log'
+require 'params'
+require 'socket'
+
+module BBFS
+  module ContentServer
+
+    class ContentDataReceiver
+      def initialize queue, port
+        @queue = queue
+        @port = port
+      end
+
+      def run
+        Socket.tcp_server_loop(@port) do |sock, client_addrinfo|
+          while size_of_data = sock.read(4)
+            size_of_data = size_of_data.unpack("l")[0]
+            Log.debug3 "Size of data: #{size_of_data}"
+            data = sock.read(size_of_data)
+            #Log.debug3 "Data received: #{data}"
+            unmarshaled_data = Marshal.load(data)
+            #Log.debug3 "Unmarshaled data: #{unmarshaled_data}"
+            @queue.push unmarshaled_data
+            Log.debug3 "Socket closed? #{sock.closed?}."
+            break if sock.closed?
+            Log.debug1 'Waiting on sock.read'
+          end
+          Log.debug1 'Exited, socket closed or read returned nil.'
+        end
+      end
+    end
+
+    class ContentDataSender
+
+      def initialize host, port
+        @host = host
+        @port = port
+        open_socket
+      end
+
+      def open_socket
+        Log.debug1 "Connecting to content server #{@host}:#{@port}."
+        @tcp_socket = TCPSocket.new(@host, @port)
+      end
+
+      def send_content_data content_data
+        open_socket if @tcp_socket.closed?
+        #Log.debug3 "Data to send: #{content_data}"
+        marshal_data = Marshal.dump(content_data)
+        Log.debug3 "Marshaled size: #{marshal_data.length}."
+        data_size = [marshal_data.length].pack("l")
+        #Log.debug3 "Marshaled data: #{marshal_data}."
+        if data_size.nil? || marshal_data.nil?
+          Log.debug3 'Send data is nil!!!!!!!!'
+        end
+        @tcp_socket.write data_size
+        @tcp_socket.write marshal_data
+      end
+    end
+
+  end
+end

data/lib/content_server/file_streamer.rb

@@ -0,0 +1,282 @@
+require 'tempfile'
+require 'thread'
+
+require 'file_indexing/index_agent'
+require 'log'
+
+module BBFS
+  module ContentServer
+
+    Params.integer('streaming_chunk_size', 2*1024*1024,
+                   'Max number of content bytes to send in one chunk.')
+    Params.integer('file_streaming_timeout', 5*60,
+                   'If no action is taken on a file streamer, abort copy.')
+    Params.string('backup_destination_folder', '',
+                  'Backup server destination folder, default is the relative local folder.')
+
+    class Stream
+      attr_reader :checksum, :path, :tmp_path, :file, :size
+      def initialize(checksum, path, file, size)
+        @checksum = checksum
+        @path = path
+        @file = file
+        @size = size
+      end
+
+      def self.close_delete_stream(checksum, streams_hash)
+        if streams_hash.key?(checksum)
+          Log.info("close_delete_stream #{streams_hash[checksum].file}")
+          begin
+            streams_hash[checksum].file.close()
+          rescue IOError => e
+            Log.warning("While closing stream, could not close file #{streams_hash[checksum].path}." \
+                        " #{e.to_s}")
+          end
+          streams_hash.delete(checksum)
+        end
+      end
+
+    end
+
+    class FileStreamer
+      attr_reader :thread
+
+      :NEW_STREAM
+      :ABORT_STREAM
+      :RESET_STREAM
+      :COPY_CHUNK
+
+      def initialize(send_chunk_clb, abort_streaming_clb=nil)
+        @send_chunk_clb = send_chunk_clb
+        @abort_streaming_clb = abort_streaming_clb
+        @stream_queue = Queue.new
+
+        # Used from internal thread only.
+        @streams = {}
+        @thread = run
+      end
+
+      def copy_another_chuck(checksum)
+        @stream_queue << [:COPY_CHUNK, checksum]
+      end
+
+      def start_streaming(checksum, path)
+        @stream_queue << [:NEW_STREAM, [checksum, path]]
+      end
+
+      def abort_streaming(checksum)
+        @stream_queue << [:ABORT_STREAM, checksum]
+      end
+
+      def reset_streaming(checksum, new_offset)
+        @stream_queue << [:RESET_STREAM, [checksum, new_offset]]
+      end
+
+      def run
+        return Thread.new do
+          loop {
+            checksum = handle(@stream_queue.pop)
+          }
+        end
+      end
+
+      def handle(message)
+        type, content = message
+        if type == :NEW_STREAM
+          checksum, path = content
+          reset_stream(checksum, path, 0)
+          @stream_queue << [:COPY_CHUNK, checksum] if @streams.key?(checksum)
+        elsif type == :ABORT_STREAM
+          checksum = content
+          Stream.close_delete_stream(checksum, @streams)
+        elsif type == :RESET_STREAM
+          checksum, new_offset = content
+          reset_stream(checksum, nil, new_offset)
+          @stream_queue << [:COPY_CHUNK, checksum] if @streams.key?(checksum)
+        elsif type == :COPY_CHUNK
+          checksum = content
+          if @streams.key?(checksum)
+            offset = @streams[checksum].file.pos
+            Log.debug1("Sending chunk for #{checksum}, offset #{offset}.")
+            chunk = @streams[checksum].file.read(Params['streaming_chunk_size'])
+            if chunk.nil?
+              # No more to read, send end of file.
+              @send_chunk_clb.call(checksum, offset, @streams[checksum].size, nil, nil)
+              Stream.close_delete_stream(checksum, @streams)
+            else
+              chunk_checksum = FileIndexing::IndexAgent.get_content_checksum(chunk)
+              @send_chunk_clb.call(checksum, offset, @streams[checksum].size, chunk, chunk_checksum)
+            end
+          else
+            Log.info("No checksum found to copy chunk. #{checksum}.")
+          end
+        end
+
+      end
+
+      def reset_stream(checksum, path, offset)
+        if !@streams.key? checksum
+          begin
+            file = File.new(path, 'rb')
+            if offset > 0
+              file.seek(offset)
+            end
+            Log.info("File streamer: #{file.to_s}.")
+          rescue IOError => e
+            Log.warning("Could not stream local file #{path}. #{e.to_s}")
+          end
+          @streams[checksum] = Stream.new(checksum, path, file, file.size)
+        else
+          @streams[checksum].file.seek(offset)
+        end
+      end
+    end
+
+    # Start implementing as dummy, no self thread for now.
+    # Later when we need it to response and send aborts, timeouts, ect, it will
+    # need self thread.
+    class FileReceiver
+
+      def initialize(file_done_clb=nil, file_abort_clb=nil, reset_copy=nil)
+        @file_done_clb = file_done_clb
+        @file_abort_clb = file_abort_clb
+        @reset_copy = reset_copy
+        @streams = {}
+      end
+
+      def receive_chunk(file_checksum, offset, file_size, content, content_checksum)
+        # If standard chunk copy.
+        if !content.nil? && !content_checksum.nil?
+          received_content_checksum = FileIndexing::IndexAgent.get_content_checksum(content)
+          comment = "Calculated received chunk with content checksum #{received_content_checksum}" \
+                    " vs message content checksum #{content_checksum}, " \
+                    "file checksum #{file_checksum}"
+          Log.debug1(comment) if content_checksum == received_content_checksum
+          # TODO should be here a kind of abort?
+          if content_checksum != received_content_checksum
+            Log.warning(comment)
+            new_offset = 0
+            if @streams.key?(file_checksum)
+              new_offset = @streams[file_checksum].file.pos
+            end
+            @reset_copy.call(file_checksum, new_offset) unless @reset_copy.nil?
+            return false
+          end
+
+          if !@streams.key?(file_checksum)
+            handle_new_stream(file_checksum, file_size)
+          end
+          # We still check @streams has the key, because file can fail to open.
+          if @streams.key?(file_checksum)
+            return handle_new_chunk(file_checksum, offset, content)
+          else
+            Log.warning('Cannot handle chunk, stream does not exists, sending abort.')
+            @file_abort_clb.call(file_checksum) unless @file_abort_clb.nil?
+            return false
+          end
+        # If last chunk copy.
+        elsif content.nil? && content_checksum.nil?
+          handle_last_chunk(file_checksum)
+          return false
+        else
+          Log.warning("Unexpected receive chuck message. file_checksum:#{file_checksum}, " \
+                      "content.nil?:#{content.nil?}, content_checksum:#{content_checksum}")
+          return false
+        end
+      end
+
+      # open new stream
+      def handle_new_stream(file_checksum, file_size)
+        # final destination path
+        tmp_path = FileReceiver.destination_filename(
+                       File.join(Params['backup_destination_folder'], 'tmp'),
+                       file_checksum)
+        path = FileReceiver.destination_filename(Params['backup_destination_folder'],
+                                                 file_checksum)
+        if File.exists?(path)
+          Log.warning("File already exists (#{path}) not writing.")
+          @file_abort_clb.call(file_checksum) unless @file_abort_clb.nil?
+        else
+          # the file will be moved from tmp location once the transfer will be done
+          # system will use the checksum and some more unique key for tmp file name
+          FileUtils.makedirs(File.dirname(tmp_path)) unless File.directory?(File.dirname(tmp_path))
+          tmp_file = file = File.new(tmp_path, 'wb')
+          @streams[file_checksum] = Stream.new(file_checksum, tmp_path, tmp_file, file_size)
+        end
+      end
+
+      # write chunk to temp file
+      def handle_new_chunk(file_checksum, offset, content)
+        if offset == @streams[file_checksum].file.pos
+          FileReceiver.write_string_to_file(content, @streams[file_checksum].file)
+          Log.info("Written already #{@streams[file_checksum].file.pos} bytes, " \
+                 "out of #{@streams[file_checksum].size} " \
+                 "(#{100.0*@streams[file_checksum].file.size/@streams[file_checksum].size}%)")
+          return true
+        else
+          # Offset is wrong, send reset/resume copy from correct offset.
+          Log.warning("Received chunk with incorrect offset #{offset}, should " \
+                      "be #{@streams[file_checksum].file.pos}, file_checksum:#{file_checksum}")
+          @reset_copy.call(file_checksum, @streams[file_checksum].file.pos) unless @reset_copy.nil?
+          return false
+        end
+      end
+
+      # copy file to permanent location
+      # close stream
+      # remove temp file
+      # check written file
+      def handle_last_chunk(file_checksum)
+        if @streams.key?(file_checksum)
+          # Make the directory if does not exists.
+          path = FileReceiver.destination_filename(Params['backup_destination_folder'],
+                                                   file_checksum)
+          Log.debug1("Moving tmp file #{@streams[file_checksum].path} to #{path}")
+          Log.debug1("Creating directory: #{path}")
+          file_dir = File.dirname(path)
+          FileUtils.makedirs(file_dir) unless File.directory?(file_dir)
+          # Move tmp file to permanent location.
+          tmp_file_path = @streams[file_checksum].path
+          Stream.close_delete_stream(file_checksum, @streams)  # temp file will be closed here
+
+          local_file_checksum = FileIndexing::IndexAgent.get_checksum(tmp_file_path)
+          message = "Local checksum (#{local_file_checksum}) received checksum (#{file_checksum})."
+          if local_file_checksum == file_checksum
+            Log.info(message)
+            begin
+              File.rename(tmp_file_path, path)
+              Log.info("End move tmp file to permanent location #{path}.")
+              @file_done_clb.call(local_file_checksum, path) unless @file_done_clb.nil?
+            rescue IOError => e
+              Log.warning("Could not move tmp file to permanent file #{path}. #{e.to_s}")
+            end
+          else
+            Log.error(message)
+            Log.debug1("Deleting tmp file: #{tmp_file_path}")
+            File.delete(tmp_file_path)
+          end
+        else
+          Log.error("Handling last chunk and tmp stream does not exists.")
+        end
+      end
+
+      def self.write_string_to_file(str, file)
+        bytes_to_write = str.bytesize
+        Log.info("writing to file: #{file.to_s}, #{bytes_to_write} bytes.")
+        while bytes_to_write > 0
+          bytes_to_write -= file.write(str)
+        end
+      end
+
+      # Creates destination filename for backup server, input is base folder and sha1.
+      # for example: folder:/mnt/hd1/bbbackup, sha1:d0be2dc421be4fcd0172e5afceea3970e2f3d940
+      # dest filename: /mnt/hd1/bbbackup/d0/be/2d/d0be2dc421be4fcd0172e5afceea3970e2f3d940
+      def self.destination_filename(folder, sha1)
+        File.join(folder, sha1[0,2], sha1[2,2], sha1)
+      end
+
+      private :handle_new_stream, :handle_new_chunk, :handle_last_chunk
+    end
+
+  end
+end

data/lib/content_server/queue_copy.rb

@@ -0,0 +1,203 @@
+require 'thread'
+
+require 'content_server/file_streamer'
+require 'file_indexing/index_agent'
+require 'log'
+require 'networking/tcp'
+
+module BBFS
+  module ContentServer
+    Params.integer('ack_timeout', 5, 'Timeout of ack from backup server in seconds.')
+
+    # Copy message types.
+    :ACK_MESSAGE
+    :COPY_MESSAGE
+    :SEND_COPY_MESSAGE
+    :COPY_CHUNK
+    :COPY_CHUNK_FROM_REMOTE
+    :ABORT_COPY # Asks the sender to abort file copy.
+    :RESET_RESUME_COPY # Sends the stream sender to resend chunk or resume from different offset.
+
+    # Simple copier, gets inputs events (files to copy), requests ack from backup to copy
+    # then copies one file.
+    class FileCopyServer
+      def initialize(copy_input_queue, port)
+        # Local simple tcp connection.
+        @backup_tcp = Networking::TCPServer.new(port, method(:receive_message))
+        @copy_input_queue = copy_input_queue
+        # Stores for each checksum, the file source path.
+        # TODO(kolman): If there are items in copy_prepare which timeout (don't get ack),
+        # resend the ack request.
+        @copy_prepare = {}
+        @file_streamer = FileStreamer.new(method(:send_chunk))
+      end
+
+      def send_chunk(*arg)
+        @copy_input_queue.push([:COPY_CHUNK, arg])
+      end
+
+      def receive_message(addr_info, message)
+        # Add ack message to copy queue.
+        Log.info("message received: #{message}")
+        @copy_input_queue.push(message)
+      end
+
+      def run()
+        threads = []
+        threads << @backup_tcp.tcp_thread if @backup_tcp != nil
+        threads << Thread.new do
+          while true do
+            Log.info 'Waiting on copy files events.'
+            message_type, message_content = @copy_input_queue.pop
+
+            if message_type == :COPY_MESSAGE
+              Log.info "Copy file event: #{message_content}"
+              # Prepare source,dest map for copy.
+              message_content.instances.each { |key, instance|
+                # If not already sending.
+                if !@copy_prepare.key?(instance.checksum) || !@copy_prepare[instance.checksum][1]
+                  @copy_prepare[instance.checksum] = [instance.full_path, false]
+                  Log.info("Sending ack for: #{instance.checksum}")
+                  @backup_tcp.send_obj([:ACK_MESSAGE, [instance.checksum, Time.now.to_i]])
+                end
+              }
+            elsif message_type == :ACK_MESSAGE
+              # Received ack from backup, copy file if all is good.
+              # The timestamp is of local content server! not backup server!
+              timestamp, ack, checksum = message_content
+
+              Log.info("Ack (#{ack}) received for: #{checksum}, timestamp: #{timestamp} " \
+                       "now: #{Time.now.to_i}")
+
+              # Copy file if ack (does not exists on backup and not too much time passed)
+              if ack && (Time.now.to_i - timestamp < Params['ack_timeout'])
+                if !@copy_prepare.key?(checksum) || @copy_prepare[checksum][1]
+                  Log.warning("File was aborted, copied, or started copy just now: #{checksum}")
+                else
+                  path = @copy_prepare[checksum][0]
+                  Log.info "Streaming file: #{checksum} #{path}."
+                  @file_streamer.start_streaming(checksum, path)
+                  # Ack received, setting prepare to true
+                  @copy_prepare[checksum][1] = true
+                end
+              else
+                Log.debug1("Ack timed out span: #{Time.now.to_i - timestamp} > " \
+                           "timeout: #{Params['ack_timeout']}")
+              end
+            elsif message_type == :COPY_CHUNK_FROM_REMOTE
+              checksum = message_content
+              @file_streamer.copy_another_chuck(checksum)
+            elsif message_type == :COPY_CHUNK
+              # We open the message here for printing info and deleting copy_prepare!
+              file_checksum, offset, file_size, content, content_checksum = message_content
+              Log.info("Send chunk for file #{file_checksum}, offset: #{offset} " \
+                       "filesize: #{file_size}.")
+              # Blocking send.
+              @backup_tcp.send_obj([:COPY_CHUNK, message_content])
+              if content.nil? and content_checksum.nil?
+                @copy_prepare.delete(file_checksum)
+              end
+            elsif message_type == :ABORT_COPY
+              Log.info("Aborting file copy: #{message_content}")
+              if @copy_prepare.key?(message_content)
+                Log.info("Aborting: #{@copy_prepare[message_content][0]}")
+                @copy_prepare.delete(message_content)
+              end
+              @file_streamer.abort_streaming(message_content)
+            elsif message_type == :RESET_RESUME_COPY
+              file_checksum, new_offset = message_content
+              Log.info("Resetting/Resuming file (#{file_checksum}) copy to #{new_offset}")
+              @file_streamer.reset_streaming(file_checksum, new_offset)
+            else
+              Log.error("Copy event not supported: #{message_type}")
+            end # handle messages here
+          end
+          Log.error("Should not reach here, loop should continue.")
+        end
+      end
+    end  # class QueueCopy
+
+    class FileCopyClient
+      def initialize(host, port, dynamic_content_data)
+        @local_queue = Queue.new
+        @dynamic_content_data = dynamic_content_data
+        @tcp_server = Networking::TCPClient.new(host, port, method(:handle_message))
+        @file_receiver = FileReceiver.new(method(:done_copy),
+                                          method(:abort_copy),
+                                          method(:reset_copy))
+        @local_thread = Thread.new do
+          loop do
+            handle(@local_queue.pop)
+          end
+        end
+        @local_thread.abort_on_exception = true
+      end
+
+      def threads
+        ret = [@local_thread]
+        ret << @tcp_server.tcp_thread if @tcp_server != nil
+        return ret
+      end
+
+      def request_copy(content_data)
+        handle_message([:SEND_COPY_MESSAGE, content_data])
+      end
+
+      def abort_copy(checksum)
+        handle_message([:ABORT_COPY, checksum])
+      end
+
+      def reset_copy(checksum, new_offset)
+        handle_message([:RESET_RESUME_COPY, [checksum, new_offset]])
+      end
+
+      def done_copy(local_file_checksum, local_path)
+        Log.info("Done copy file: #{local_path}, #{local_file_checksum}")
+      end
+
+      def handle_message(message)
+        Log.debug2('QueueFileReceiver handle message')
+        @local_queue.push(message)
+      end
+
+      # This is a function which receives the messages (file or ack) and return answer in case
+      # of ack. Note that it is being executed from the class thread only!
+      def handle(message)
+        message_type, message_content = message
+        if message_type == :SEND_COPY_MESSAGE
+          Log.debug1("Requesting file (content data) to copy.")
+          Log.debug3("File requested: #{message_content.to_s}")
+          bytes_written = @tcp_server.send_obj([:COPY_MESSAGE, message_content])
+          Log.debug1("Sending copy message succeeded? bytes_written: #{bytes_written}.")
+        elsif message_type == :COPY_CHUNK
+          Log.debug1('Chunk received.')
+          if @file_receiver.receive_chunk(*message_content)
+            file_checksum, offset, file_size, content, content_checksum = message_content
+            @tcp_server.send_obj([:COPY_CHUNK_FROM_REMOTE, file_checksum])
+          end
+        elsif message_type == :ACK_MESSAGE
+          checksum, timestamp = message_content
+          # Here we should check file existence
+          Log.debug1("Returning ack for: #{checksum}, timestamp: #{timestamp}")
+          Log.debug1("Ack: #{!@dynamic_content_data.exists?(checksum)}")
+          @tcp_server.send_obj([:ACK_MESSAGE, [timestamp,
+                                               !@dynamic_content_data.exists?(checksum),
+                                               checksum]])
+        elsif message_type == :ABORT_COPY
+          @tcp_server.send_obj([:ABORT_COPY, message_content])
+        elsif message_type == :RESET_RESUME_COPY
+          @tcp_server.send_obj([:RESET_RESUME_COPY, message_content])
+        else
+          Log.error("Unexpected message type: #{message_type}")
+        end
+      end
+
+      # Creates destination filename for backup server, input is base folder and sha1.
+      # for example: folder:/mnt/hd1/bbbackup, sha1:d0be2dc421be4fcd0172e5afceea3970e2f3d940
+      # dest filename: /mnt/hd1/bbbackup/d0/be/2d/d0be2dc421be4fcd0172e5afceea3970e2f3d940
+      def self.destination_filename(folder, sha1)
+        File.join(folder, sha1[0,2], sha1[2,2], sha1)
+      end
+    end # class QueueFileReceiver
+  end
+end

data/lib/content_server/queue_indexer.rb

@@ -0,0 +1,110 @@
+require 'file_indexing/index_agent'
+require 'file_indexing/indexer_patterns'
+require 'log'
+
+module BBFS
+  module ContentServer
+
+    # Simple indexer, gets inputs events (files to index) and outputs
+    # content data updates into output queue.
+    class QueueIndexer
+
+      def initialize(input_queue, output_queue, content_data_path)
+        @input_queue = input_queue
+        @output_queue = output_queue
+        @content_data_path = content_data_path
+      end
+
+      def run
+        server_content_data = ContentData::ContentData.new
+        # Shallow check content data files.
+        tmp_content_data = ContentData::ContentData.new
+        tmp_content_data.from_file(@content_data_path) if File.exists?(@content_data_path)
+        tmp_content_data.instances.each_value do |instance|
+          # Skipp instances (files) which did not pass the shallow check.
+          Log.info('Shallow checking content data:')
+          if shallow_check(instance)
+            Log.info("exists: #{instance.full_path}")
+            server_content_data.add_content(tmp_content_data.contents[instance.checksum])
+            server_content_data.add_instance(instance)
+          else
+            Log.info("changed: #{instance.full_path}")
+            # Add non existing and changed files to index queue.
+            @input_queue.push([FileMonitoring::FileStatEnum::STABLE, instance.full_path])
+          end
+        end
+
+        # Start indexing on demand and write changes to queue
+        thread = Thread.new do
+          while true do
+            Log.info 'Waiting on index input queue.'
+            state, is_dir, path = @input_queue.pop
+            Log.info "event: #{state}, #{is_dir}, #{path}."
+
+            # index files and add to copy queue
+            # delete directory with it's sub files
+            # delete file
+            if state == FileMonitoring::FileStatEnum::STABLE && !is_dir
+              Log.info "Indexing content #{path}."
+              index_agent = FileIndexing::IndexAgent.new
+              indexer_patterns = FileIndexing::IndexerPatterns.new
+              indexer_patterns.add_pattern(path)
+              index_agent.index(indexer_patterns, server_content_data)
+              Log.info("Failed files: #{index_agent.failed_files.to_a.join(',')}.") \
+                  if !index_agent.failed_files.empty?
+              Log.info("indexed content #{index_agent.indexed_content}.")
+              server_content_data.merge index_agent.indexed_content
+            elsif ((state == FileMonitoring::FileStatEnum::NON_EXISTING ||
+              state == FileMonitoring::FileStatEnum::CHANGED) && !is_dir)
+              # If file content changed, we should remove old instance.
+              key = FileIndexing::IndexAgent.global_path(path)
+              # Check if deleted file exists at content data.
+              Log.info("Instance to remove: #{key}")
+              if server_content_data.instances.key?(key)
+                instance_to_remove = server_content_data.instances[key]
+                # Remove file from content data only if it does not pass the shallow check, i.e.,
+                # content has changed/removed.
+                if !shallow_check(instance_to_remove)
+                  content_to_remove = server_content_data.contents[instance_to_remove.checksum]
+                  # Remove the deleted instance.
+                  content_data_to_remove = ContentData::ContentData.new
+                  content_data_to_remove.add_content(content_to_remove)
+                  content_data_to_remove.add_instance(instance_to_remove)
+                  # Remove the file.
+                  server_content_data = ContentData::ContentData.remove_instances(
+                      content_data_to_remove, server_content_data)
+                end
+              end
+            elsif state == FileMonitoring::FileStatEnum::NON_EXISTING && is_dir
+              Log.info("NonExisting/Changed: #{path}")
+              # Remove directory but only when non-existing.
+              Log.info("Directory to remove: #{path}")
+              global_dir = FileIndexing::IndexAgent.global_path(path)
+              server_content_data = ContentData::ContentData.remove_directory(
+              server_content_data, global_dir)
+            else
+              Log.info("This case should not be handled: #{state}, #{is_dir}, #{path}.")
+            end
+            # TODO(kolman): Don't write to file each change?
+            Log.info "Writing server content data to #{@content_data_path}."
+            server_content_data.to_file(@content_data_path)
+
+            Log.info 'Adding server content data to queue.'
+            @output_queue.push(ContentData::ContentData.new(server_content_data))
+          end  # while true do
+        end  # Thread.new do
+        thread
+      end  # def run
+
+      # Check file existence, check it's size and modification date.
+      # If something wrong reindex the file and update content data.
+      def shallow_check(instance)
+        shallow_instance = FileIndexing::IndexAgent.create_shallow_instance(instance.full_path)
+        return false unless shallow_instance
+        return (shallow_instance.size == instance.size &&
+                shallow_instance.modification_time == instance.modification_time)
+      end
+
+    end  # class QueueIndexer
+  end
+end

data/lib/content_server/remote_content.rb

@@ -0,0 +1,97 @@
+require 'thread'
+
+require 'content_data/dynamic_content_data'
+require 'log'
+require 'networking/tcp'
+require 'params'
+
+module BBFS
+  module ContentServer
+
+    Params.integer('remote_content_timeout', 10, 'Remote content desired freshness in seconds.')
+    Params.integer('max_content_timeout', 60*60, 'Remote content force refresh in seconds.')
+
+    # TODO(kolman): Use only one tcp/ip socket by utilizing one NQueue for many queues!
+    class RemoteContent
+      def initialize(dynamic_content_data, host, port, local_backup_folder)
+        @dynamic_content_data = dynamic_content_data
+        @remote_tcp = Networking::TCPClient.new(host, port, method(:receive_content))
+        @last_update_timestamp = nil
+        @content_server_content_data_path = File.join(local_backup_folder, 'remote',
+                                                      host + '_' + port.to_s)
+      end
+
+      def receive_content(message)
+        Log.debug1("Remote content data received: #{message.to_s}")
+        ref = @dynamic_content_data.last_content_data
+        @dynamic_content_data.update(message)
+
+        max_time_span = Params['max_content_timeout']
+        if !@last_update_timestamp.nil?
+          max_time_span = Time.now.to_i - @last_update_timestamp
+        end
+
+        @last_update_timestamp = Time.now.to_i
+
+        if ref != message || max_time_span >= Params['max_content_timeout']
+          Log.debug2("Remote content data changed or max time span is large, writing.")
+          Log.debug3("max_time_span: #{max_time_span}")
+          write_to = File.join(@content_server_content_data_path,
+                               @last_update_timestamp.to_s + '.cd')
+          FileUtils.makedirs(@content_server_content_data_path) unless \
+              File.directory?(@content_server_content_data_path)
+          count = File.open(write_to, 'wb') { |f| f.write(message.to_s) }
+        else
+          Log.debug2("No need to write remote content data, it has not changed.")
+        end
+      end
+
+      def run()
+        threads = []
+        threads << @remote_tcp.tcp_thread if @remote_tcp != nil
+        threads << Thread.new do
+          loop do
+            # if need content data
+            if @last_update_timestamp.nil?
+              sleep_time_span = Params['remote_content_timeout']
+            else
+              sleep_time_span = Time.now.to_i - @last_update_timestamp
+            end
+
+            if sleep_time_span >= Params['remote_content_timeout']
+              # Send ping!
+              Log.debug2('Sending remote contend request.')
+              bytes_written = @remote_tcp.send_obj(nil)
+              Log.debug3("Bytes written #{bytes_written}.")
+            end
+
+            sleep_time_span = Time.now.to_i - @last_update_timestamp \
+                unless @last_update_timestamp.nil?
+            Log.debug2("sleep_time_span: #{sleep_time_span}")
+            sleep(sleep_time_span) if sleep_time_span > 0
+          end
+        end
+      end
+    end
+
+    class RemoteContentClient
+      def initialize(dynamic_content_data, port)
+        @dynamic_content_data = dynamic_content_data
+        @tcp_server = Networking::TCPServer.new(port, method(:content_requested))
+      end
+
+      def content_requested(addr_info, message)
+        # Send response.
+        Log.debug1('Local content data requested.')
+        @tcp_server.send_obj(@dynamic_content_data.last_content_data)
+      end
+
+      def tcp_thread
+        return @tcp_server.tcp_thread if @tcp_server != nil
+        nil
+      end
+
+    end
+
+  end
+end

data/lib/content_server/version.rb

@@ -0,0 +1,5 @@
+module BBFS
+  module ContentServer
+    VERSION = "0.0.1"
+  end
+end

data/lib/content_server.rb

@@ -0,0 +1,189 @@
+require 'fileutils'
+require 'set'
+require 'thread'
+
+require 'content_data'
+require 'content_server/content_receiver'
+require 'content_server/queue_indexer'
+require 'content_server/queue_copy'
+require 'content_server/remote_content'
+require 'file_indexing'
+require 'file_monitoring'
+require 'log'
+require 'networking/tcp'
+require 'params'
+
+
+
+# Content server. Monitors files, index local files, listen to backup server content,
+# copy changes and new files to backup server.
+module BBFS
+  module ContentServer
+    Params.string('remote_server', 'localhost', 'IP or DNS of backup server.')
+    Params.string('backup_username', nil, 'Backup server username.')
+    Params.string('backup_password', nil, 'Backup server password.')
+    Params.integer('backup_file_listening_port', 4444, 'Listening port in backup server for files')
+    Params.string('content_data_path', File.expand_path('~/.bbfs/var/content.data'),
+                     'ContentData file path.')
+    Params.string('monitoring_config_path', File.expand_path('~/.bbfs/etc/file_monitoring.yml'),
+                     'Configuration file for monitoring.')
+    Params.integer('remote_content_port', 3333, 'Default port for remote content copy.')
+    Params.integer('backup_check_delay', 5, 'Time between two content vs backup checks.')
+
+    def run
+      all_threads = []
+
+      # # # # # # # # # # # #
+      # Initialize/Start monitoring
+      monitoring_events = Queue.new
+      fm = FileMonitoring::FileMonitoring.new
+      fm.set_config_path(Params['monitoring_config_path'])
+      fm.set_event_queue(monitoring_events)
+      # Start monitoring and writing changes to queue
+      all_threads << Thread.new do
+        fm.monitor_files
+      end
+
+      # # # # # # # # # # # # # # # # # # # # # # # # #
+      # Initialize/Start backup server content data listener
+      #backup_server_content_data = nil
+      #backup_server_content_data_queue = Queue.new
+      #content_data_receiver = ContentDataReceiver.new(
+      #    backup_server_content_data_queue,
+      #    Params['remote_listening_port'])
+      # Start listening to backup server
+      #all_threads << Thread.new do
+      #  content_data_receiver.run
+      #end
+
+      # # # # # # # # # # # # # #
+      # Initialize/Start local indexer
+      local_server_content_data_queue = Queue.new
+      queue_indexer = QueueIndexer.new(monitoring_events,
+                                       local_server_content_data_queue,
+                                       Params['content_data_path'])
+      # Start indexing on demand and write changes to queue
+      all_threads << queue_indexer.run
+
+      # # # # # # # # # # # # # # # # # # # # # #
+      # Initialize/Start content data comparator
+      copy_files_events = Queue.new
+      local_dynamic_content_data = ContentData::DynamicContentData.new
+      all_threads << Thread.new do
+      #  backup_server_content_data = ContentData::ContentData.new
+      #  local_server_content_data = nil
+        while true do
+
+          # Note: This thread should be the only consumer of local_server_content_data_queue
+          Log.info 'Waiting on local server content data.'
+          local_server_content_data = local_server_content_data_queue.pop
+          local_dynamic_content_data.update(local_server_content_data)
+      #
+      #    # Note: This thread should be the only consumer of backup_server_content_data_queue
+      #    # Note: The server will wait in the first time on pop until backup sends it's content data
+      #    while backup_server_content_data_queue.size > 0
+      #      Log.info 'Waiting on backup server content data.'
+      #      backup_server_content_data = backup_server_content_data_queue.pop
+      #    end
+
+      #    Log.info 'Updating file copy queue.'
+      #    Log.debug1 "local_server_content_data #{local_server_content_data}."
+      #    Log.debug1 "backup_server_content_data #{backup_server_content_data}."
+      #    # Remove backup content data from local server
+      #    content_to_copy = ContentData::ContentData.remove(backup_server_content_data, local_server_content_data)
+      #    content_to_copy = local_server_content_data
+      #    # Add copy instruction in case content is not empty
+      #    Log.debug1 "Content to copy: #{content_to_copy}"
+      #    copy_files_events.push([:COPY_MESSAGE, content_to_copy]) unless content_to_copy.empty?
+        end
+      end
+
+      remote_content_client = RemoteContentClient.new(local_dynamic_content_data,
+                                                      Params['remote_content_port'])
+      all_threads << remote_content_client.tcp_thread
+
+      # # # # # # # # # # # # # # # #
+      # Start copying files on demand
+      copy_server = FileCopyServer.new(copy_files_events, Params['backup_file_listening_port'])
+      all_threads.concat(copy_server.run())
+
+      # Finalize server threads.
+      all_threads.each { |t| t.abort_on_exception = true }
+      all_threads.each { |t| t.join }
+      # Should never reach this line.
+    end
+    module_function :run
+
+    def run_backup_server
+      all_threads = []
+
+      # # # # # # # # # # # #
+      # Initialize/Start monitoring
+      monitoring_events = Queue.new
+      fm = FileMonitoring::FileMonitoring.new
+      fm.set_config_path(Params['monitoring_config_path'])
+      fm.set_event_queue(monitoring_events)
+      # Start monitoring and writing changes to queue
+      all_threads << Thread.new do
+        fm.monitor_files
+      end
+
+      # # # # # # # # # # # # # #
+      # Initialize/Start local indexer
+      local_server_content_data_queue = Queue.new
+      queue_indexer = QueueIndexer.new(monitoring_events,
+                                       local_server_content_data_queue,
+                                       Params['content_data_path'])
+      # Start indexing on demand and write changes to queue
+      all_threads << queue_indexer.run
+
+      # # # # # # # # # # # # # # # # # # # # # # # # # # #
+      # Initialize/Start backup server content data sender
+      dynamic_content_data = ContentData::DynamicContentData.new
+      #content_data_sender = ContentDataSender.new(
+      #    Params['remote_server'],
+      #    Params['remote_listening_port'])
+      # Start sending to backup server
+      all_threads << Thread.new do
+        while true do
+          Log.info 'Waiting on local server content data queue.'
+          cd = local_server_content_data_queue.pop
+      #    content_data_sender.send_content_data(cd)
+          dynamic_content_data.update(cd)
+        end
+      end
+
+      content_server_dynamic_content_data = ContentData::DynamicContentData.new
+      remote_content = ContentServer::RemoteContent.new(content_server_dynamic_content_data,
+                                                        Params['remote_server'],
+                                                        Params['remote_content_port'],
+                                                        Params['backup_destination_folder'])
+      all_threads.concat(remote_content.run())
+
+      file_copy_client = FileCopyClient.new(Params['remote_server'],
+                                               Params['backup_file_listening_port'],
+                                               dynamic_content_data)
+      all_threads.concat(file_copy_client.threads)
+
+      # Each
+      all_threads << Thread.new do
+        loop do
+          sleep(Params['backup_check_delay'])
+          local_cd = dynamic_content_data.last_content_data()
+          remote_cd = content_server_dynamic_content_data.last_content_data()
+          diff = ContentData::ContentData.remove(local_cd, remote_cd)
+          Log.debug2("Files to send? #{!diff.empty?}")
+          file_copy_client.request_copy(diff) unless diff.empty?
+        end
+      end
+
+
+      all_threads.each { |t| t.abort_on_exception = true }
+      all_threads.each { |t| t.join }
+      # Should never reach this line.
+    end
+    module_function :run_backup_server
+
+  end # module ContentServer
+end # module BBFS
+

data/spec/content_server/content_server_spec.rb

@@ -0,0 +1,27 @@
+require 'rspec'
+
+require_relative '../../lib/file_copy/copy.rb'
+
+module BBFS
+  module ContentServer
+    module Spec
+
+      describe 'Backup Listener' do
+
+      end
+
+      describe 'Local file monitor' do
+
+      end
+
+      describe 'Local file indexer' do
+
+      end
+
+      describe 'File copier' do
+
+      end
+
+    end
+  end
+end

data/spec/content_server/file_streamer_spec.rb

@@ -0,0 +1,71 @@
+require 'log'
+require 'rspec'
+require 'stringio'
+
+require_relative '../../lib/content_server/file_streamer'
+
+# Uncomment to debug spec.
+BBFS::Params['log_write_to_console'] = false
+BBFS::Params['log_write_to_file'] = false
+BBFS::Params['log_debug_level'] = 0
+BBFS::Params['streaming_chunk_size'] = 5
+BBFS::Params.init ARGV
+BBFS::Log.init
+# Have to be set to test chunking mechanism.
+
+module BBFS
+  module ContentServer
+    module Spec
+      describe 'FileStreamer' do
+        it 'should copy one file chunk by chunks and validate content' do
+          Log.info('#0 start')
+          orig_file = StringIO.new('Some content. Some content. Some content. Some content.')
+          Log.info("orig_file #{orig_file.to_s}.")
+
+          # should simulate Tempfile object, thus need to add to this object Tempfile methsods
+          # that are absent in StringIO, but used in tested ruby code
+          dest_file = StringIO.new
+          def dest_file.path
+            '/tmp/path/tmp_basename'
+          end
+          def dest_file.unlink
+            true
+          end
+          Log.info("dest_file #{dest_file.to_s}.")
+
+          streamer = nil
+          done = lambda{ |checksum, filename|
+            Log.info('#4 streaming done, check content ok.')
+            dest_file.string().should eq(orig_file.string())
+
+            Log.info('#5 exiting streamer thread.')
+            streamer.thread.exit
+          }
+
+          receiver = BBFS::ContentServer::FileReceiver.new(done)
+          send_chunk = lambda { |*args|
+            receiver.receive_chunk(*args)
+            streamer.copy_another_chuck('da39a3ee5e6b4b0d3255bfef95601890afd80709')
+          }
+
+          Log.info('#2 start streaming.')
+          # This is for FileStreamer :NEW_STREAM and FileReceiver :receive_chunk
+          ::File.stub(:new).and_return(orig_file, dest_file)
+          ::FileUtils.stub(:makedirs).and_return(true)
+          ::FileUtils.stub(:copy_file).and_return(true)
+          # This is for FileReceiver :handle_last_chunk
+          ::File.stub(:rename)
+          # This is for Index agent 'get_checksum' which opens file, read content and validates
+          # checksum.
+          ::File.stub(:open).and_return(dest_file)
+
+          streamer = BBFS::ContentServer::FileStreamer.new(send_chunk)
+          Log.info('#3 start streaming.')
+          streamer.start_streaming('da39a3ee5e6b4b0d3255bfef95601890afd80709', 'dummy')
+          streamer.thread.join()
+          sleep Params['log_param_max_elapsed_time_in_seconds_from_last_flush'] + 1
+        end
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,136 @@
+--- !ruby/object:Gem::Specification
+name: cs_fix
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Gena Petelko, Kolman Vornovitsky
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-10-06 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: content_data
+  requirement: &7713640 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *7713640
+- !ruby/object:Gem::Dependency
+  name: file_indexing
+  requirement: &7713220 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *7713220
+- !ruby/object:Gem::Dependency
+  name: file_monitoring
+  requirement: &7712780 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *7712780
+- !ruby/object:Gem::Dependency
+  name: log
+  requirement: &7712300 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *7712300
+- !ruby/object:Gem::Dependency
+  name: networking
+  requirement: &7711880 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *7711880
+- !ruby/object:Gem::Dependency
+  name: params
+  requirement: &7711440 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *7711440
+- !ruby/object:Gem::Dependency
+  name: rib_fix
+  requirement: &7698680 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - =
+      - !ruby/object:Gem::Version
+        version: 0.0.2
+  type: :runtime
+  prerelease: false
+  version_requirements: *7698680
+description: Monitor and Index a directory and back it up to backup server.
+email: kolmanv@gmail.com
+executables:
+- content_server
+- backup_server
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/content_server.rb
+- lib/content_server/file_streamer.rb
+- lib/content_server/queue_copy.rb
+- lib/content_server/content_receiver.rb
+- lib/content_server/remote_content.rb
+- lib/content_server/version.rb
+- lib/content_server/queue_indexer.rb
+- spec/content_server/content_server_spec.rb
+- spec/content_server/file_streamer_spec.rb
+- bin/content_server
+- bin/backup_server
+homepage: http://github.com/kolmanv/bbfs
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.10
+signing_key: 
+specification_version: 3
+summary: Servers for backing up content.
+test_files:
+- spec/content_server/content_server_spec.rb
+- spec/content_server/file_streamer_spec.rb