content_server 0.0.8 → 0.0.9a

data/bin/content_server

@@ -5,16 +5,25 @@
 # Each unique content is backed up to the remote (backup) server.
 
 begin
+  print "1"
   require 'yaml'
+  print "2"
   require 'params'
+  print "3"
   require 'run_in_background'
+  print "4"
   require 'content_server'
-rescue LoadError => e
-  print e.to_s
+  print "5"
+rescue LoadError
+  print "6"
   require 'rubygems'
+  print "7"
   require 'yaml'
+  print "8"
   require 'params'
+  print "9"
   require 'run_in_background'
+  print "0"
   require 'content_server'
 end
 include BBFS

data/lib/content_server/file_streamer.rb

@@ -1,3 +1,4 @@
+require 'tempfile'
 require 'thread'
 
 require 'file_indexing/index_agent'
@@ -6,7 +7,7 @@ require 'log'
 module BBFS
   module ContentServer
 
-    Params.integer('streaming_chunk_size', 64*1024,
+    Params.integer('streaming_chunk_size', 2*1024*1024,
                    'Max number of content bytes to send in one chunk.')
     Params.integer('file_streaming_timeout', 5*60,
                    'If no action is taken on a file streamer, abort copy.')
@@ -14,7 +15,7 @@ module BBFS
                   'Backup server destination folder, default is the relative local folder.')
 
     class Stream
-      attr_reader :checksum, :path, :file, :size
+      attr_reader :checksum, :path, :tmp_path, :file, :size
       def initialize(checksum, path, file, size)
         @checksum = checksum
         @path = path
@@ -42,6 +43,7 @@ module BBFS
 
       :NEW_STREAM
       :ABORT_STREAM
+      :RESET_STREAM
       :COPY_CHUNK
 
       def initialize(send_chunk_clb, abort_streaming_clb=nil)
@@ -54,6 +56,10 @@ module BBFS
         @thread = run
       end
 
+      def copy_another_chuck(checksum)
+        @stream_queue << [:COPY_CHUNK, checksum]
+      end
+
       def start_streaming(checksum, path)
         @stream_queue << [:NEW_STREAM, [checksum, path]]
       end
@@ -62,6 +68,10 @@ module BBFS
         @stream_queue << [:ABORT_STREAM, checksum]
       end
 
+      def reset_streaming(checksum, new_offset)
+        @stream_queue << [:RESET_STREAM, [checksum, new_offset]]
+      end
+
       def run
         return Thread.new do
           loop {
@@ -74,30 +84,28 @@ module BBFS
         type, content = message
         if type == :NEW_STREAM
           checksum, path = content
-          if !@streams.key? checksum
-            begin
-              file = File.new(path, 'rb')
-              Log.info("File streamer: #{file.to_s}.")
-            rescue IOError => e
-              Log.warning("Could not stream local file #{path}. #{e.to_s}")
-            end
-            @streams[checksum] = Stream.new(checksum, path, file, file.size)
-            @stream_queue << [:COPY_CHUNK, checksum]
-          end
+          reset_stream(checksum, path, 0)
+          @stream_queue << [:COPY_CHUNK, checksum] if @streams.key?(checksum)
         elsif type == :ABORT_STREAM
-          Stream.close_delete_stream(content, @streams)
+          checksum = content
+          Stream.close_delete_stream(checksum, @streams)
+        elsif type == :RESET_STREAM
+          checksum, new_offset = content
+          reset_stream(checksum, nil, new_offset)
+          @stream_queue << [:COPY_CHUNK, checksum] if @streams.key?(checksum)
         elsif type == :COPY_CHUNK
           checksum = content
           if @streams.key?(checksum)
+            offset = @streams[checksum].file.pos
+            Log.debug1("Sending chunk for #{checksum}, offset #{offset}.")
             chunk = @streams[checksum].file.read(Params['streaming_chunk_size'])
             if chunk.nil?
               # No more to read, send end of file.
-              @send_chunk_clb.call(checksum, @streams[checksum].size, nil, nil)
+              @send_chunk_clb.call(checksum, offset, @streams[checksum].size, nil, nil)
               Stream.close_delete_stream(checksum, @streams)
             else
               chunk_checksum = FileIndexing::IndexAgent.get_content_checksum(chunk)
-              @send_chunk_clb.call(checksum, @streams[checksum].size, chunk, chunk_checksum)
-              @stream_queue << [:COPY_CHUNK, checksum]
+              @send_chunk_clb.call(checksum, offset, @streams[checksum].size, chunk, chunk_checksum)
             end
           else
             Log.info("No checksum found to copy chunk. #{checksum}.")
@@ -105,6 +113,23 @@ module BBFS
         end
 
       end
+
+      def reset_stream(checksum, path, offset)
+        if !@streams.key? checksum
+          begin
+            file = File.new(path, 'rb')
+            if offset > 0
+              file.seek(offset)
+            end
+            Log.info("File streamer: #{file.to_s}.")
+          rescue IOError => e
+            Log.warning("Could not stream local file #{path}. #{e.to_s}")
+          end
+          @streams[checksum] = Stream.new(checksum, path, file, file.size)
+        else
+          @streams[checksum].file.seek(offset)
+        end
+      end
     end
 
     # Start implementing as dummy, no self thread for now.
@@ -112,67 +137,132 @@ module BBFS
     # need self thread.
     class FileReceiver
 
-      def initialize(file_done_clb=nil, file_abort_clb=nil)
+      def initialize(file_done_clb=nil, file_abort_clb=nil, reset_copy=nil)
         @file_done_clb = file_done_clb
         @file_abort_clb = file_abort_clb
+        @reset_copy = reset_copy
         @streams = {}
       end
 
-      def receive_chunk(file_checksum, file_size, content, content_checksum)
+      def receive_chunk(file_checksum, offset, file_size, content, content_checksum)
+        # If standard chunk copy.
         if !content.nil? && !content_checksum.nil?
           received_content_checksum = FileIndexing::IndexAgent.get_content_checksum(content)
           comment = "Calculated received chunk with content checksum #{received_content_checksum}" \
                     " vs message content checksum #{content_checksum}, " \
                     "file checksum #{file_checksum}"
           Log.debug1(comment) if content_checksum == received_content_checksum
-          Log.error(comment) if content_checksum != received_content_checksum
+          # TODO should be here a kind of abort?
+          if content_checksum != received_content_checksum
+            Log.warning(comment)
+            new_offset = 0
+            if @streams.key?(file_checksum)
+              new_offset = @streams[file_checksum].file.pos
+            end
+            @reset_copy.call(file_checksum, new_offset) unless @reset_copy.nil?
+            return false
+          end
 
           if !@streams.key?(file_checksum)
-            path = FileReceiver.destination_filename(Params['backup_destination_folder'],
-                                                     file_checksum)
-            if File.exists?(path)
-              Log.warning("File already exists (#{path}) not writing.")
-              @file_abort_clb.call(file_checksum) unless @file_abort_clb.nil?
-            else
-              begin
-                # Make the directory if does not exists.
-                Log.debug1("Writing to: #{path}")
-                Log.debug1("Creating directory: #{File.dirname(path)}")
-                FileUtils.makedirs(File.dirname(path))
-                file = File.new(path, 'wb')
-                @streams[file_checksum] = Stream.new(file_checksum, path, file, file_size)
-              rescue IOError => e
-                Log.warning("Could not stream write to local file #{path}. #{e.to_s}")
-              end
-            end
+            handle_new_stream(file_checksum, file_size)
           end
           # We still check @streams has the key, because file can fail to open.
           if @streams.key?(file_checksum)
-            FileReceiver.write_string_to_file(content, @streams[file_checksum].file)
-            Log.info("Written already #{@streams[file_checksum].file.size} bytes, " \
-                     "out of #{file_size} (#{100.0*@streams[file_checksum].file.size/file_size}%)")
+            return handle_new_chunk(file_checksum, offset, content)
+          else
+            Log.warning('Cannot handle chunk, stream does not exists, sending abort.')
+            @file_abort_clb.call(file_checksum) unless @file_abort_clb.nil?
+            return false
           end
+        # If last chunk copy.
         elsif content.nil? && content_checksum.nil?
-          if @streams.key?(file_checksum)
-            # Check written file checksum!
-            local_path = @streams[file_checksum].path
-            Stream.close_delete_stream(file_checksum, @streams)
-
-            local_file_checksum = FileIndexing::IndexAgent.get_checksum(local_path)
-            message = "Local checksum (#{local_file_checksum}) received checksum (#{file_checksum})."
-            Log.info(message) ? local_file_checksum == file_checksum : Log.error(message)
-            Log.info("File fully received #{local_path}")
-            @file_done_clb.call(local_file_checksum, local_path) unless @file_done_clb.nil?
-          end
+          handle_last_chunk(file_checksum)
+          return false
         else
           Log.warning("Unexpected receive chuck message. file_checksum:#{file_checksum}, " \
                       "content.nil?:#{content.nil?}, content_checksum:#{content_checksum}")
+          return false
+        end
+      end
+
+      # open new stream
+      def handle_new_stream(file_checksum, file_size)
+        # final destination path
+        tmp_path = FileReceiver.destination_filename(
+                       File.join(Params['backup_destination_folder'], 'tmp'),
+                       file_checksum)
+        path = FileReceiver.destination_filename(Params['backup_destination_folder'],
+                                                 file_checksum)
+        if File.exists?(path)
+          Log.warning("File already exists (#{path}) not writing.")
+          @file_abort_clb.call(file_checksum) unless @file_abort_clb.nil?
+        else
+          # the file will be moved from tmp location once the transfer will be done
+          # system will use the checksum and some more unique key for tmp file name
+          FileUtils.makedirs(File.dirname(tmp_path)) unless File.directory?(File.dirname(tmp_path))
+          tmp_file = file = File.new(tmp_path, 'wb')
+          @streams[file_checksum] = Stream.new(file_checksum, tmp_path, tmp_file, file_size)
+        end
+      end
+
+      # write chunk to temp file
+      def handle_new_chunk(file_checksum, offset, content)
+        if offset == @streams[file_checksum].file.pos
+          FileReceiver.write_string_to_file(content, @streams[file_checksum].file)
+          Log.info("Written already #{@streams[file_checksum].file.pos} bytes, " \
+                 "out of #{@streams[file_checksum].size} " \
+                 "(#{100.0*@streams[file_checksum].file.size/@streams[file_checksum].size}%)")
+          return true
+        else
+          # Offset is wrong, send reset/resume copy from correct offset.
+          Log.warning("Received chunk with incorrect offset #{offset}, should " \
+                      "be #{@streams[file_checksum].file.pos}, file_checksum:#{file_checksum}")
+          @reset_copy.call(file_checksum, @streams[file_checksum].file.pos) unless @reset_copy.nil?
+          return false
+        end
+      end
+
+      # copy file to permanent location
+      # close stream
+      # remove temp file
+      # check written file
+      def handle_last_chunk(file_checksum)
+        if @streams.key?(file_checksum)
+          # Make the directory if does not exists.
+          path = FileReceiver.destination_filename(Params['backup_destination_folder'],
+                                                   file_checksum)
+          Log.debug1("Moving tmp file #{@streams[file_checksum].path} to #{path}")
+          Log.debug1("Creating directory: #{path}")
+          file_dir = File.dirname(path)
+          FileUtils.makedirs(file_dir) unless File.directory?(file_dir)
+          # Move tmp file to permanent location.
+          tmp_file_path = @streams[file_checksum].path
+          Stream.close_delete_stream(file_checksum, @streams)  # temp file will be closed here
+
+          local_file_checksum = FileIndexing::IndexAgent.get_checksum(tmp_file_path)
+          message = "Local checksum (#{local_file_checksum}) received checksum (#{file_checksum})."
+          if local_file_checksum == file_checksum
+            Log.info(message)
+            begin
+              File.rename(tmp_file_path, path)
+              Log.info("End move tmp file to permanent location #{path}.")
+              @file_done_clb.call(local_file_checksum, path) unless @file_done_clb.nil?
+            rescue IOError => e
+              Log.warning("Could not move tmp file to permanent file #{path}. #{e.to_s}")
+            end
+          else
+            Log.error(message)
+            Log.debug1("Deleting tmp file: #{tmp_file_path}")
+            File.delete(tmp_file_path)
+          end
+        else
+          Log.error("Handling last chunk and tmp stream does not exists.")
         end
       end
 
       def self.write_string_to_file(str, file)
-        Log.info("writing to file: #{file.to_s}.")
         bytes_to_write = str.bytesize
+        Log.info("writing to file: #{file.to_s}, #{bytes_to_write} bytes.")
         while bytes_to_write > 0
           bytes_to_write -= file.write(str)
         end
@@ -185,6 +275,7 @@ module BBFS
         File.join(folder, sha1[0,2], sha1[2,2], sha1)
       end
 
+      private :handle_new_stream, :handle_new_chunk, :handle_last_chunk
     end
 
   end

data/lib/content_server/queue_copy.rb

@@ -14,7 +14,9 @@ module BBFS
     :COPY_MESSAGE
     :SEND_COPY_MESSAGE
     :COPY_CHUNK
-    :ABORT_COPY
+    :COPY_CHUNK_FROM_REMOTE
+    :ABORT_COPY # Asks the sender to abort file copy.
+    :RESET_RESUME_COPY # Sends the stream sender to resend chunk or resume from different offset.
 
     # Simple copier, gets inputs events (files to copy), requests ack from backup to copy
     # then copies one file.
@@ -52,9 +54,12 @@ module BBFS
               Log.info "Copy file event: #{message_content}"
               # Prepare source,dest map for copy.
               message_content.instances.each { |key, instance|
-                @copy_prepare[instance.checksum] = instance.full_path
-                Log.info("Sending ack for: #{instance.checksum}")
-                @backup_tcp.send_obj([:ACK_MESSAGE, [instance.checksum, Time.now.to_i]])
+                # If not already sending.
+                if !@copy_prepare.key?(instance.checksum) || !@copy_prepare[instance.checksum][1]
+                  @copy_prepare[instance.checksum] = [instance.full_path, false]
+                  Log.info("Sending ack for: #{instance.checksum}")
+                  @backup_tcp.send_obj([:ACK_MESSAGE, [instance.checksum, Time.now.to_i]])
+                end
               }
             elsif message_type == :ACK_MESSAGE
               # Received ack from backup, copy file if all is good.
@@ -66,20 +71,28 @@ module BBFS
 
               # Copy file if ack (does not exists on backup and not too much time passed)
               if ack && (Time.now.to_i - timestamp < Params['ack_timeout'])
-                if !@copy_prepare.key?(checksum)
-                  Log.warning("Ack was already received:#{checksum}")
+                if !@copy_prepare.key?(checksum) || @copy_prepare[checksum][1]
+                  Log.warning("File was aborted, copied, or started copy just now: #{checksum}")
                 else
-                  path = @copy_prepare[checksum]
+                  path = @copy_prepare[checksum][0]
                   Log.info "Streaming file: #{checksum} #{path}."
                   @file_streamer.start_streaming(checksum, path)
+                  # Ack received, setting prepare to true
+                  @copy_prepare[checksum][1] = true
                 end
               else
                 Log.debug1("Ack timed out span: #{Time.now.to_i - timestamp} > " \
                            "timeout: #{Params['ack_timeout']}")
               end
+            elsif message_type == :COPY_CHUNK_FROM_REMOTE
+              checksum = message_content
+              @file_streamer.copy_another_chuck(checksum)
             elsif message_type == :COPY_CHUNK
-              file_checksum, file_size, content, content_checksum = message_content
-              Log.info "Send chunk for file #{file_checksum}."
+              # We open the message here for printing info and deleting copy_prepare!
+              file_checksum, offset, file_size, content, content_checksum = message_content
+              Log.info("Send chunk for file #{file_checksum}, offset: #{offset} " \
+                       "filesize: #{file_size}.")
+              # Blocking send.
               @backup_tcp.send_obj([:COPY_CHUNK, message_content])
               if content.nil? and content_checksum.nil?
                 @copy_prepare.delete(file_checksum)
@@ -87,14 +100,19 @@ module BBFS
             elsif message_type == :ABORT_COPY
               Log.info("Aborting file copy: #{message_content}")
               if @copy_prepare.key?(message_content)
-                Log.info("Aborting: #{@copy_prepare[message_content]}")
+                Log.info("Aborting: #{@copy_prepare[message_content][0]}")
                 @copy_prepare.delete(message_content)
               end
               @file_streamer.abort_streaming(message_content)
+            elsif message_type == :RESET_RESUME_COPY
+              file_checksum, new_offset = message_content
+              Log.info("Resetting/Resuming file (#{file_checksum}) copy to #{new_offset}")
+              @file_streamer.reset_streaming(file_checksum, new_offset)
             else
               Log.error("Copy event not supported: #{message_type}")
             end # handle messages here
           end
+          Log.error("Should not reach here, loop should continue.")
         end
       end
     end  # class QueueCopy
@@ -104,12 +122,15 @@ module BBFS
         @local_queue = Queue.new
         @dynamic_content_data = dynamic_content_data
         @tcp_server = Networking::TCPClient.new(host, port, method(:handle_message))
-        @file_receiver = FileReceiver.new(method(:done_copy), method(:abort_copy))
+        @file_receiver = FileReceiver.new(method(:done_copy),
+                                          method(:abort_copy),
+                                          method(:reset_copy))
         @local_thread = Thread.new do
           loop do
             handle(@local_queue.pop)
           end
         end
+        @local_thread.abort_on_exception = true
       end
 
       def threads
@@ -126,6 +147,10 @@ module BBFS
         handle_message([:ABORT_COPY, checksum])
       end
 
+      def reset_copy(checksum, new_offset)
+        handle_message([:RESET_RESUME_COPY, [checksum, new_offset]])
+      end
+
       def done_copy(local_file_checksum, local_path)
         Log.info("Done copy file: #{local_path}, #{local_file_checksum}")
       end
@@ -145,7 +170,11 @@ module BBFS
           bytes_written = @tcp_server.send_obj([:COPY_MESSAGE, message_content])
           Log.debug1("Sending copy message succeeded? bytes_written: #{bytes_written}.")
         elsif message_type == :COPY_CHUNK
-          @file_receiver.receive_chunk(*message_content)
+          Log.debug1('Chunk received.')
+          if @file_receiver.receive_chunk(*message_content)
+            file_checksum, offset, file_size, content, content_checksum = message_content
+            @tcp_server.send_obj([:COPY_CHUNK_FROM_REMOTE, file_checksum])
+          end
         elsif message_type == :ACK_MESSAGE
           checksum, timestamp = message_content
           # Here we should check file existence
@@ -156,6 +185,8 @@ module BBFS
                                                checksum]])
         elsif message_type == :ABORT_COPY
           @tcp_server.send_obj([:ABORT_COPY, message_content])
+        elsif message_type == :RESET_RESUME_COPY
+          @tcp_server.send_obj([:RESET_RESUME_COPY, message_content])
         else
           Log.error("Unexpected message type: #{message_type}")
         end

data/lib/content_server/remote_content.rb

@@ -1,96 +1,97 @@
-require 'thread'
-
-require 'content_data/dynamic_content_data'
-require 'log'
-require 'networking/tcp'
-require 'params'
-
-module BBFS
-  module ContentServer
-
-    Params.integer('remote_content_timeout', 10, 'Remote content desired freshness in seconds.')
-    Params.integer('max_content_timeout', 60*60, 'Remote content force refresh in seconds.')
-
-    # TODO(kolman): Use only one tcp/ip socket by utilizing one NQueue for many queues!
-    class RemoteContent
-      def initialize(dynamic_content_data, host, port, local_backup_folder)
-        @dynamic_content_data = dynamic_content_data
-        @remote_tcp = Networking::TCPClient.new(host, port, method(:receive_content))
-        @last_update_timestamp = nil
-        @content_server_content_data_path = File.join(local_backup_folder, 'remote',
-                                                      host + '_' + port.to_s)
-        FileUtils.makedirs(@content_server_content_data_path)
-      end
-
-      def receive_content(message)
-        Log.debug1("Remote content data received: #{message.to_s}")
-        ref = @dynamic_content_data.last_content_data
-        @dynamic_content_data.update(message)
-
-        max_time_span = Params['max_content_timeout']
-        if !@last_update_timestamp.nil?
-          max_time_span = Time.now.to_i - @last_update_timestamp
-        end
-
-        @last_update_timestamp = Time.now.to_i
-
-        if ref != message || max_time_span >= Params['max_content_timeout']
-          Log.debug2("Remote content data changed or max time span is large, writing.")
-          Log.debug3("max_time_span: #{max_time_span}")
-          write_to = File.join(@content_server_content_data_path,
-                               @last_update_timestamp.to_s + '.cd')
-          count = File.open(write_to, 'wb') { |f| f.write(message.to_s) }
-        else
-          Log.debug2("No need to write remote content data, it has not changed.")
-        end
-      end
-
-      def run()
-        threads = []
-        threads << @remote_tcp.tcp_thread if @remote_tcp != nil
-        threads << Thread.new do
-          loop do
-            # if need content data
-            if @last_update_timestamp.nil?
-              sleep_time_span = Params['remote_content_timeout']
-            else
-              sleep_time_span = Time.now.to_i - @last_update_timestamp
-            end
-
-            if sleep_time_span >= Params['remote_content_timeout']
-              # Send ping!
-              Log.debug2('Sending remote contend request.')
-              bytes_written = @remote_tcp.send_obj(nil)
-              Log.debug3("Bytes written #{bytes_written}.")
-            end
-
-            sleep_time_span = Time.now.to_i - @last_update_timestamp \
-                unless @last_update_timestamp.nil?
-            Log.debug2("sleep_time_span: #{sleep_time_span}")
-            sleep(sleep_time_span) if sleep_time_span > 0
-          end
-        end
-      end
-    end
-
-    class RemoteContentClient
-      def initialize(dynamic_content_data, port)
-        @dynamic_content_data = dynamic_content_data
-        @tcp_server = Networking::TCPServer.new(port, method(:content_requested))
-      end
-
-      def content_requested(addr_info, message)
-        # Send response.
-        Log.debug1('Local content data requested.')
-        @tcp_server.send_obj(@dynamic_content_data.last_content_data)
-      end
-
-      def tcp_thread
-        return @tcp_server.tcp_thread if @tcp_server != nil
-        nil
-      end
-
-    end
-
-  end
-end
+require 'thread'
+
+require 'content_data/dynamic_content_data'
+require 'log'
+require 'networking/tcp'
+require 'params'
+
+module BBFS
+  module ContentServer
+
+    Params.integer('remote_content_timeout', 10, 'Remote content desired freshness in seconds.')
+    Params.integer('max_content_timeout', 60*60, 'Remote content force refresh in seconds.')
+
+    # TODO(kolman): Use only one tcp/ip socket by utilizing one NQueue for many queues!
+    class RemoteContent
+      def initialize(dynamic_content_data, host, port, local_backup_folder)
+        @dynamic_content_data = dynamic_content_data
+        @remote_tcp = Networking::TCPClient.new(host, port, method(:receive_content))
+        @last_update_timestamp = nil
+        @content_server_content_data_path = File.join(local_backup_folder, 'remote',
+                                                      host + '_' + port.to_s)
+      end
+
+      def receive_content(message)
+        Log.debug1("Remote content data received: #{message.to_s}")
+        ref = @dynamic_content_data.last_content_data
+        @dynamic_content_data.update(message)
+
+        max_time_span = Params['max_content_timeout']
+        if !@last_update_timestamp.nil?
+          max_time_span = Time.now.to_i - @last_update_timestamp
+        end
+
+        @last_update_timestamp = Time.now.to_i
+
+        if ref != message || max_time_span >= Params['max_content_timeout']
+          Log.debug2("Remote content data changed or max time span is large, writing.")
+          Log.debug3("max_time_span: #{max_time_span}")
+          write_to = File.join(@content_server_content_data_path,
+                               @last_update_timestamp.to_s + '.cd')
+          FileUtils.makedirs(@content_server_content_data_path) unless \
+              File.directory?(@content_server_content_data_path)
+          count = File.open(write_to, 'wb') { |f| f.write(message.to_s) }
+        else
+          Log.debug2("No need to write remote content data, it has not changed.")
+        end
+      end
+
+      def run()
+        threads = []
+        threads << @remote_tcp.tcp_thread if @remote_tcp != nil
+        threads << Thread.new do
+          loop do
+            # if need content data
+            if @last_update_timestamp.nil?
+              sleep_time_span = Params['remote_content_timeout']
+            else
+              sleep_time_span = Time.now.to_i - @last_update_timestamp
+            end
+
+            if sleep_time_span >= Params['remote_content_timeout']
+              # Send ping!
+              Log.debug2('Sending remote contend request.')
+              bytes_written = @remote_tcp.send_obj(nil)
+              Log.debug3("Bytes written #{bytes_written}.")
+            end
+
+            sleep_time_span = Time.now.to_i - @last_update_timestamp \
+                unless @last_update_timestamp.nil?
+            Log.debug2("sleep_time_span: #{sleep_time_span}")
+            sleep(sleep_time_span) if sleep_time_span > 0
+          end
+        end
+      end
+    end
+
+    class RemoteContentClient
+      def initialize(dynamic_content_data, port)
+        @dynamic_content_data = dynamic_content_data
+        @tcp_server = Networking::TCPServer.new(port, method(:content_requested))
+      end
+
+      def content_requested(addr_info, message)
+        # Send response.
+        Log.debug1('Local content data requested.')
+        @tcp_server.send_obj(@dynamic_content_data.last_content_data)
+      end
+
+      def tcp_thread
+        return @tcp_server.tcp_thread if @tcp_server != nil
+        nil
+      end
+
+    end
+
+  end
+end

data/lib/content_server/version.rb

@@ -1,5 +1,5 @@
 module BBFS
   module ContentServer
-    VERSION = "0.0.8"
+    VERSION = "0.0.9a"
   end
 end

data/spec/content_server/file_streamer_spec.rb

@@ -5,7 +5,7 @@ require 'stringio'
 require_relative '../../lib/content_server/file_streamer'
 
 # Uncomment to debug spec.
-#BBFS::Params['log_write_to_console'] = true
+BBFS::Params['log_write_to_console'] = false
 BBFS::Params['log_write_to_file'] = false
 BBFS::Params['log_debug_level'] = 0
 BBFS::Params['streaming_chunk_size'] = 5
@@ -21,8 +21,18 @@ module BBFS
           Log.info('#0 start')
           orig_file = StringIO.new('Some content. Some content. Some content. Some content.')
           Log.info("orig_file #{orig_file.to_s}.")
+
+          # should simulate Tempfile object, thus need to add to this object Tempfile methsods
+          # that are absent in StringIO, but used in tested ruby code
           dest_file = StringIO.new
+          def dest_file.path
+            '/tmp/path/tmp_basename'
+          end
+          def dest_file.unlink
+            true
+          end
           Log.info("dest_file #{dest_file.to_s}.")
+
           streamer = nil
           done = lambda{ |checksum, filename|
             Log.info('#4 streaming done, check content ok.')
@@ -31,13 +41,20 @@ module BBFS
             Log.info('#5 exiting streamer thread.')
             streamer.thread.exit
           }
+
           receiver = BBFS::ContentServer::FileReceiver.new(done)
           send_chunk = lambda { |*args|
             receiver.receive_chunk(*args)
+            streamer.copy_another_chuck('da39a3ee5e6b4b0d3255bfef95601890afd80709')
           }
+
           Log.info('#2 start streaming.')
-          # This is for 1) FileStreamer :NEW_STREAM and 2) FileReceiver receive_chunk.
+          # This is for FileStreamer :NEW_STREAM and FileReceiver :receive_chunk
           ::File.stub(:new).and_return(orig_file, dest_file)
+          ::FileUtils.stub(:makedirs).and_return(true)
+          ::FileUtils.stub(:copy_file).and_return(true)
+          # This is for FileReceiver :handle_last_chunk
+          ::File.stub(:rename)
           # This is for Index agent 'get_checksum' which opens file, read content and validates
           # checksum.
           ::File.stub(:open).and_return(dest_file)
@@ -46,6 +63,7 @@ module BBFS
           Log.info('#3 start streaming.')
           streamer.start_streaming('da39a3ee5e6b4b0d3255bfef95601890afd80709', 'dummy')
           streamer.thread.join()
+          sleep Params['log_param_max_elapsed_time_in_seconds_from_last_flush'] + 1
         end
       end
     end

metadata

@@ -1,15 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: content_server
 version: !ruby/object:Gem::Version
-  version: 0.0.8
-  prerelease: 
+  version: 0.0.9a
+  prerelease: 5
 platform: ruby
 authors:
 - Gena Petelko, Kolman Vornovitsky
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-09-02 00:00:00.000000000 Z
+date: 2012-09-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: content_data
@@ -157,12 +157,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
-  - - ! '>='
+  - - ! '>'
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.23
+rubygems_version: 1.8.24
 signing_key: 
 specification_version: 3
 summary: Servers for backing up content.