content_server 1.3.1 → 1.4.0

data/lib/content_data/content_data.rb

@@ -29,6 +29,8 @@ module ContentData
   #      instances which was added to @contents_info
   class ContentData
 
+    CHUNK_SIZE = 5000
+
     def initialize(other = nil)
       if other.nil?
         @contents_info = {}  # Checksum --> [size, paths-->time(instance), time(content)]
@@ -46,34 +48,44 @@ module ContentData
     end
 
     def clone_instances_info
-      @instances_info.keys.inject({}) { |clone_instances_info, location|
+      clone_instances_info = {}
+      instances_info_enum = @instances_info.each_key
+      loop {
+        location = instances_info_enum.next rescue break
         clone_instances_info[[location[0].clone, location[1].clone]] = @instances_info[location].clone
-        clone_instances_info
       }
+      clone_instances_info
     end
 
     def clone_contents_info
-      @contents_info.keys.inject({}) { |clone_contents_info, checksum|
+      clone_contents_info = {}
+      contents_info_enum = @contents_info.each_key
+      loop {
+        checksum = contents_info_enum.next rescue break
         instances = @contents_info[checksum]
         size = instances[0]
         content_time = instances[2]
         instances_db = instances[1]
         instances_db_cloned = {}
-        instances_db.keys.each { |location|
+        instances_db_enum = instances_db.each_key
+        loop {
+          location =  instances_db_enum.next rescue break
           instance_mtime = instances_db[location]
           instances_db_cloned[[location[0].clone,location[1].clone]]=instance_mtime
         }
         clone_contents_info[checksum] = [size,
                               instances_db_cloned,
                               content_time]
-        clone_contents_info
       }
+      clone_contents_info
     end
 
     # iterator over @contents_info data structure (not including instances)
     # block is provided with: checksum, size and content modification time
     def each_content(&block)
-      @contents_info.keys.each { |checksum|
+      contents_enum = @contents_info.each_key
+      loop {
+        checksum = contents_enum.next rescue break
         content_val = @contents_info[checksum]
         # provide checksum, size and content modification time to the block
         block.call(checksum,content_val[0], content_val[2])
@@ -84,9 +96,13 @@ module ContentData
     # block is provided with: checksum, size, content modification time,
     #   instance modification time, server and file path
     def each_instance(&block)
-      @contents_info.keys.each { |checksum|
+      contents_enum = @contents_info.each_key
+      loop {
+        checksum = contents_enum.next rescue break
         content_info = @contents_info[checksum]
-        content_info[1].keys.each {|location|
+        content_info_enum = content_info[1].each_key
+        loop {
+          location = content_info_enum.next rescue break
           # provide the block with: checksum, size, content modification time,instance modification time,
           #   server and path.
           instance_modification_time = content_info[1][location]
@@ -101,7 +117,9 @@ module ContentData
     #   instance modification time, server and file path
     def content_each_instance(checksum, &block)
       content_info = @contents_info[checksum]
-      content_info[1].keys.each {|location|
+      instances_db_enum = content_info[1].each_key
+      loop {
+        location = instances_db_enum.next rescue break
         # provide the block with: checksum, size, content modification time,instance modification time,
         #   server and path.
         instance_modification_time = content_info[1][location]
@@ -115,11 +133,7 @@ module ContentData
     end
 
     def instances_size()
-      counter=0
-      @contents_info.values.each { |content_info|
-        counter += content_info[1].length
-      }
-      counter
+      @instances_info.length
     end
 
     def checksum_instances_size(checksum)
@@ -137,6 +151,13 @@ module ContentData
 
     def add_instance(checksum, size, server, path, modification_time)
       location = [server, path]
+
+      # file was changed but remove_instance was not called
+      if (@instances_info.include?(location) && @instances_info[location] != checksum)
+        Log.warning("#{server}:#{path} file already exists with different checksum")
+        remove_instance(server, path)
+      end
+
       content_info = @contents_info[checksum]
       if content_info.nil?
         @contents_info[checksum] = [size,
@@ -186,8 +207,10 @@ module ContentData
     # found records are removed from both @instances_info and @instances_info.
     # input params: server & dir_to_remove - are used to check each instance unique key (called location)
     # removes also content\s, if a content\s become\s empty after removing instance\s
-    def remove_directory(server, dir_to_remove)
-      @contents_info.keys.each { |checksum|
+    def remove_directory(dir_to_remove, server)
+      contents_enum = @contents_info.each_key
+      loop {
+        checksum = contents_enum.next rescue break
         instances =  @contents_info[checksum][1]
         instances.each_key { |location|
           if location[0] == server and location[1].scan(dir_to_remove).size > 0
@@ -247,70 +270,181 @@ module ContentData
       return_str
     end
 
+    # Write content data to file.
+    # Write is using chunks (for both content chunks and instances chunks)
+    # Chunk is used to maximize GC affect. The temporary memory of each chunk is GCed.
+    # Without the chunks used in a dipper stack level, GC keeps the temporary objects as part of the stack context.
     def to_file(filename)
       content_data_dir = File.dirname(filename)
       FileUtils.makedirs(content_data_dir) unless File.directory?(content_data_dir)
-      file = File.open(filename, 'w')
-      file.write("#{@contents_info.length}\n")
-      each_content { |checksum, size, content_mod_time|
-        file.write("#{checksum},#{size},#{content_mod_time}\n")
-      }
-      file.write("#{@instances_info.length}\n")
-      each_instance { |checksum, size, _, instance_mod_time, server, path|
-        file.write("#{checksum},#{size},#{server},#{path},#{instance_mod_time}\n")
+      File.open(filename, 'w') { |file|
+        file.write("#{@contents_info.length}\n")
+        contents_enum = @contents_info.each_key
+        content_chunks = @contents_info.length / CHUNK_SIZE + 1
+        chunks_counter = 0
+        while chunks_counter < content_chunks
+          to_file_contents_chunk(file,contents_enum, CHUNK_SIZE)
+          GC.start
+          chunks_counter += 1
+        end
+        file.write("#{@instances_info.length}\n")
+        contents_enum = @contents_info.each_key
+        chunks_counter = 0
+        while chunks_counter < content_chunks
+          to_file_instances_chunk(file,contents_enum, CHUNK_SIZE)
+          GC.start
+          chunks_counter += 1
+        end
       }
-      file.close
+    end
+
+    def to_file_contents_chunk(file, contents_enum, chunk_size)
+      chunk_counter = 0
+      while chunk_counter < chunk_size
+        checksum = contents_enum.next rescue return
+        content_info = @contents_info[checksum]
+        file.write("#{checksum},#{content_info[0]},#{content_info[2]}\n")
+        chunk_counter += 1
+      end
+    end
+
+    def to_file_instances_chunk(file, contents_enum, chunk_size)
+      chunk_counter = 0
+      while chunk_counter < chunk_size
+        checksum = contents_enum.next rescue return
+        content_info = @contents_info[checksum]
+        instances_db_enum = content_info[1].each_key
+        loop {
+          location = instances_db_enum.next rescue break
+          # provide the block with: checksum, size, content modification time,instance modification time,
+          #   server and path.
+          instance_modification_time = content_info[1][location]
+          file.write("#{checksum},#{content_info[0]},#{location[0]},#{location[1]},#{instance_modification_time}\n")
+        }
+        chunk_counter += 1
+        break if chunk_counter == chunk_size
+      end
     end
 
     # TODO validation that file indeed contains ContentData missing
+    # Loading db from file using chunks for better memory performance
     def from_file(filename)
-      lines = IO.readlines(filename)
-      number_of_contents = lines[0].to_i
-      i = 1 + number_of_contents
-      number_of_instances = lines[i].to_i
-      i += 1
-      number_of_instances.times {
-        if lines[i].nil?
-          Log.warning("line ##{i} is nil !!!, Backing filename: #{filename} to #{filename}.bad")
-          FileUtils.cp(filename, "#{filename}.bad")
-          Log.warning("Lines:\n#{lines[i].join("\n")}")
-        else
-          parameters = lines[i].split(',')
-          # bugfix: if file name consist a comma then parsing based on comma separating fails
-          if (parameters.size > 5)
-            (4..parameters.size-2).each do |i|
-              parameters[3] = [parameters[3], parameters[i]].join(",")
-            end
-            (4..parameters.size-2).each do |i|
-              parameters.delete_at(4)
-            end
+      # read first line (number of contents)
+      # calculate line number (number of instances)
+      # read number of instances.
+      # loop over instances lines (using chunks) and add instances
+
+      File.open(filename, 'r') { |file|
+        # Get number of contents (at first line)
+        number_of_contents = file.gets  # this gets the next line or return nil at EOF
+        unless (number_of_contents and number_of_contents.match(/^[\d]+$/))  # check that line is of Number format
+          return reset_load_from_file(filename, file, "number of contents should be a number. We got:#{number_of_contents}")
+        end
+        number_of_contents = number_of_contents.to_i
+        # advance file lines over all contents. We need only the instances data to build the content data object
+        # use chunks and GC
+        contents_chunks = number_of_contents / CHUNK_SIZE
+        contents_chunks += 1 if (contents_chunks * CHUNK_SIZE < number_of_contents)
+        chunk_index = 0
+        while chunk_index < contents_chunks
+          chunk_size = CHUNK_SIZE
+          if chunk_index + 1 == contents_chunks
+            # update last chunk size
+            chunk_size = number_of_contents - (chunk_index * CHUNK_SIZE)
           end
+          return unless read_contents_chunk(filename, file, chunk_size)
+          GC.start
+          chunk_index += 1
+        end
 
-          add_instance(parameters[0],
-                       parameters[1].to_i,
-                       parameters[2],
-                       parameters[3],
-                       parameters[4].to_i)
+        # get number of instances
+        number_of_instances = file.gets
+        unless (number_of_instances and number_of_instances.match(/^[\d]+$/))  # check that line is of Number format
+          return reset_load_from_file(filename, file, "number of instances should be a Number. We got:#{number_of_instances}")
+        end
+        number_of_instances = number_of_instances.to_i
+        # read in instances chunks and GC
+        instances_chunks = number_of_instances / CHUNK_SIZE
+        instances_chunks += 1 if (instances_chunks * CHUNK_SIZE < number_of_instances)
+        chunk_index = 0
+        while chunk_index < instances_chunks
+          chunk_size = CHUNK_SIZE
+          if chunk_index + 1 == instances_chunks
+            # update last chunk size
+            chunk_size = number_of_instances - (chunk_index * CHUNK_SIZE)
+          end
+          return unless read_instances_chunk(filename, file, chunk_size)
+          GC.start
+          chunk_index += 1
         end
-        i += 1
       }
     end
 
+    def read_contents_chunk(filename, file, chunk_size)
+      chunk_index = 0
+      while chunk_index < chunk_size
+        return reset_load_from_file(filename, file, "Expecting content line but " +
+            "reached end of file after line #{$.}") unless file.gets
+        chunk_index += 1
+      end
+      true
+    end
+
+    def  read_instances_chunk(filename, file, chunk_size)
+      chunk_index = 0
+      while chunk_index < chunk_size
+        instance_line = file.gets
+        return reset_load_from_file(filename, file, "Expected to read Instance line but reached EOF") unless instance_line
+        parameters = instance_line.split(',')
+        # bugfix: if file name consist a comma then parsing based on comma separating fails
+        if (parameters.size > 5)
+          (4..parameters.size-2).each do |i|
+            parameters[3] = [parameters[3], parameters[i]].join(",")
+          end
+          (4..parameters.size-2).each do |i|
+            parameters.delete_at(4)
+          end
+        end
+
+        add_instance(parameters[0],
+                     parameters[1].to_i,
+                     parameters[2],
+                     parameters[3],
+                     parameters[4].to_i)
+        chunk_index += 1
+      end
+      true
+    end
+
+    def reset_load_from_file(file_name, file_io, err_msg)
+      Log.error("unexpected error reading file:#{file_name}\nError message:#{err_msg}")
+      @contents_info = {}  # Checksum --> [size, paths-->time(instance), time(content)]
+      @instances_info = {}  # location --> checksum to optimize instances query
+      file_io.close
+      nil
+    end
+
     # for each content, all time fields (content and instances) are replaced with the
     # min time found, while going through all time fields.
     def unify_time()
-      @contents_info.keys.each { |checksum|
+      contents_enum = @contents_info.each_key
+      loop {
+        checksum = contents_enum.next rescue break
         content_info = @contents_info[checksum]
         min_time_per_checksum = content_info[2]
         instances = content_info[1]
-        instances.keys.each { |location|
+        instances_enum = instances.each_key
+        loop {
+          location = instances_enum.next rescue break
           instance_mod_time = instances[location]
           if instance_mod_time < min_time_per_checksum
             min_time_per_checksum = instance_mod_time
           end
         }
         # update all instances with min time
-        instances.keys.each { |location|
+        instances_enum = instances.each_key
+        loop {
+          location = instances_enum.next rescue break
           instances[location] = min_time_per_checksum
         }
         # update content time with min time
@@ -353,11 +487,15 @@ module ContentData
       end
 
       is_valid = true
-      @contents_info.keys.each { |checksum|
+      contents_enum = @contents_info.each_key
+      loop {
+        checksum = contents_enum.next rescue break
         instances = @contents_info[checksum]
         content_size = instances[0]
         content_mtime = instances[2]
-        instances[1].keys.each { |unique_path|
+        instances_enum = instances[1].each_key
+        loop {
+          unique_path = instances_enum.next rescue break
           instance_mtime = instances[1][unique_path]
           instance_info = [checksum, content_mtime, content_size, instance_mtime]
           instance_info.concat(unique_path)

data/lib/content_data/version.rb

@@ -1,3 +1,3 @@
 module ContentData
-  VERSION = "1.1.0"
+  VERSION = "1.2.0"
 end

data/lib/content_server/backup_server.rb

@@ -4,7 +4,6 @@ require 'thread'
 
 require 'content_data'
 require 'content_server/content_receiver'
-require 'content_server/queue_indexer'
 require 'content_server/queue_copy'
 require 'content_server/remote_content'
 require 'content_server/server'
@@ -57,14 +56,14 @@ module ContentServer
     # initial global local content data object
     $local_content_data_lock = Mutex.new
     $local_content_data = ContentData::ContentData.new
+    $last_content_data_id = $local_content_data.unique_id
 
     # Read here for initial content data that exist from previous system run
     content_data_path = Params['local_content_data_path']
-    last_content_data_id = nil
     if File.exists?(content_data_path) and !File.directory?(content_data_path)
       Log.info("reading initial content data that exist from previous system run from file:%s", content_data_path)
       $local_content_data.from_file(content_data_path)
-      last_content_data_id = $local_content_data.unique_id
+      $last_content_data_id = $local_content_data.unique_id
     else
       if File.directory?(content_data_path)
         raise("Param:'local_content_data_path':'%s' cannot be a directory name", Params['local_content_data_path'])
@@ -75,21 +74,13 @@ module ContentServer
     end
 
     Log.info("Init monitoring")
-    monitoring_events = Queue.new
     fm = FileMonitoring::FileMonitoring.new()
-    fm.set_event_queue(monitoring_events)
+
     # Start monitoring and writing changes to queue
     all_threads << Thread.new do
       fm.monitor_files
     end
 
-    # # # # # # # # # # # # # #
-    # Initialize/Start local indexer
-    Log.debug1('Start indexer')
-    queue_indexer = QueueIndexer.new(monitoring_events)
-    # Start indexing on demand and write changes to queue
-    all_threads << queue_indexer.run
-
     # # # # # # # # # # # # # # # # # # # # # # # #
     # thread: Start dump local content data to file
     Log.debug1('Init thread: flush local content data to file')
@@ -97,21 +88,7 @@ module ContentServer
       FileUtils.mkdir_p(Params['tmp_path']) unless File.directory?(Params['tmp_path'])
       loop{
         sleep(Params['data_flush_delay'])
-        Log.info('Start flush local content data to file.')
-        $testing_memory_log.info('Start flush content data to file') if $testing_memory_active
-        written_to_file = false
-        $local_content_data_lock.synchronize{
-          local_content_data_unique_id = $local_content_data.unique_id
-          if (local_content_data_unique_id != last_content_data_id)
-            last_content_data_id = local_content_data_unique_id
-            $local_content_data.to_file($tmp_content_data_file)
-            written_to_file = true
-          else
-            Log.info('no need to flush. content data has not changed')
-          end
-        }
-        File.rename($tmp_content_data_file, Params['local_content_data_path']) if written_to_file
-        $testing_memory_log.info("End flush content data to file") if $testing_memory_active
+        ContentServer.flush_content_data
       }
     end
 

data/lib/content_server/content_server.rb

@@ -4,7 +4,6 @@ require 'thread'
 
 require 'content_data'
 require 'content_server/content_receiver'
-require 'content_server/queue_indexer'
 require 'content_server/queue_copy'
 require 'content_server/remote_content'
 require 'file_indexing'
@@ -46,14 +45,14 @@ module ContentServer
     # initial global local content data object
     $local_content_data_lock = Mutex.new
     $local_content_data = ContentData::ContentData.new
+    $last_content_data_id = $local_content_data.unique_id
 
     # Read here for initial content data that exist from previous system run
     content_data_path = Params['local_content_data_path']
-    last_content_data_id = nil
     if File.exists?(content_data_path) and !File.directory?(content_data_path)
       Log.info("reading initial content data that exist from previous system run from file:%s", content_data_path)
       $local_content_data.from_file(content_data_path)
-      last_content_data_id = $local_content_data.unique_id
+      $last_content_data_id = $local_content_data.unique_id
     else
       if File.directory?(content_data_path)
         raise("Param:'local_content_data_path':'%s'cannot be a directory name", Params['local_content_data_path'])
@@ -64,21 +63,12 @@ module ContentServer
     end
 
     Log.info('Init monitoring')
-    monitoring_events = Queue.new
     fm = FileMonitoring::FileMonitoring.new()
-    fm.set_event_queue(monitoring_events)
     # Start monitoring and writing changes to queue
     all_threads << Thread.new do
       fm.monitor_files
     end
 
-    # # # # # # # # # # # # # #
-    # Initialize/Start local indexer
-    Log.debug1('Start indexer')
-    queue_indexer = QueueIndexer.new(monitoring_events)
-    # Start indexing on demand and write changes to queue
-    all_threads << queue_indexer.run
-
     # # # # # # # # # # # # # # # # # # # # # # # #
     # thread: Start dump local content data to file
     Log.debug1('Init thread: flush local content data to file')
@@ -86,21 +76,7 @@ module ContentServer
       FileUtils.mkdir_p(Params['tmp_path']) unless File.directory?(Params['tmp_path'])
       loop{
         sleep(Params['data_flush_delay'])
-        Log.info('Start flush local content data to file.')
-        $testing_memory_log.info('Start flush content data to file') if $testing_memory_active
-        written_to_file = false
-        $local_content_data_lock.synchronize{
-          local_content_data_unique_id = $local_content_data.unique_id
-          if (local_content_data_unique_id != last_content_data_id)
-            last_content_data_id = local_content_data_unique_id
-            $local_content_data.to_file($tmp_content_data_file)
-            written_to_file = true
-          else
-            Log.info('no need to flush. content data has not changed')
-          end
-        }
-        File.rename($tmp_content_data_file, Params['local_content_data_path']) if written_to_file
-        $testing_memory_log.info("End flush content data to file") if $testing_memory_active
+        ContentServer.flush_content_data
       }
     end