content_server 1.5.0 → 1.6.0

data/lib/content_server/version.rb

@@ -1,3 +1,3 @@
   module ContentServer
-    VERSION = "1.5.0"
+    VERSION = "1.6.0"
   end

data/lib/file_monitoring/file_monitoring.rb

@@ -11,6 +11,93 @@ module FileMonitoring
   # Manages file monitoring of number of file system locations
   class FileMonitoring
 
+    def create_sub_paths(path)
+      sub_paths=[]
+      while path != '.' && path != '/'
+        sub_paths.push(path)
+        path = File.dirname(path)
+      end
+      sub_paths.reverse!
+    end
+
+    def load_instances(file_attr_to_checksum, dir_stat_array)
+      inst_count = 0
+      # If file is under monitoring path - Add to DirStat tree as stable with path,size,mod_time read from file
+      # If file is NOT under monitoring path - skip (not a valid usage)
+      $local_content_data.each_instance {
+          |checksum, size, _, mod_time, _, path, index_time|
+
+        if Params['manual_file_changes']
+          file_attr_key = [File.basename(path), size, mod_time]
+          ident_file_info = file_attr_to_checksum[file_attr_key]
+          unless ident_file_info
+            #  Add file checksum to map
+            file_attr_to_checksum[file_attr_key] = IdentFileInfo.new(checksum, index_time)
+          else
+            # File already in map. Need to mark as not unique
+            ident_file_info.unique = false  # file will be skipped if found at new location
+          end
+        end
+        # construct sub paths array from full file path:
+        # Example:
+        #   instance path = /dir1/dir2/file_name
+        # sub_paths holds array => ["/dir1","/dir1/dir2","/dir1/dir2/file_name"]
+        # sub paths would create DirStat objs or FileStat(FileStat create using last sub path).
+        sub_paths = create_sub_paths(path)
+
+        # Loop over monitor paths to start build tree under each
+        dir_stat_array.each { | dir_stat|
+          # check if monitor path is one of the sub paths and find it's sub path index
+          # if index is found then it the monitor path
+          # the next index indicates the next sub path to insert to the tree
+          # the index will be raised at each recursive call down the tree
+          sub_paths_index = sub_paths.index(dir_stat[0].path)
+          if sub_paths_index
+            # monitor path was found. Add to tree
+            # start the recursive call with next sub path index
+            ::FileMonitoring.stable_state = dir_stat[1]
+            inst_count += 1
+            dir_stat[0].load_instance(sub_paths, sub_paths_index+1, size, mod_time)
+            break
+          end
+        }
+      }
+      Log.info("loaded instances:#{inst_count}")
+    end
+
+    def load_symlinks(dir_stat_array)
+      # If symlink file is under monitoring path - Add to DirStat tree
+      # If file is NOT under monitoring path - skip (not a valid usage)
+      symlink_count = 0
+      $local_content_data.each_symlink {
+          |_, symlink_path, symlink_target|
+
+        # construct sub paths array from symlink path:
+        # Example:
+        #   symlink path = /dir1/dir2/file_name
+        # sub_paths holds array => ["/dir1","/dir1/dir2","/dir1/dir2/file_name"]
+        # sub paths should match the paths of DirStat objs in the tree to reach the symlink location in Tree.
+        sub_paths = create_sub_paths(symlink_path)
+
+        # Loop over monitor paths to start enter tree
+        dir_stat_array.each { | dir_stat|
+          # check if monitor path is one of the sub paths and find it's sub path index
+          # if index is found then it the monitor path
+          # the next index indicates the next sub path to insert to the tree
+          # the index will be raised at each recursive call down the tree
+          sub_paths_index = sub_paths.index(dir_stat[0].path)
+          if sub_paths_index
+            # monitor path was found. Add to tree
+            # start the recursive call with next sub path index
+            ::FileMonitoring.stable_state = dir_stat[1]
+            symlink_count += 1
+            dir_stat[0].load_symlink(sub_paths, sub_paths_index+1, symlink_path, symlink_target)
+            break
+          end
+        }
+      }
+      Log.info("loaded symlinks:#{symlink_count}")
+    end
     # The main method. Loops on all paths, each time span and monitors them.
     #
     # =Algorithm:
@@ -45,7 +132,7 @@ module FileMonitoring
       @log4r.outputters << file_outputter
       ::FileMonitoring::DirStat.set_log(@log4r)
       
-     conf_array = Params['monitoring_paths']
+      conf_array = Params['monitoring_paths']
 
       # create root dirs of monitoring
       dir_stat_array = []
@@ -54,58 +141,15 @@ module FileMonitoring
         dir_stat_array.push([dir_stat, elem['stable_state']])
       }
 
+      # This structure is used to optimize indexing when user specifies a directory was moved.
+      file_attr_to_checksum = {}
+
       #Look over loaded content data if not empty
-      # If file is under monitoring path - Add to DirStat tree as stable with path,size,mod_time read from file
-      # If file is NOT under monitoring path - skip (not a valid usage)
-      file_attr_to_checksum = {}  # This structure is used to optimize indexing when user specifies a directory was moved.
       unless $local_content_data.empty?
         Log.info("Start build data base from loaded file. This could take several minutes")
-        inst_count = 0
-        $local_content_data.each_instance {
-            |checksum, size, _, mod_time, _, path, index_time|
-
-          if Params['manual_file_changes']
-            file_attr_key = [File.basename(path), size, mod_time]
-            ident_file_info = file_attr_to_checksum[file_attr_key]
-            unless ident_file_info
-              #  Add file checksum to map
-              file_attr_to_checksum[file_attr_key] = IdentFileInfo.new(checksum, index_time)
-            else
-              # File already in map. Need to mark as not unique
-              ident_file_info.unique = false  # file will be skipped if found at new location
-            end
-          end
-          # construct sub paths array from full file path:
-          # Example:
-          #   instance path = /dir1/dir2/file_name
-          #   Sub path 1: /dir1
-          #   Sub path 2: /dir1/dir2
-          #   Sub path 3: /dir1/dir2/file_name
-          # sub paths would create DirStat objs or FileStat(FileStat create using last sub path).
-          split_path = path.split(File::SEPARATOR)
-          sub_paths = (0..split_path.size-1).inject([]) { |paths, i|
-            paths.push(File.join(*split_path.values_at(0..i)))
-          }
-          # sub_paths holds array => ["/dir1","/dir1/dir2","/dir1/dir2/file_name"]
-
-          # Loop over monitor paths to start build tree under each
-          dir_stat_array.each { | dir_stat|
-            # check if monitor path is one of the sub paths and find it's sub path index
-            # if index is found then it the monitor path
-            # the next index indicates the next sub path to insert to the tree
-            # the index will be raised at each recursive call down the tree
-            sub_paths_index = sub_paths.index(dir_stat[0].path)
-            next if sub_paths_index.nil?  # monitor path was not found. skip this instance.
-
-            # monitor path was found. Add to tree
-            # start the recursive call with next sub path index
-            ::FileMonitoring.stable_state = dir_stat[1]
-            inst_count += 1
-            dir_stat[0].load_instance(sub_paths, sub_paths_index+1, size, mod_time)
-            break
-          }
-        }
-        Log.info("End build data base from loaded file. loaded instances:#{inst_count}")
+        load_instances(file_attr_to_checksum, dir_stat_array)
+        load_symlinks(dir_stat_array)
+        Log.info("End build data base from loaded file")
         $last_content_data_id = $local_content_data.unique_id
 
         if Params['manual_file_changes']

data/lib/file_monitoring/monitor_path.rb

@@ -137,6 +137,13 @@ module FileMonitoring
       @@log = log
     end
 
+    def write_to_log(msg)
+      if @@log
+        @@log.info(msg)
+        @@log.outputters[0].flush if Params['log_flush_each_message']
+      end
+    end
+
     public_class_method :set_log
 
     #  Initializes new directory monitoring object
@@ -148,7 +155,7 @@ module FileMonitoring
       @dirs = {}
       @files = {}
       @non_utf8_paths = {}  # Hash: ["path" -> true|false]
-
+      @symlinks = {}  # Hash: [[server, link file name]] -> " target file name"]
       # indicates if path EXISTS in file system.
       #   If true, file will not be removed during removed_unmarked_paths phase.
       @marked = false
@@ -169,7 +176,7 @@ module FileMonitoring
     #   size - the instance size to insert to the tree
     #   modification_time - the instance modification_time to insert to the tree
     def load_instance(sub_paths, sub_paths_index, size, modification_time)
-      # initialize dirs and files. This will indicate that the current DirStat is not new.
+      # initialize dirs and files.
       @dirs = {} unless @dirs
       @files = {} unless @files
       if sub_paths.size-1 == sub_paths_index
@@ -189,6 +196,40 @@ module FileMonitoring
       end
     end
 
+    # add symlink while initializing tree using content data from file.
+    #   Assumption is that Tree already built
+    # Parameters:
+    #   sub_paths - Array of sub paths of the symlink which is added to tree
+    #               Example:
+    #                 instance path = /dir1/dir2/file_name
+    #                   Sub path 1: /dir1
+    #                   Sub path 2: /dir1/dir2
+    #                   Sub path 3: /dir1/dir2/file_name
+    #               sub paths would create DirStat objs or FileStat(FileStat create using last sub path).
+    #   sub_paths_index - the index indicates the next sub path to insert to the tree
+    #                     the index will be raised at each recursive call down the tree
+    #   symlink_path - symlink file path
+    #   symlink_target - the target path pointed by the symlink
+    def load_symlink(sub_paths, sub_paths_index, symlink_path, symlink_target)
+      # initialize dirs and files.
+      @dirs = {} unless @dirs
+      @files = {} unless @files
+      if sub_paths.size-1 == sub_paths_index
+        # index points to last entry - leaf case. Add the symlink.
+        @symlinks[symlink_path] = symlink_target
+      else
+        # Add Dir to tree if not present. index points to new dir path.
+        dir_stat = @dirs[sub_paths[sub_paths_index]]
+        #create new dir if not exist
+        unless dir_stat
+          dir_stat = DirStat.new(sub_paths[sub_paths_index])
+          add_dir(dir_stat)
+        end
+        # continue recursive call on tree with next sub path index
+        dir_stat.load_instance(sub_paths, sub_paths_index+1, symlink_path, symlink_target)
+      end
+    end
+
     #  Adds directory for monitoring.
     def add_dir (dir)
       @dirs[dir.path] = dir
@@ -223,7 +264,7 @@ module FileMonitoring
     def to_s(indent = 0)
       indent_increment = 2
       child_indent = indent + indent_increment
-      res = super
+      res = super()
       @files.each_value do |file|
         res += "\n" + file.to_s(child_indent)
       end if @files
@@ -245,10 +286,7 @@ module FileMonitoring
           dir_stat.removed_unmarked_paths
         else
           # directory is not marked. Remove it, since it does not exist.
-          if @@log
-            @@log.info("NON_EXISTING dir: " + dir_stat.path)
-            @@log.outputters[0].flush if Params['log_flush_each_message']
-          end
+          write_to_log("NON_EXISTING dir: " + dir_stat.path)
           # remove file with changed checksum
           $local_content_data_lock.synchronize{
             $local_content_data.remove_directory(dir_stat.path, Params['local_server_name'])
@@ -265,10 +303,7 @@ module FileMonitoring
           file_stat.marked = false  # unset flag for next monitoring\index\remove phase
         else
           # file not marked meaning it is no longer exist. Remove.
-          if @@log
-            @@log.info("NON_EXISTING file: " + file_stat.path)
-            @@log.outputters[0].flush if Params['log_flush_each_message']
-          end
+          write_to_log("NON_EXISTING file: " + file_stat.path)
           # remove file with changed checksum
           $local_content_data_lock.synchronize{
             $local_content_data.remove_instance(Params['local_server_name'], file_stat.path)
@@ -279,14 +314,139 @@ module FileMonitoring
       end
     end
 
-    # Recursively, read files and dirs from file system (using Glob)
-    # Handle new files\dirs.
-    # Change state for existing files\dirs
-    # Index stable files
-    # Remove non existing files\dirs is handled in method: remove_unmarked_paths
+    ################ All monitoring helper methods #################
+    
+    # Checks that the globed path is valid.
+    def is_globed_path_valid(globed_path)
+      # UTF-8 - keep only files with names in
+      return true if @non_utf8_paths[globed_path]
+      check_utf_8_encoding_file = globed_path.clone
+      unless check_utf_8_encoding_file.force_encoding("UTF-8").valid_encoding?
+        Log.warning("Non UTF-8 file name '#{check_utf_8_encoding_file}', skipping.")
+        @non_utf8_paths[globed_path] = true
+        # TODO(bbfsdev): Remove line below and redundant clones of string after
+        # those lines are not a GC problem.
+        check_utf_8_encoding_file = nil
+        return false
+      end
+
+      true
+    end
+
+    def handle_existing_file(child_stat, globed_path, globed_path_stat)
+      if child_stat.changed?(globed_path_stat)
+        # ---------- STATUS CHANGED
+        # Update changed status
+        child_stat.state = FileStatEnum::CHANGED
+        child_stat.cycles = 0
+        child_stat.size = globed_path_stat.size
+        child_stat.modification_time = globed_path_stat.mtime.to_i
+        write_to_log("CHANGED file: " + globed_path)
+        # remove file with changed checksum. File will be added once indexed
+        $local_content_data_lock.synchronize{
+          $local_content_data.remove_instance(Params['local_server_name'], globed_path)
+        }
+      else  # case child_stat did not change
+        # ---------- SAME STATUS
+        # File status is the same
+        if child_stat.state != FileStatEnum::STABLE
+          child_stat.cycles += 1
+          if child_stat.cycles >= ::FileMonitoring.stable_state
+            child_stat.state = FileStatEnum::STABLE
+            write_to_log("STABLE file: " + globed_path)
+          else
+            child_stat.state = FileStatEnum::UNCHANGED
+            write_to_log("UNCHANGED file: " + globed_path)
+          end
+        end
+      end
+    end
+
+    # This method handles the case where we set the 'manual_file_changes' param meaning
+    # some files were moved/copied and no need to reindex them. In that case search "new files"
+    # in old files to get the checksum (skipp the index phase).
+    # The lookup is done via specially prepared file_attr_to_checksum map.
+    def handle_moved_file(globed_path, globed_path_stat, file_attr_to_checksum)
+      # --------------------- MANUAL MODE
+      # check if file name and attributes exist in global file attr map
+      file_attr_key = [File.basename(globed_path), globed_path_stat.size, globed_path_stat.mtime.to_i]
+      file_ident_info = file_attr_to_checksum[file_attr_key]
+      # If not found (real new file) or found but not unique then file needs indexing. skip in manual mode.
+      if file_ident_info && file_ident_info.unique
+        Log.debug1("update content data with file:%s  checksum:%s  index_time:%s",
+                   File.basename(globed_path), file_ident_info.checksum, file_ident_info.index_time.to_s)
+        # update content data (no need to update Dir tree)
+        $local_content_data_lock.synchronize{
+          $local_content_data.add_instance(file_ident_info.checksum,
+                                           globed_path_stat.size,
+                                           Params['local_server_name'],
+                                           globed_path,
+                                           globed_path_stat.mtime.to_i,
+                                           file_ident_info.index_time)
+        }
+      end
+    end
+
+    def handle_new_file(child_stat, globed_path, globed_path_stat)
+      child_stat = FileStat.new(globed_path,
+                                FileStatEnum::NEW,
+                                globed_path_stat.size,
+                                globed_path_stat.mtime.to_i)
+      write_to_log("NEW file: " + globed_path)
+      child_stat.marked = true
+      add_file(child_stat)
+    end
+
+    def handle_dir(globed_path, file_attr_to_checksum)
+      # ------------------------------ DIR -----------------------
+      child_stat = @dirs[globed_path]
+      unless child_stat
+        # ----------- ADD NEW DIR
+        child_stat = DirStat.new(globed_path)
+        add_dir(child_stat)
+        write_to_log("NEW dir: " + globed_path)
+      end
+      child_stat.marked = true
+      # recursive call for dirs
+      child_stat.monitor(file_attr_to_checksum)
+    end
+
+    def add_found_symlinks(globed_path, found_symlinks)
+      # if symlink - add to symlink temporary map and content data (even override).
+      # later all non existing symlinks will be removed from content data
+      pointed_file_name = File.readlink(globed_path)
+      found_symlinks[globed_path] = pointed_file_name
+      # add to content data
+      $local_content_data_lock.synchronize{
+        $local_content_data.add_symlink(Params['local_server_name'], globed_path, pointed_file_name)
+      }
+    end
+
+    def remove_not_found_symlinks(found_symlinks)
+      # check if any symlink was removed and update current symlinks map
+      symlinks_enum = @symlinks.each_key
+      loop {
+        symlink_key = symlinks_enum.next rescue break
+        unless found_symlinks.has_key?(symlink_key)
+          # symlink was removed. remove from content data
+          $local_content_data_lock.synchronize{
+            $local_content_data.remove_symlink(Params['local_server_name'], symlink_key)
+          }
+        end
+      }
+      @symlinks = found_symlinks
+    end
+
+    # Recursively, read files and dirs lists from file system (using Glob)
+    # - Adds new files\dirs.
+    # - Change state for existing files\dirs
+    # - Index stable files
+    # - Remove non existing files\dirs is handled in method: remove_unmarked_paths
+    # - Handles special case for param 'manual_file_changes' where files are moved and
+    #   there is no need to index them
     def monitor(file_attr_to_checksum=nil)
 
-      # Algorithm:
+      # Marking/Removing Algorithm:
       # assume that current dir is present
       # ls (glob) the dir path for child dirs and files
       # if child file is not already present, add it as new, mark it and handle its state
@@ -298,19 +458,14 @@ module FileMonitoring
 
       # ls (glob) the dir path for child dirs and files
       globed_paths_enum = Dir.glob(@path + "/*").to_enum
+      
+      found_symlinks = {}  # Store found symlinks under dir
       loop do
         globed_path = globed_paths_enum.next rescue break
 
-        # if symlink - skip
-        next if File.symlink?(globed_path)
-
-        # UTF-8 - keep only files with names in
-        next if @non_utf8_paths[globed_path]
-        check_utf_8_encoding_file = globed_path.clone
-        unless check_utf_8_encoding_file.force_encoding("UTF-8").valid_encoding?
-          Log.warning("Non UTF-8 file name '#{check_utf_8_encoding_file}', skipping.")
-          @non_utf8_paths[globed_path]=true
-          check_utf_8_encoding_file=nil
+        next unless is_globed_path_valid(globed_path)
+        if File.symlink?(globed_path)
+          add_found_symlinks(globed_path, found_symlinks)
           next
         end
 
@@ -320,99 +475,27 @@ module FileMonitoring
           # ----------------------------- FILE -----------------------
           child_stat = @files[globed_path]
           if child_stat
-            # -------------- EXISTS in Tree
-            unless Params['manual_file_changes']
-              # --------- NON MANUAL MODE
-              child_stat.marked = true
-              if child_stat.changed?(globed_path_stat)
-                # ---------- STATUS CHANGED
-                # Update changed status
-                child_stat.state = FileStatEnum::CHANGED
-                child_stat.cycles = 0
-                child_stat.size = globed_path_stat.size
-                child_stat.modification_time = globed_path_stat.mtime.to_i
-                if @@log
-                  @@log.info("CHANGED file: " + globed_path)
-                  @@log.outputters[0].flush if Params['log_flush_each_message']
-                end
-                # remove file with changed checksum. File will be added once indexed
-                $local_content_data_lock.synchronize{
-                  $local_content_data.remove_instance(Params['local_server_name'], globed_path)
-                }
-              else  # case child_stat did not change
-                # ---------- SAME STATUS
-                # File status is the same
-                if child_stat.state != FileStatEnum::STABLE
-                  child_stat.state = FileStatEnum::UNCHANGED
-                  child_stat.cycles += 1
-                  if child_stat.cycles >= ::FileMonitoring.stable_state
-                    child_stat.state = FileStatEnum::STABLE
-                    if @@log
-                      @@log.info("STABLE file: " + globed_path)
-                      @@log.outputters[0].flush if Params['log_flush_each_message']
-                    end
-                  else
-                    if @@log
-                      @@log.info("UNCHANGED file: " + globed_path)
-                      @@log.outputters[0].flush if Params['log_flush_each_message']
-                    end
-                  end
-                end
-              end
-            else  # case Params['manual_file_changes']
-              # --------- MANUAL MODE
-              child_stat.marked = true
-            end
+            # Mark that file exists (will not be deleted at end of monitoring)
+            child_stat.marked = true
+            # Handle existing file If we are not in manual mode.
+            # In manual mode do nothing
+            handle_existing_file(child_stat, globed_path, globed_path_stat) unless Params['manual_file_changes']
           else
-            # ---------------------------- NEW FILE ----------
             unless Params['manual_file_changes']
-              child_stat = FileStat.new(globed_path,
-                                        FileStatEnum::NEW,
-                                        globed_path_stat.size,
-                                        globed_path_stat.mtime.to_i)
-              if @@log
-                @@log.info("NEW file: " + globed_path)
-                @@log.outputters[0].flush if Params['log_flush_each_message']
-              end
-              child_stat.marked = true
-              add_file(child_stat)
-            else  # case Params['manual_file_changes']
-              # --------------------- MANUAL MODE
-              # check if file name and attributes exist in global file attr map
-              file_attr_key = [File.basename(globed_path), globed_path_stat.size, globed_path_stat.mtime.to_i]
-              file_ident_info = file_attr_to_checksum[file_attr_key]
-              # If not found (real new file) or found but not unique then file needs indexing. skip in manual mode.
-              next unless (file_ident_info and file_ident_info.unique)
-              Log.debug1("update content data with file:%s  checksum:%s  index_time:%s",
-                         File.basename(globed_path), file_ident_info.checksum, file_ident_info.index_time.to_s)
-              # update content data (no need to update Dir tree)
-              $local_content_data_lock.synchronize{
-                $local_content_data.add_instance(file_ident_info.checksum,
-                                                 globed_path_stat.size,
-                                                 Params['local_server_name'],
-                                                 globed_path,
-                                                 globed_path_stat.mtime.to_i,
-                                                 file_ident_info.index_time)
-              }
+              # Handle regular case of new file.
+              handle_new_file(child_stat, globed_path, globed_path_stat)
+            else
+              # Only create new content data instance based on copied/moved filed.
+              handle_moved_file(globed_path, globed_path_stat, file_attr_to_checksum)
             end
           end
         else
-          # ------------------------------ DIR -----------------------
-          child_stat = @dirs[globed_path]
-          unless child_stat
-            # ----------- ADD NEW DIR
-            child_stat = DirStat.new(globed_path)
-            add_dir(child_stat)
-            if @@log
-              @@log.info("NEW dir: " + globed_path)
-              @@log.outputters[0].flush if Params['log_flush_each_message']
-            end
-          end
-          child_stat.marked = true
-          # recursive call for dirs
-          child_stat.monitor(file_attr_to_checksum)
+          handle_dir(globed_path, file_attr_to_checksum)
         end
       end
+
+      remove_not_found_symlinks(found_symlinks)
+
       GC.start
     end