content_server 1.2.1 → 1.3.0

data/lib/content_server/queue_indexer.rb

@@ -21,47 +21,35 @@ module ContentServer
       # Start indexing on demand and write changes to queue
       thread = Thread.new do
         while true do
-          Log.debug1 'Waiting on index input queue.'
           (state, is_dir, path, mtime, size) = @input_queue.pop
           $process_vars.set('monitor to index queue size', @input_queue.size)
-          Log.debug1 "index event: state:#{state}, dir?#{is_dir}, path:#{path}, mtime:#{mtime}, size:#{size}."
+          Log.debug1("index event: state:%s  dir?%s  path:%s  time:%s  size:%s ",
+                     state, is_dir, path, mtime, size)
           if state == FileMonitoring::FileStatEnum::STABLE && !is_dir
             # Calculating checksum
-            instance_stats = nil  # definition
+            checksum = calc_SHA1(path)
+            $process_vars.inc('indexed_files')
+            $indexed_file_count += 1
+            Log.debug1("Index checksum:%s Adding index to content data. put in queue for dynamic update.",
+                       checksum)
             $local_content_data_lock.synchronize{
-              instance_stats = $local_content_data.stats_by_location([Params['local_server_name'], path])
+              $local_content_data.add_instance(checksum, size, Params['local_server_name'], path, mtime)
             }
-            Log.debug1("instance !#{instance_stats}! mtime: #{mtime.to_i}, size: #{size}")
-            if instance_stats.nil? || mtime.to_i != instance_stats[1] || size != instance_stats[0]
-              Log.debug1 "Indexing file:'#{path}'."
-              checksum = calc_SHA1(path)
-              $process_vars.inc('indexed_files')
-              $indexed_file_count += 1
-              Log.debug1("Index info:checksum:#{checksum} size:#{size} time:#{mtime.to_i}")
-              Log.debug1('Adding index to content data. put in queue for dynamic update.')
-              $local_content_data_lock.synchronize{
-                $local_content_data.add_instance(checksum, size, Params['local_server_name'], path, mtime.to_i)
-              }
-            else
-              Log.info("Skip file #{path} indexing (shallow check passed)")
-            end
           elsif ((state == FileMonitoring::FileStatEnum::NON_EXISTING ||
               state == FileMonitoring::FileStatEnum::CHANGED) && !is_dir)
-            Log.debug2("NonExisting/Changed (file): #{path}")
             # Remove file but only when non-existing.
-            Log.debug1("File to remove: #{path}")
+            Log.debug1("File to remove: %s", path)
             $local_content_data_lock.synchronize{
               $local_content_data.remove_instance(Params['local_server_name'], path)
             }
           elsif state == FileMonitoring::FileStatEnum::NON_EXISTING && is_dir
-            Log.debug2("NonExisting/Changed (dir): #{path}")
             # Remove directory but only when non-existing.
-            Log.debug1("Directory to remove: #{path}")
+            Log.debug1("Directory to remove: %s", path)
             $local_content_data_lock.synchronize{
               $local_content_data.remove_directory(Params['local_server_name'], path)
             }
           else
-            Log.debug1("This case should not be handled: #{state}, #{is_dir}, #{path}.")
+            Log.debug1("This case should not be handled: %s, %s, %s", state, is_dir, path)
           end
         end  # while true do
       end  # Thread.new do

data/lib/content_server/remote_content.rb

@@ -27,7 +27,8 @@ module ContentServer
       @last_content_data_id = nil
       @content_server_content_data_path = File.join(local_backup_folder, 'remote',
                                                     host + '_' + port.to_s)
-      Log.debug3("Initialized RemoteContentClient: host:#{host}   port:#{port}  local_backup_folder:#{local_backup_folder}")
+      Log.debug3("Initialized RemoteContentClient: host:%s   port:%s  local_backup_folder:%s",
+                 host, port, local_backup_folder)
     end
 
     # Receives content data,  and writes it to file
@@ -47,7 +48,7 @@ module ContentServer
           $remote_content_data.to_file(path)
           @last_content_data_id = message.unique_id   # save last content data ID
         }
-        Log.debug1("Written content data to file:#{path}.")
+        Log.debug1("Written content data to file:%s.", path)
       else
         Log.debug1("No need to write remote content data, it has not changed.")
       end
@@ -59,14 +60,13 @@ module ContentServer
       threads = []
       threads << @remote_tcp.tcp_thread if @remote_tcp != nil
       threads << Thread.new do
-        Log.debug1("New thread.")
         loop do
           # if need content data
           sleep_time_span = Params['remote_content_save_timeout']
           if @last_fetch_timestamp
             sleep_time_span = Time.now.to_i - @last_fetch_timestamp
           end
-          Log.debug1("sleep_time_span: #{sleep_time_span}")
+          Log.debug1("sleep_time_span: %s", sleep_time_span)
           if sleep_time_span >= Params['remote_content_save_timeout']
             # Send ping!
             @remote_tcp.send_obj(nil)
@@ -85,7 +85,7 @@ module ContentServer
     # initializes class variables
     def initialize(port)
       @tcp_server = Networking::TCPServer.new(port, method(:content_requested))
-      Log.debug3("initialize RemoteContentServer on port:#{port}")
+      Log.debug3("initialize RemoteContentServer on port:%s", port)
     end
 
     # content_requested
@@ -95,7 +95,7 @@ module ContentServer
       # Send response.
       Log.info("Content server received content data request")
       $local_content_data_lock.synchronize{
-        Log.debug1("Sending content data:#{$local_content_data}")
+        Log.debug1("Sending content data:%s", $local_content_data)
         @tcp_server.send_obj($local_content_data)
       }
       Log.info('Content server sent content data')

data/lib/content_server/server.rb

@@ -42,12 +42,13 @@ module ContentServer
       "Backtrace:\n#{exception.backtrace.join("\n")}"
     puts(message)
 
+    # Block force write to file resolving Issue 217 https://github.com/bbfsdev/bbfs/issues/217
     # force write content data to file
-    if $local_content_data
-      puts("\nForce writing local content data to #{Params['local_content_data_path']}.")
-      $local_content_data.to_file($tmp_content_data_file)
-      File.rename($tmp_content_data_file, Params['local_content_data_path'])
-    end
+    #if $local_content_data
+     # puts("\nForce writing local content data to #{Params['local_content_data_path']}.")
+      #$local_content_data.to_file($tmp_content_data_file)
+      #File.rename($tmp_content_data_file, Params['local_content_data_path'])
+    #end
 
     #Write exception message to log and mail(if enabled)
     Log.error(message)
@@ -83,9 +84,9 @@ module ContentServer
         report += "Type:#{type} raised in:#{diff}   \n"
         objects_counters[type] = current_objects_counters[type]
       }
-      Log.info("MEM REPORT at:#{time}:\n#{report}\n")
+      Log.info("MEM REPORT:\n%s\n", report)
     end
   end
 
   module_function :init_globals, :handle_program_termination, :monitor_general_process_vars
-end
+end

data/lib/content_server/version.rb

@@ -1,3 +1,3 @@
   module ContentServer
-    VERSION = "1.2.1"
+    VERSION = "1.3.0"
   end

data/lib/file_copy/copy.rb

@@ -13,7 +13,8 @@ module FileCopy
     password = (password and password.length > 0) ? password : nil
     port = 22 # 22 is a standart SSH port
     raise "Undefined server" unless server
-    Log.debug1 "Trying to connect(ssh): #{username}, #{password}, #{server}, #{port}."
+    Log.debug1("Trying to connect(ssh): %s, %s, %s, %s.",
+               username, password, server, port)
     if (username and password)
       Net::SSH.start(server, username,
                      :password => password,
@@ -22,7 +23,7 @@ module FileCopy
       Net::SSH.start(server, username,
                      :port => port)
     else
-      raise "Undefined username"
+      raise("Undefined username")
     end
   end
   module_function :ssh_connect
@@ -36,11 +37,11 @@ module FileCopy
       uploads = files_map.map { |from,to|
         remote_dir = File.dirname(to)
         sftp_mkdir_recursive(sftp, File.dirname(to))
-        Log.debug1 "Copying #{from} to #{to}"
+        Log.debug1("Copying %s to %s",from, to)
         sftp.upload(from, to)
       }
       uploads.each { |u| u.wait }
-      Log.debug1 "Done."
+      Log.debug1("Done.")
     end # ssh.sftp.connect
   end # def initialize
   module_function :sftp_copy
@@ -48,17 +49,17 @@ module FileCopy
   def sftp_mkdir_recursive(sftp, path)
     dir_stat = nil
     begin
-      Log.debug1 "Stat remote dir: #{path}."
+      Log.debug1("Stat remote dir: %s.", path)
       dir_stat = sftp.stat!(path).directory?
-      Log.debug1 "Stat result #{dir_stat}."
+      Log.debug1("Stat result %s.", dir_stat)
     rescue Net::SFTP::StatusException
     end
     if !dir_stat
-      Log.debug1 "Directory does not exists: #{path}."
+      Log.debug1("Directory does not exists: %s.", path)
       sftp_mkdir_recursive sftp, File.dirname(path)
-      Log.debug1 "Making dir #{path}."
+      Log.debug1("Making dir %s.", path)
       response = sftp.mkdir!(path)
-      Log.debug1 "Making dir ok:#{response.ok?}."
+      Log.debug1("Making dir ok:%s.", response.ok?)
     end
   end
   module_function :sftp_mkdir_recursive

data/lib/file_monitoring/file_monitoring.rb

@@ -9,19 +9,6 @@ module FileMonitoring
   # Manages file monitoring of number of file system locations
   class FileMonitoring
 
-    def initialize ()
-      @content_data_cache = Set.new
-      $local_content_data_lock.synchronize {
-        $local_content_data.each_instance(){
-            |_, _, _, _, _, file_path|
-          # save files to cache
-          Log.info("File in cache: #{file_path.clone}")
-          @content_data_cache.add(file_path.clone)
-        }
-      }
-    end
-
-
     # Set event queue used for communication between different proceses.
     # @param queue [Queue]
     def set_event_queue(queue)
@@ -41,16 +28,60 @@ module FileMonitoring
     # that provides path and file monitoring configuration data
     def monitor_files
       conf_array = Params['monitoring_paths']
+
+      # create root dirs of monitoring
+      dir_stat_array = []
+      conf_array.each { |elem|
+        dir_stat = DirStat.new(File.expand_path(elem['path']), elem['stable_state'])
+        dir_stat.set_event_queue(@event_queue) if @event_queue
+        dir_stat_array.push(dir_stat)
+      }
+
+      #Look over loaded content data if not empty
+      # If file is under monitoring path - Add to DirStat tree as stable with path,size,mod_time read from file
+      # If file is NOT under monitoring path - skip (not a valid usage)
+      unless $local_content_data.empty?
+        Log.info("Start build data base from loaded file")
+        $local_content_data.each_instance {
+            |_, size, _, mod_time, _, path|
+          # construct sub paths array from full file path:
+          # Example:
+          #   instance path = /dir1/dir2/file_name
+          #   Sub path 1: /dir1
+          #   Sub path 2: /dir1/dir2
+          #   Sub path 3: /dir1/dir2/file_name
+          # sub paths would create DirStat objs or FileStat(FileStat create using last sub path).
+          split_path = path.split(File::SEPARATOR)
+          sub_paths = (0..split_path.size-1).inject([]) { |paths, i|
+            paths.push(File.join(*split_path.values_at(0..i)))
+          }
+          # sub_paths holds array => ["/dir1","/dir1/dir2","/dir1/dir2/file_name"]
+
+          # Loop over monitor paths to start build tree under each
+          dir_stat_array.each { | dir_stat|
+            # check if monitor path is one of the sub paths and find it's sub path index
+            # if index is found then it the monitor path
+            # the next index indicates the next sub path to insert to the tree
+            # the index will be raised at each recursive call down the tree
+            sub_paths_index = sub_paths.index(dir_stat.path)
+            next if sub_paths_index.nil?  # monitor path was not found. skip this instance.
+            # monitor path was found. Add to tree as stable.
+            dir_stat.state = FileStatEnum::STABLE
+            # start the recursive call with next sub path index
+            dir_stat.load_instance(sub_paths, sub_paths_index+1, size, mod_time)
+          }
+        }
+        Log.info("End build data base from loaded file")
+      end
+
       # Directories states stored in the priority queue,
       # where the key (priority) is a time when it should be checked next time.
       # Priority queue means that all entries arranged by key (time to check) in increasing order.
       pq = Containers::PriorityQueue.new
-      conf_array.each { |elem|
+      conf_array.each_with_index { |elem, index|
         priority = (Time.now + elem['scan_period']).to_i
-        dir_stat = DirStat.new(File.expand_path(elem['path']), elem['stable_state'], @content_data_cache, FileStatEnum::NON_EXISTING)
-        dir_stat.set_event_queue(@event_queue) if @event_queue
-        Log.debug1("File monitoring started for: #{elem}")
-        pq.push([priority, elem, dir_stat], -priority)
+        #Log.info("File monitoring started for: #{elem}")
+        pq.push([priority, elem, dir_stat_array[index]], -priority)
       }
 
       #init log4r
@@ -85,7 +116,9 @@ module FileMonitoring
         end
 
         unless $testing_memory_active
+          Log.info("Start monitor for dir:%s", dir_stat.path)
           dir_stat.monitor
+          Log.info("End monitor for dir:%s", dir_stat.path)
         else
           $testing_memory_log.info("Start monitor at :#{Time.now}")
           puts "Start monitor at :#{Time.now}"

data/lib/file_monitoring/monitor_path.rb

@@ -23,7 +23,8 @@ module FileMonitoring
 
   #  This class holds current state of file and methods to control and report changes
   class FileStat
-    attr_reader :cycles, :path, :stable_state, :state, :size, :modification_time
+    attr_reader :cycles, :path, :stable_state
+    attr_accessor :state, :size, :modification_time
 
     DEFAULT_STABLE_STATE = 10
 
@@ -33,15 +34,16 @@ module FileMonitoring
     # ==== Arguments:
     #
     # * <tt>path</tt> - File location
+    # * <tt>size</tt> - File size [Byte]
+    # * <tt>modification_time</tt> - file mod time [seconds]
+    # * <tt>cycles</tt> - # number of iterations from the last state change.
     # * <tt>stable_state</tt> - Number of iterations to move unchanged file to stable state
-    def initialize(path, stable_state = DEFAULT_STABLE_STATE, content_data_cache, state)
+    def initialize(path, stable_state = DEFAULT_STABLE_STATE)
       @path ||= path
       @size = nil
-      @creation_time = nil
       @modification_time = nil
-      @cycles = 0  # number of iterations from the last file modification
-      @state = state
-      @stable_state = stable_state  # number of iteration to move unchanged file to stable state
+      @cycles = 0
+      @stable_state = stable_state
     end
 
     def set_output_queue(event_queue)
@@ -61,23 +63,23 @@ module FileMonitoring
     def monitor
       file_stats = File.lstat(@path) rescue nil
       new_state = nil
-      if file_stats == nil
+      if file_stats.nil?
         new_state = FileStatEnum::NON_EXISTING
         @size = nil
-        @creation_time = nil
+        #@creation_time = nil
         @modification_time = nil
         @cycles = 0
-      elsif @size == nil
+      elsif @size.nil?
         new_state = FileStatEnum::NEW
         @size = file_stats.size
-        @creation_time = file_stats.ctime.utc
-        @modification_time = file_stats.mtime.utc
+        #@creation_time = file_stats.ctime.utc
+        @modification_time = file_stats.mtime.to_i
         @cycles = 0
       elsif changed?(file_stats)
         new_state = FileStatEnum::CHANGED
         @size = file_stats.size
-        @creation_time = file_stats.ctime.utc
-        @modification_time = file_stats.mtime.utc
+        #@creation_time = file_stats.ctime.utc
+        @modification_time = file_stats.mtime.to_i
         @cycles = 0
       else
         new_state = FileStatEnum::UNCHANGED
@@ -88,14 +90,13 @@ module FileMonitoring
       end
 
       # The assignment
-      self.state= new_state
+      set_state(new_state)
     end
 
     #  Checks that stored file attributes are the same as file attributes taken from file system.
     def changed?(file_stats)
-      not (file_stats.size == @size &&
-          file_stats.ctime.utc == @creation_time.utc &&
-          file_stats.mtime.utc == @modification_time.utc)
+      !((file_stats.size == @size) &&
+        (file_stats.mtime.to_i == @modification_time))
     end
 
     def set_event_queue(queue)
@@ -103,7 +104,7 @@ module FileMonitoring
     end
 
     #  Sets and writes to the log a new state.
-    def state= (new_state)
+    def set_state(new_state)
       if (@state != new_state or @state == FileStatEnum::CHANGED)
         @state = new_state
         if (@@log)
@@ -111,9 +112,8 @@ module FileMonitoring
           @@log.outputters[0].flush if Params['log_flush_each_message']
         end
         if @event_queue and FileStatEnum::NEW != @state  # NEW state is ignored in indexer
-          Log.debug1 "Writing to event queue [#{self.state}, #{self.path}]"
-          @event_queue.push([self.state, self.instance_of?(DirStat), self.path,
-                             self.modification_time, self.size])
+          Log.debug1("Writing to event queue [%s, %s]", @state, @path)
+          @event_queue.push([@state, self.instance_of?(DirStat), @path, @modification_time, @size])
           $process_vars.set('monitor to index queue size', @event_queue.size)
         end
       end
@@ -137,12 +137,51 @@ module FileMonitoring
     #
     # * <tt>path</tt> - File location
     # * <tt>stable_state</tt> - Number of iterations to move unchanged directory to stable state
-    def initialize(path, stable_state = DEFAULT_STABLE_STATE, content_data_cache, state)
+    def initialize(path, stable_state = DEFAULT_STABLE_STATE)
       super
-      @dirs = nil
-      @files = nil
-      @non_utf8_paths = {}
-      @content_data_cache = content_data_cache
+      @dirs = nil  # Hash: ["path" -> DirStat]
+      @files = nil # Hash: ["path" -> FileStat]
+      @non_utf8_paths = {}  # Hash: ["path" -> true|false]
+    end
+
+    # add instance while initializing tree using content data from file
+    # Parameters:
+    #   sub_paths - Array of sub paths of the instance which is added to tree
+    #               Example:
+    #                 instance path = /dir1/dir2/file_name
+    #                   Sub path 1: /dir1
+    #                   Sub path 2: /dir1/dir2
+    #                   Sub path 3: /dir1/dir2/file_name
+    #               sub paths would create DirStat objs or FileStat(FileStat create using last sub path).
+    #   sub_paths_index - the index indicates the next sub path to insert to the tree
+    #                     the index will be raised at each recursive call down the tree
+    #   size - the instance size to insert to the tree
+    #   modification_time - the instance modification_time to insert to the tree
+    def load_instance(sub_paths, sub_paths_index, size, modification_time)
+      # initialize dirs and files. This will indicate that the current DirStat is not new.
+      @dirs = {} unless @dirs
+      @files = {} unless @files
+      if sub_paths.size-1 == sub_paths_index
+        # Add File case - index points to last entry - leaf case.
+        file_stat = FileStat.new(sub_paths[sub_paths_index], @stable_state)
+        file_stat.set_event_queue(@event_queue)
+        file_stat.size = size
+        file_stat.modification_time = modification_time
+        file_stat.state = FileStatEnum::STABLE
+        add_file(file_stat)
+      else
+        # Add Dir to tree if not present. index points to new dir path.
+        dir_stat = @dirs[sub_paths[sub_paths_index]]
+        #create new dir if not exist
+        unless dir_stat
+          dir_stat = DirStat.new(sub_paths[sub_paths_index], @stable_state)
+          dir_stat.state = FileStatEnum::STABLE
+          dir_stat.set_event_queue(@event_queue)
+          add_dir(dir_stat)
+        end
+        # continue recursive call on tree with next sub path index
+        dir_stat.load_instance(sub_paths, sub_paths_index+1, size, modification_time)
+      end
     end
 
     #  Adds directory for monitoring.
@@ -218,7 +257,7 @@ module FileMonitoring
       end
 
       # The assignment
-      self.state= new_state
+      set_state(new_state)
     end
 
     # Updates the files and directories hashes and globs the directory for changes.
@@ -270,7 +309,7 @@ module FileMonitoring
                                   # change state only for existing directories
                                   # newly added directories have to remain with NEW state
             was_changed = true
-            ds = DirStat.new(file, self.stable_state, @content_data_cache, FileStatEnum::NON_EXISTING)
+            ds = DirStat.new(file, self.stable_state)
             ds.set_event_queue(@event_queue) unless @event_queue.nil?
             ds.monitor
             add_dir(ds)
@@ -282,11 +321,7 @@ module FileMonitoring
             was_changed = true
             # check if file exist in content data cache - set state to STABLE
             file_state = FileStatEnum::NON_EXISTING
-            if !@content_data_cache.nil? && @content_data_cache.include?(file)
-              file_state = FileStatEnum::STABLE
-            end
-            fs = FileStat.new(file, self.stable_state, @content_data_cache, file_state)
-
+            fs = FileStat.new(file, self.stable_state)
             fs.set_event_queue(@event_queue) unless @event_queue.nil?
             fs.monitor
             add_file(fs)

data/lib/log.rb

@@ -141,34 +141,70 @@ module Log
   end
 
   # Log info massages
-  def Log.info(msg)
+  # params:
+  #  msg, *args - Works to construct a string using: msg % args
+  #  Examples:
+  #   "hello"  # no args provided
+  #   "Time is %s. have a good day %s", Time.now, "Sir"
+  #  Note: Use ONLY %s in msg. Since %d will crush for nil objects, while %s will print empty string.
+  #        Implicit to_s is used to convert arg to %s.
+  #        If no to_s exists then ruby uses inspection.
+  def Log.info(msg, *args)
     Log.init if @log4r.nil?
+    msg = msg % args
     @log4r.info(msg_with_caller(msg))
     Log.flush if Params['log_flush_each_message']
   end
 
   # Log debug level 1 massages
-  def Log.debug1(msg)
+  # params:
+  #  msg, *args - Works to construct a string using: msg % args
+  #  Examples:
+  #   "hello"  # no args provided
+  #   "Time is %s. have a good day %s", Time.now, "Sir"
+  #  Note: Use ONLY %s in msg. Since %d will crush for nil objects, while %s will print empty string.
+  #        Implicit to_s is used to convert arg to %s.
+  #        If no to_s exists then ruby uses inspection.
+  def Log.debug1(msg, *args)
     if Params['log_debug_level'] >= 1
       Log.init if @log4r.nil?
+      msg = msg % args
       @log4r.debug(msg_with_caller(msg))
       Log.flush if Params['log_flush_each_message']
     end
   end
 
   # Log debug level 2 massages
-  def Log.debug2(msg)
+  # params:
+  #  msg, *args - Works to construct a string using: msg % args
+  #  Examples:
+  #   "hello"  # no args provided
+  #   "Time is %s. have a good day %s", Time.now, "Sir"
+  #  Note: Use ONLY %s in msg. Since %d will crush for nil objects, while %s will print empty string.
+  #        Implicit to_s is used to convert arg to %s.
+  #        If no to_s exists then ruby uses inspection.
+  def Log.debug2(msg, *args)
     if Params['log_debug_level'] >= 2
       Log.init if @log4r.nil?
+      msg = msg % args
       @log4r.debug(msg_with_caller(msg))
       Log.flush if Params['log_flush_each_message']
     end
   end
 
   # Log debug level 3 massages
-  def Log.debug3(msg)
+  # params:
+  #  msg, *args - Works to construct a string using: msg % args
+  #  Examples:
+  #   "hello"  # no args provided
+  #   "Time is %s. have a good day %s", Time.now, "Sir"
+  #  Note: Use ONLY %s in msg. Since %d will crush for nil objects, while %s will print empty string.
+  #        Implicit to_s is used to convert arg to %s.
+  #        If no to_s exists then ruby uses inspection.
+  def Log.debug3(msg, *args)
     if Params['log_debug_level'] >= 3
       Log.init if @log4r.nil?
+      msg = msg % args
       @log4r.debug(msg_with_caller(msg))
       Log.flush if Params['log_flush_each_message']
     end