content_server 1.1.0 → 1.2.0

data/lib/file_monitoring/monitor_path.rb

@@ -0,0 +1,304 @@
+require 'content_server/server'
+require 'log'
+require 'params'
+
+module FileMonitoring
+  #  Path monitoring.
+
+  #  Enum-like structure that includes possible filesystem entities (files/directories) states:
+  # * <tt>NON_EXISTING</tt> - Entity that was treated during previous run, but absent currently
+  # * <tt>NEW</tt> - Entity that was found and added to control during this run
+  # * <tt>CHANGED</tt> - State was changed between two checks
+  # * <tt>UNCHANGED</tt> - Opposite to CHANGED
+  # * <tt>STABLE</tt> - Entity is in the UNCHANGED state for a defined (by user) number of iterations
+
+
+  class FileStatEnum
+    NON_EXISTING = "NON_EXISTING"
+    NEW = "NEW"
+    CHANGED = "CHANGED"
+    UNCHANGED = "UNCHANGED"
+    STABLE = "STABLE"
+  end
+
+  #  This class holds current state of file and methods to control and report changes
+  class FileStat
+    attr_reader :cycles, :path, :stable_state, :state, :size, :modification_time
+
+    DEFAULT_STABLE_STATE = 10
+
+    @@log = nil
+
+    #  Initializes new file monitoring object
+    # ==== Arguments:
+    #
+    # * <tt>path</tt> - File location
+    # * <tt>stable_state</tt> - Number of iterations to move unchanged file to stable state
+    def initialize(path, stable_state = DEFAULT_STABLE_STATE, content_data_cache, state)
+      @path ||= path
+      @size = nil
+      @creation_time = nil
+      @modification_time = nil
+      @cycles = 0  # number of iterations from the last file modification
+      @state = state
+      @stable_state = stable_state  # number of iteration to move unchanged file to stable state
+    end
+
+    def set_output_queue(event_queue)
+      @event_queue = event_queue
+    end
+
+    #  Sets a log file to report changes
+    # ==== Arguments:
+    #
+    # * <tt>log</tt> - already opened ruby File object
+    def self.set_log (log)
+      @@log = log
+    end
+
+    # Checks whether file was changed from the last iteration.
+    # For files, size and modification time are checked.
+    def monitor
+      file_stats = File.lstat(@path) rescue nil
+      new_state = nil
+      if file_stats == nil
+        new_state = FileStatEnum::NON_EXISTING
+        @size = nil
+        @creation_time = nil
+        @modification_time = nil
+        @cycles = 0
+      elsif @size == nil
+        new_state = FileStatEnum::NEW
+        @size = file_stats.size
+        @creation_time = file_stats.ctime.utc
+        @modification_time = file_stats.mtime.utc
+        @cycles = 0
+      elsif changed?(file_stats)
+        new_state = FileStatEnum::CHANGED
+        @size = file_stats.size
+        @creation_time = file_stats.ctime.utc
+        @modification_time = file_stats.mtime.utc
+        @cycles = 0
+      else
+        new_state = FileStatEnum::UNCHANGED
+        @cycles += 1
+        if @cycles >= @stable_state
+          new_state = FileStatEnum::STABLE
+        end
+      end
+
+      # The assignment
+      self.state= new_state
+    end
+
+    #  Checks that stored file attributes are the same as file attributes taken from file system.
+    def changed?(file_stats)
+      not (file_stats.size == @size &&
+          file_stats.ctime.utc == @creation_time.utc &&
+          file_stats.mtime.utc == @modification_time.utc)
+    end
+
+    def set_event_queue(queue)
+      @event_queue = queue
+    end
+
+    #  Sets and writes to the log a new state.
+    def state= (new_state)
+      if (@state != new_state or @state == FileStatEnum::CHANGED)
+        @state = new_state
+        if (@@log)
+          @@log.info(state + ": " + path)
+          @@log.outputters[0].flush if Params['log_flush_each_message']
+        end
+        if @event_queue and FileStatEnum::NEW != @state  # NEW state is ignored in indexer
+          Log.debug1 "Writing to event queue [#{self.state}, #{self.path}]"
+          @event_queue.push([self.state, self.instance_of?(DirStat), self.path,
+                             self.modification_time, self.size])
+          $process_vars.set('monitor to index queue size', @event_queue.size)
+        end
+      end
+    end
+
+    #  Checks whether path and state are the same as of the argument
+    def == (other)
+      @path == other.path and @stable_state == other.stable_state
+    end
+
+    #  Returns path and state of the file with indentation
+    def to_s (indent = 0)
+      (" " * indent) + path.to_s + " : " + state.to_s
+    end
+  end
+
+  #  This class holds current state of directory and methods to control changes
+  class DirStat < FileStat
+    #  Initializes new directory monitoring object
+    # ==== Arguments:
+    #
+    # * <tt>path</tt> - File location
+    # * <tt>stable_state</tt> - Number of iterations to move unchanged directory to stable state
+    def initialize(path, stable_state = DEFAULT_STABLE_STATE, content_data_cache, state)
+      super
+      @dirs = nil
+      @files = nil
+      @non_utf8_paths = {}
+      @content_data_cache = content_data_cache
+    end
+
+    #  Adds directory for monitoring.
+    def add_dir (dir)
+      @dirs[dir.path] = dir
+    end
+
+    #  Adds file for monitoring.
+    def add_file (file)
+      @files[file.path] = file
+    end
+
+    #  Removes directory from monitoring.
+    def rm_dir(dir)
+      @dirs.delete(dir.path)
+    end
+
+    #  Removes file from monitoring.
+    def rm_file(file)
+      @files.delete(file.path)
+    end
+
+    #  Checks that there is a sub-folder with a given path.
+    def has_dir?(path)
+      @dirs.has_key?(path)
+    end
+
+    #  Checks that there is a file with a given path.
+    def has_file?(path)
+      @files.has_key?(path)
+    end
+
+    #  Returns string which contains path and state of this directory as well as it's structure.
+    def to_s(indent = 0)
+      indent_increment = 2
+      child_indent = indent + indent_increment
+      res = super
+      @files.each_value do |file|
+        res += "\n" + file.to_s(child_indent)
+      end if @files
+      @dirs.each_value do |dir|
+        res += "\n" + dir.to_s(child_indent)
+      end if @dirs
+      res
+    end
+
+    # Checks that directory structure (i.e. files and directories located directly under this directory)
+    # wasn't changed since the last iteration.
+    def monitor
+      was_changed = false
+      new_state = nil
+      self_stat = File.lstat(@path) rescue nil
+      if self_stat == nil
+        new_state = FileStatEnum::NON_EXISTING
+        @files = nil
+        @dirs = nil
+        @cycles = 0
+      elsif @files == nil
+        new_state = FileStatEnum::NEW
+        @files = Hash.new
+        @dirs = Hash.new
+        @cycles = 0
+        update_dir
+      elsif update_dir
+        new_state = FileStatEnum::CHANGED
+        @cycles = 0
+      else
+        new_state = FileStatEnum::UNCHANGED
+        @cycles += 1
+        if @cycles >= @stable_state
+          new_state = FileStatEnum::STABLE
+        end
+      end
+
+      # The assignment
+      self.state= new_state
+    end
+
+    # Updates the files and directories hashes and globs the directory for changes.
+    def update_dir
+      was_changed = false
+
+      # monitor existing and absent files
+      @files.each_value do |file|
+        file.monitor
+
+        if file.state == FileStatEnum::NON_EXISTING
+          was_changed = true
+          rm_file(file)
+        end
+      end
+
+      @dirs.each_value do |dir|
+        dir.monitor
+
+        if dir.state == FileStatEnum::NON_EXISTING
+          was_changed = true
+          rm_dir(dir)
+        end
+      end
+
+      was_changed = was_changed || glob_me
+
+      return was_changed
+    end
+
+    #  Globs the directory for new files and directories
+    def glob_me
+      was_changed = false
+      files = Dir.glob(path + "/*")
+
+      # add and monitor new files and directories
+      files.each do |file|
+        # keep only files with names in UTF-8
+        next if @non_utf8_paths[file]
+        check_utf_8_encoding_file = file.clone
+        unless check_utf_8_encoding_file.force_encoding("UTF-8").valid_encoding?
+          Log.warning("Non UTF-8 file name '#{check_utf_8_encoding_file}', skipping.")
+          @non_utf8_paths[file]=true
+          next
+        end
+        file_stat = File.lstat(file) rescue nil
+        if (file_stat.directory?)
+          unless (has_dir?(file)) # new directory
+                                  # change state only for existing directories
+                                  # newly added directories have to remain with NEW state
+            was_changed = true
+            ds = DirStat.new(file, self.stable_state, @content_data_cache, FileStatEnum::NON_EXISTING)
+            ds.set_event_queue(@event_queue) unless @event_queue.nil?
+            ds.monitor
+            add_dir(ds)
+          end
+        else # it is a file
+          unless(has_file?(file)) # new file
+                                  # change state only for existing directories
+                                  # newly added directories have to remain with NEW state
+            was_changed = true
+            # check if file exist in content data cache - set state to STABLE
+            file_state = FileStatEnum::NON_EXISTING
+            if !@content_data_cache.nil? && @content_data_cache.include?(file)
+              file_state = FileStatEnum::STABLE
+            end
+            fs = FileStat.new(file, self.stable_state, @content_data_cache, file_state)
+
+            fs.set_event_queue(@event_queue) unless @event_queue.nil?
+            fs.monitor
+            add_file(fs)
+          end
+        end
+      end
+
+      return was_changed
+    end
+
+    protected :add_dir, :add_file, :rm_dir, :rm_file, :update_dir, :glob_me
+  end
+
+end
+

data/lib/file_monitoring/version.rb

@@ -0,0 +1,3 @@
+module FileMonitoring
+  VERSION = "1.1.0"
+end

data/lib/file_monitoring.rb

@@ -0,0 +1,29 @@
+require 'file_monitoring/file_monitoring'
+
+# Daemon for monitoring directories for changes.
+# Paths are checked for changes per user-defined period of time.
+#
+# Directory defined changed when:
+# 1. Directory structure changed, i.e. sub-directories or files were added/removed
+# 2. One of the files located in the directory or one of its sub-directories was changed
+# 3. One of sub-directories changed (see 1. and 2. above)
+#
+# File monitoring controled by following configuration parameters:
+# * <tt>default_monitoring_log_path</tt> - holds path of file monitoring log.
+#   This log containd track of changes found during monitoring
+# * <tt>monitoring_paths</tt> - path and file monitoring configuration data
+#   regarding these paths.
+
+module FileMonitoring
+  Params.path('default_monitoring_log_path', '~/.bbfs/log/file_monitoring.log',
+              'Default path for file monitoring log file. ' \
+              'This log containd track of changes found during monitoring')
+  Params.complex('monitoring_paths', [], 'Array of Hashes with 3 fields: ' \
+                 'path, scan_period and stable_state.')
+
+  # @see FileMonitoring#monitor_files
+  def monitor_files
+    fm = FileMonitoring.new
+    fm.monitor_files
+  end
+end

data/lib/file_utils/file_generator/README

@@ -0,0 +1,97 @@
+File Generator utility
+------------------------------------------------------------------
+A. Short program description:
+	Generates files and folders with random content with random file size according to
+	the given FileGenerator Parameters.
+	The parameters can be provided by two ways. One way is by YML file. Second way is
+	by passing parameters to application trought command line.
+	The two ways and meaning of parameters described in next sections.
+
+B. Instalation of application and it dependencies:
+	1. The File Generator utility is integrated to general file utility.
+	   You have to install file_utils and log
+	   Install file_utils as follow:
+	   bbfs> gem build file_utils.gemspec
+	   after run this you will get next message: "Successfully built RubyGem..."
+	   and run:
+       bbfs> gem install file_utils-0.0.1.gem
+       after run this you will get next message:
+       "Successfully installed file_utils-0.0.1..."
+
+	2. The File Generation Util is depend on log gem. Install log gem as follow:
+	   bbfs> gem build log.gemspec
+	   after run this you will get next message: "Successfully built RubyGem..."
+       run:
+       bbfs> gem install log-0.0.1.gem
+       and get "Successfully installed log-0.0.1..."
+
+C. How to run.
+	1. Create YML configuration file and locate it in file system. The path where the
+	   YML file is located must be provided by --conf_file parameter that passed to
+	   program in command line.
+    2. Run the program by
+       bbfs> ruby -Ilib bin/file_utils --command=generate_files --conf_file=~/.bbfs/conf/file_generator.yml
+    3. In order to see all parameters that can be provided to application run next:
+       bbfs> ruby -Ilib bin/file_utils --print_params_to_stdout=true
+    4. In order to get help run: bbfs> ruby -Ilib bin/file_utils --help
+
+D. Desription of YML file
+	The parameters of YML file is divided to two relevant group:
+	"Location and naming", "Processing"
+
+	********************************Location and naming*******************************
+	target_path      - Represents the root path where directories and files will be
+	                   generated.
+	dir_name_prefix  - Represents the directory name template. Name of created
+	                   directories will be dir_name_prefix_xxx. where xxx is random
+	                   characters
+	file_name_prefix - Represents the file name template for generated file. Name of
+	                   created files will be dir_name_prefix_xxx. where xxx is random
+	                   characters
+
+	*******************************************Processing**********************************************
+	total_created_directories  - Represents the total created directories. Use any
+	                             negative value or zero for Infinity. Not relevant if
+	                             random calculation is triggered.
+	is_tot_files_in_dir_random - Indicates that total files in a directory will be
+	                             calculated randomly. If false then value will be
+	                             taken from parameter:'total_files_in_dir'.
+	                             Also lower\upper_limit_4_files_in_dir parameters will
+	                             not be used.
+	lower_limit_4_files_in_dir - Represents the Lower limit for total files in
+	                             directory for random calculation. Active only if
+	                             parameter:'is_tot_files_in_dir_random' is true.
+	upper_limit_4_files_in_dir - Represents the Upper limit for total files in
+	                             directory for random calculation. Active only if
+	                             parameter:'is_tot_files_in_dir_random' is true.
+	total_files_in_dir         - Represents the total files in directory. Use any
+	                             negative value or zero for Infinity. Active only if
+	                             parameter:'is_tot_files_in_dir_random' is false.
+	is_use_random_size         - Indicates that file size will be calculated randomly.
+	                             If false then value will be taken from
+	                             parameter:'file_size_in_mb'. Also next 2 parameters
+	                             will not be used.
+	upper_size_in_mb           - Represents the upper limit file size in MB for random
+	                             size calculation.
+	                             Active only if parameter:'is_use_random_size' is
+	                             true.
+	down_size_in_mb            - Represents the Lower limit file size in MB for random
+	                             size calculation. Active only if
+	                             parameter:'is_use_random_size' is true.
+	file_size_in_mb            - Represents the file size in MB. Not relevant if
+	                             random calculation is triggered. Active only if
+	                             parameter:'is_use_random_size' is false.
+	sleep_time_in_seconds      - Represent delay between two close cycles of
+	                             processing. In each cycle of processing
+	                             file\s and\or folder\s created.
+
+	Example of using parameter in yml document is in line below:
+	target_path: home/user_name/.bbfs/test_files
+
+    Note: you can override parameters given in YML file by providing the parameter
+    trough command line.
+
+
+
+
+

data/lib/file_utils/file_generator/file_generator.rb

@@ -0,0 +1,156 @@
+# Author: Slava Pasechnik (slavapas13@gmail.com)
+# Run from bbfs> file_utils generate_files
+
+require 'digest'
+require 'fileutils'
+require 'params'
+require 'yaml'
+
+require 'log'
+
+module FileGenerator
+  Params.path('target_path', '~/.bbfs/test_files', 'Represents the target path for files generation')
+  Params.boolean('is_clear_target_path', false, 'Should files and directories already presented in target path be removed')
+  Params.string('file_name_prefix', 'auto_generated_file_4_backup_server',
+                'Represents the file name template for generated file')
+  Params.string('dir_name_prefix', 'test_dir_4_backup_server',
+                'Represents the directory name template for generated file')
+  Params.integer('file_size_in_bytes', 500,
+                 'Represents the required file size in MB. Not relevant if random calculation is triggered')
+  Params.integer('total_created_directories', -1,
+                 'Represents the total created directories. Use any negative value or zero for Infinity.
+                      Not relevant if random calculation is triggered')
+  Params.integer('max_total_files', 0, 'Maximum number of files to generate, 0 for no limit.')
+  Params.integer('total_files_in_dir', 10,
+                 'Represents the total files in directory. Use any negative value or zero for Infinity')
+  Params.float('sleep_time_in_seconds', 10, 'Represents the sleeping time for generation files')
+  Params.float('upper_size_in_bytes', 500, 'Represents the upper limit file size in MB for random size calculation')
+  Params.float('down_size_in_bytes', 50, 'Represents the Lower limit file size in MB for random size calculation')
+  Params.integer('lower_limit_4_files_in_dir', 10,
+                 'Represents the Lower limit for total files in directory for random calculation')
+  Params.integer('upper_limit_4_files_in_dir', 20,
+                 'Represents the Upper limit for total files in directory for random calculation')
+  Params.boolean('is_tot_files_in_dir_random', true,
+                 'Indicates that total files in a directory will be calculated randomly.')
+  Params.boolean('is_use_random_size', true, 'Indicates that file size will be calculated randomly.')
+
+  #Generates files with random content with random file size according to the given Params
+  class FileGenerator
+    attr_reader :one_bytes_string
+    FILE_EXT = 'txt'
+
+    #Gets one MB string
+    def one_bytes_string
+      @one_bytes_string ||= prepare_one_bytes_string
+    end
+
+    #Gets the some random string
+    def get_small_rand_unique_str
+      rand(36**8).to_s(36)
+    end
+
+    #Gets the unique name according to current time and random string
+    def get_unique_name
+      "#{Time.new.to_i}_#{get_small_rand_unique_str}"
+    end
+
+    #Gets the random letter
+    def get_random_letter
+      (rand(122-97) + 97).chr
+    end
+
+    #Gets the new directory name
+    def get_new_directory_name
+      "#{Params['dir_name_prefix']}_#{get_unique_name}"
+    end
+
+    #Gets the new file name
+    def get_new_file_name
+      "#{Params['file_name_prefix']}_#{get_unique_name}.#{FILE_EXT}"
+    end
+
+    #Gets the required file size im MB
+    def get_file_bytes_size
+      if Params['is_use_random_size']
+        rand(Params['upper_size_in_bytes'] - Params['down_size_in_bytes']) +
+            Params['down_size_in_bytes']
+      else
+        Params['file_size_in_bytes']
+      end
+    end
+
+    #Generates one MB string with random content
+    def prepare_one_bytes_string
+      get_random_letter
+    end
+
+    #Determines whether to generate new directory or not
+    def is_generate_dir(dir_counter, total_file_counter)
+      if Params['max_total_files'] > 0 && total_file_counter > Params['max_total_files']
+        return false
+      end
+
+      if Params['total_created_directories'] < 1 || #Any negative value or zero will be indication for Infinity
+          dir_counter < Params['total_created_directories'] then
+        return true
+      else
+        return false
+      end
+    end
+
+    #Determines whether to generate new file or not
+    def is_generate_file(file_counter, total_file_counter)
+      if Params['max_total_files'] > 0 && total_file_counter > Params['max_total_files']
+        return false
+      end
+
+      total_files_in_dir = Params['total_files_in_dir']
+      if file_counter == 0 && Params['is_tot_files_in_dir_random']  then
+        total_files_in_dir =
+            rand(Params['upper_limit_4_files_in_dir'] -
+                     Params['lower_limit_4_files_in_dir']) +
+                Params['lower_limit_4_files_in_dir']
+      end
+
+      #When total_files_in_dir < 1 it will be treated as unlimited files creation in directory
+      if total_files_in_dir < 1 ||
+          file_counter < total_files_in_dir then
+        return true
+      else
+        return false
+      end
+    end
+
+    #Generates files with random content with random file size according to the given Params
+    def run
+      dir_counter = 0
+      total_file_count = 0
+      if Params['is_clear_target_path']
+        target_path_all_regexp = File.expand_path(File.join(Params['target_path'], '*'))
+        Dir[target_path_all_regexp].each {|file| FileUtils.rm_rf file}
+      end
+      while is_generate_dir dir_counter, total_file_count
+        new_dir_name = File.expand_path(File.join Params['target_path'], get_new_directory_name)
+        ::FileUtils.mkdir_p new_dir_name unless File.directory?(new_dir_name)
+        dir_counter += 1
+        file_counter = 0
+
+        while is_generate_file file_counter, total_file_count
+          new_file_name = get_new_file_name
+          File.open(File.join(new_dir_name, new_file_name), "w") do |f|
+            get_file_bytes_size.to_i.times { f.write(get_random_letter) }
+          end
+          file_counter += 1
+          total_file_count += 1
+
+          sleep Params['sleep_time_in_seconds'] if Params['sleep_time_in_seconds'] > 0
+        end
+      end
+    end
+  end
+end # module FileGenerator
+
+
+
+
+