fs_cache 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: c7307ee577d076235b7d885087638f0576b59d408339a5c02150eeeaec12bc00
+  data.tar.gz: 8f66ac733ff6144a2fdcfc0f1fd052996daa24d78d8d64fc4e11415f46352d3a
+SHA512:
+  metadata.gz: 9889b8e9d5371facd30e07d7cf46c6d84ddb8bbc8b5059bf7a702bb5bf122c86bba5b01f0f2b37581ffbfaba77ebba7ec80be43ab4d219d02fa15e1de1b25fad
+  data.tar.gz: b660da8740076a449a15a8dca09080b5b89e3819181ebce93b335de544dc365bc51043eafdae1dffdca6a9286082206af3594390042779355cc82aedcc4f0741

data/lib/fs_cache.rb

@@ -0,0 +1,364 @@
+require 'progressbar'
+require 'fs_cache/attribute'
+
+# Implement a cache of the file system: directories and files presence.
+# Plugins can be used to also cache attributes of the files, like crc, size...
+class FsCache
+
+  ATTRIBUTE_PLUGINS_MODULE = FsCache::Attributes
+
+  # Constructor
+  #
+  # Parameters::
+  # * *attribute_plugins_dirs* (Array<String>): List of directories containing possible attribute plugins [default = []]
+  def initialize(attribute_plugins_dirs: [])
+    # List of possible attribute plugins, per attribute name
+    # Hash<Symbol, Attribute>
+    @attribute_plugins = {}
+    # Tree of dependent attributes: for each attribute in this tree, the list of attributes to be invalidated if this attribute changes.
+    # Hash<Symbol, Array<Symbol> >
+    @dependent_attributes = {}
+    # Big database of files information
+    # Hash<String, Hash<Symbol,Object> >: For each file name, the file info (can be incomplete if it was never fetched):
+    # * *exist* (Boolean): Does the file exist?
+    # * *size* (Integer): File size.
+    # * *crc* (String): File CRC.
+    # * *corruption* (false or Object): Info about this file's corruption, or false if sane.
+    @files = Hash.new { |h, k| h[k] = {} }
+    # Directories information
+    # Hash<String, Hash<Symbol,Object> >: For each directory name, the dir info (can be incomplete if it was never fetched):
+    # * *files* (Hash<String,nil>): Set of files (base names)
+    # * *dirs* (Hash<String,nil>): Set of directories (base names)
+    # * *recursive_dirs* (Hash<String,nil>): Set of recursive sub-directories (full paths)
+    # * *recursive_files* (Hash<String,nil>): Set of recursive files (full paths)
+    @dirs = Hash.new { |h, k| h[k] = {} }
+
+    # Automatically register attributes from the plugins dirs
+    (["#{__dir__}/fs_cache/attributes"] + attribute_plugins_dirs).each do |attribute_plugins_dir|
+      Dir.glob("#{attribute_plugins_dir}/*.rb") do |attribute_plugin_file|
+        attribute = File.basename(attribute_plugin_file)[0..-4].to_sym
+        require attribute_plugin_file
+        class_name = attribute.to_s.split('_').collect(&:capitalize).join.to_sym
+        if ATTRIBUTE_PLUGINS_MODULE.const_defined?(class_name)
+          plugin_class = ATTRIBUTE_PLUGINS_MODULE.const_get(class_name)
+          register_attribute_plugin(attribute, plugin_class.new)
+        else
+          raise "Attributes plugin #{attribute_plugin_file} does not define the class #{class_name} inside #{ATTRIBUTE_PLUGINS_MODULE}" if plugin_class.nil?
+        end
+      end
+    end
+  end
+
+  # Register a new attributes' plugin.
+  # The constructor already registers all plugins found in the plugins directories.
+  # This method exists in order to register plugins that could be dynamically instantiated.
+  #
+  # Parameters::
+  # * *attribute* (Symbol): The attribute
+  # * *plugin* (Attribute): The attribute plugin
+  def register_attribute_plugin(attribute, plugin)
+    puts "Registering attribute plugin #{attribute}..."
+    raise "Attributes plugin #{attribute} is already defined (by class #{@attribute_plugins[attribute].class.name})." if @attribute_plugins.key?(attribute)
+    @attribute_plugins[attribute] = plugin
+    # Define the getter methods for this attribute, directly in the base class for performance purposes
+
+    # Get the attribute for a given file.
+    # Use the cache if possible.
+    #
+    # Parameters::
+    # * *file* (String): File path for which we look for the attribute
+    # Result::
+    # * Object: Corresponding attribute value, or nil if the file does not exist
+    define_singleton_method("#{attribute}_for".to_sym) do |file|
+      @files[file][attribute] = plugin.attribute_for(file) if !@files[file].key?(attribute) && exist?(file)
+      @files[file][attribute]
+    end
+
+    # If there are some helpers, register them too
+    if plugin.class.const_defined?(:Helpers)
+      helpers_module = plugin.class.const_get(:Helpers)
+      self.class.include helpers_module unless helpers_module.nil?
+    end
+    # If this attribute is dependent on others, remember it too
+    plugin.invalidated_on_change_of.each do |parent_attribute|
+      @dependent_attributes[parent_attribute] = [] unless @dependent_attributes.key?(parent_attribute)
+      @dependent_attributes[parent_attribute] << attribute
+    end
+  end
+
+  # Is a file existing?
+  #
+  # Parameters::
+  # * *file* (String): File name
+  # Result::
+  # * String: Is the file existing?
+  def exist?(file)
+    # If there is at least 1 attribute for this file it means that it exists
+    unless @files[file].key?(:exist)
+      @files[file][:exist] =
+        # If we have an attribute for this file, it means it exist
+        if @files[file].size > 0
+          true
+        else
+          dir = File.dirname(file)
+          if @dirs.key?(dir)
+            # We know about its directory, so we should know if it is there
+            @dirs[dir][:files].key?(File.basename(file))
+          else
+            File.exist?(file)
+          end
+        end
+    end
+    @files[file][:exist]
+  end
+
+  # Get list of files from a directory (base names)
+  #
+  # Parameters::
+  # * *dir* (String): The directory to get files from
+  # Result::
+  # * Array<String>: List of file base names
+  def files_in(dir)
+    ensure_dir_data(dir)
+    @dirs[dir][:files].keys
+  end
+
+  # Get recursive list of directories from a directory
+  #
+  # Parameters::
+  # * *dir* (String): The directory to get other directories from
+  # Result::
+  # * Array<String>: List of directories
+  def dirs_from(dir)
+    unless @dirs[dir].key?(:recursive_dirs)
+      ensure_dir_data(dir)
+      recursive_dirs = {}
+      @dirs[dir][:dirs].keys.each do |subdir|
+        full_subdir = "#{dir}/#{subdir}"
+        recursive_dirs[full_subdir] = nil
+        recursive_dirs.merge!(Hash[dirs_from(full_subdir).map { |subsubdir| [subsubdir, nil] }])
+      end
+      @dirs[dir][:recursive_dirs] = recursive_dirs
+    end
+    @dirs[dir][:recursive_dirs].keys
+  end
+
+  # Get recursive list of files from a directory
+  #
+  # Parameters::
+  # * *dir* (String): The directory to get other directories from
+  # Result::
+  # * Array<String>: List of files
+  def files_from(dir)
+    unless @dirs[dir].key?(:recursive_files)
+      ensure_dir_data(dir)
+      recursive_files = Hash[@dirs[dir][:files].keys.map { |file| ["#{dir}/#{file}", nil] }]
+      @dirs[dir][:dirs].keys.each do |subdir|
+        recursive_files.merge!(Hash[files_from("#{dir}/#{subdir}").map { |file| [file, nil] }])
+      end
+      @dirs[dir][:recursive_files] = recursive_files
+    end
+    @dirs[dir][:recursive_files].keys
+  end
+
+  # Scan files and directories from a list of directories.
+  # Use a progress bar.
+  #
+  # Parameters::
+  # * *dirs* (Array<String>): List of directories to preload
+  # * *include_attributes* (Array<Symbol> or nil): List of attributes to scan, or nil for all [default = nil]
+  # * *exclude_attributes* (Array<Symbol>): List of attributes to ignore while scanning [default = []]
+  def scan(dirs, include_attributes: nil, exclude_attributes: [])
+    progressbar = ProgressBar.create(title: 'Indexing files')
+    attributes_to_scan = (include_attributes.nil? ? @attribute_plugins.keys : include_attributes) - exclude_attributes
+    files = dirs.
+      map do |dir|
+        dirs_from(dir)
+        files_from(dir)
+      end.
+      flatten
+    progressbar.total = files.size
+    files.each do |file|
+      exist?(file)
+      attributes_to_scan.each do |attribute|
+        self.send "#{attribute}_for", file
+      end
+      progressbar.increment
+    end
+  end
+
+  # Serialize into JSON.
+  #
+  # Result::
+  # * Object: JSON object
+  def to_json
+    {
+      files: @files,
+      dirs: @dirs
+    }
+  end
+
+  # Get data from JSON.
+  #
+  # Parameters::
+  # * *json* (Object): JSON object
+  def from_json(json)
+    json = json.transform_keys(&:to_sym)
+    @files = Hash[json[:files].map { |file, file_info| [file, file_info.transform_keys(&:to_sym)] }]
+    @files.default_proc = proc { |h, k| h[k] = {} }
+    @dirs = Hash[json[:dirs].map { |dir, dir_info| [dir, dir_info.transform_keys(&:to_sym)] }]
+    @dirs.default_proc = proc { |h, k| h[k] = {} }
+  end
+
+  # Notify the file system that a given file has been deleted
+  #
+  # Parameters::
+  # * *file* (String): File being deleted
+  def notify_file_rm(file)
+    @files[file] = { exist: false }
+    unregister_file_from_dirs(file)
+  end
+
+  # Notify the file system of a file copy
+  #
+  # Parameters::
+  # * *src* (String): Origin file
+  # * *dst* (String): Destination file
+  def notify_file_cp(src, dst)
+    if @files.key?(src)
+      @files[dst] = @files[src].clone
+    else
+      @files[src] = { exist: true }
+      @files[dst] = { exist: true }
+    end
+    register_file_in_dirs(dst)
+  end
+
+  # Notify the file system of a file move
+  #
+  # Parameters::
+  # * *src* (String): Origin file
+  # * *dst* (String): Destination file
+  def notify_file_mv(src, dst)
+    notify_file_cp(src, dst)
+    notify_file_rm(src)
+  end
+
+  # Check our info against file system changes.
+  # This detects
+  # * files that have been deleted,
+  # * any change in the directories structure,
+  # * any change in the attributes that are already part of the cache and that are not ignored explicitely.
+  #
+  # Parameters::
+  # * *include_attributes* (Array<Symbol> or nil): List of attributes to scan, or nil for all [default = nil]
+  # * *exclude_attributes* (Array<Symbol>): List of attributes to ignore while scanning [default = []]
+  def check(include_attributes: nil, exclude_attributes: [])
+    progressbar = ProgressBar.create(title: 'Refreshing files info')
+    attributes_to_scan = (include_attributes.nil? ? @attribute_plugins.keys : include_attributes) - exclude_attributes
+    progressbar.total = @files.size
+    @files.each do |file, file_info|
+      if File.exist?(file)
+        if file_info.key?(:exist) && !file_info[:exist]
+          # This file has been added when we thought it was missing
+          file_info.replace(exist: true)
+        else
+          # Check attributes that are already present
+          (file_info.keys & attributes_to_scan).each do |attribute|
+            current_attribute = file_info[attribute]
+            new_attribute = @attribute_plugins[attribute].attribute_for(file)
+            if current_attribute != new_attribute
+              # Attribute has changed
+              file_info[attribute] = new_attribute
+              # If some other attributes were depending on this one, invalidate them
+              if @dependent_attributes.key?(attribute)
+                @dependent_attributes[attribute].each do |dependent_attribute|
+                  file_info.delete(dependent_attribute)
+                end
+              end
+            end
+          end
+        end
+      elsif !file_info.key?(:exist) || file_info[:exist]
+        # This file has been removed when we thought it was there
+        file_info.replace(exist: false)
+      end
+      progressbar.increment
+    end
+    # Rebuilding @dirs structure needs to make the Dir.glob commands once again. Therefore there is no need to check it. Removing it will rebuild it anyway.
+    @dirs.clear
+  end
+
+  # Remove attributes for a list of files
+  #
+  # Parameters::
+  # * *files* (Array<String>): The list of files to invalidate attributes for
+  # * *include_attributes* (Array<Symbol> or nil): List of attributes to scan, or nil for all [default = nil]
+  # * *exclude_attributes* (Array<Symbol>): List of attributes to ignore while scanning [default = []]
+  def invalidate(files, include_attributes: nil, exclude_attributes: [])
+    attributes_to_invalidate = ((include_attributes.nil? ? @attribute_plugins.keys : include_attributes) - exclude_attributes)
+    files.each do |file|
+      if @files.key?(file)
+        attributes_to_invalidate.each do |attribute|
+          @files[file].delete(attribute)
+        end
+      end
+    end
+  end
+
+  private
+
+  # Register a file in the @dirs structure
+  #
+  # Parameters::
+  # * *file* (String): File to register in @dirs
+  def register_file_in_dirs(file)
+    file_dir = File.dirname(file)
+    split_dir = file_dir.split('/')
+    split_dir.size.times do |idx|
+      dir = split_dir[0..idx].join('/')
+      @dirs[dir][:recursive_files][file] = nil if @dirs.key?(dir) && @dirs[dir].key?(:recursive_files) && !@dirs[dir][:recursive_files].key?(file)
+    end
+    base_name = File.basename(file)
+    @dirs[file_dir][:files][base_name] = nil if @dirs.key?(file_dir) && @dirs[file_dir].key?(:files) && !@dirs[file_dir][:files].key?(base_name)
+  end
+
+  # Unregister a file in the @dirs structure
+  #
+  # Parameters::
+  # * *file* (String): File to unregister from @dirs
+  def unregister_file_from_dirs(file)
+    file_dir = File.dirname(file)
+    split_dir = file_dir.split('/')
+    split_dir.size.times do |idx|
+      dir = split_dir[0..idx].join('/')
+      # Remove any reference of our file to this dir info
+      @dirs[dir][:recursive_files].delete(file) if @dirs.key?(dir) && @dirs[dir].key?(:recursive_files)
+    end
+    @dirs[file_dir][:files].delete(File.basename(file)) if @dirs.key?(file_dir) && @dirs[file_dir].key?(:files)
+  end
+
+  # Populate a given directory data (files and dirs)
+  #
+  # Parameters::
+  # * *dir* (String): Directory to get data from
+  def ensure_dir_data(dir)
+    unless @dirs[dir].key?(:files)
+      files = {}
+      dirs = {}
+      Dir.glob("#{dir}/*", File::FNM_DOTMATCH).each do |file|
+        base_name = File.basename(file)
+        if File.directory?(file)
+          dirs[base_name] = nil if base_name != '.' && base_name != '..'
+        else
+          files[base_name] = nil
+        end
+      end
+      @dirs[dir] = {
+        files: files,
+        dirs: dirs
+      }
+    end
+  end
+
+end

data/lib/fs_cache/attribute.rb

@@ -0,0 +1,20 @@
+class FsCache
+
+  module Attributes
+  end
+
+  # Common ancestor for all attributes
+  class Attribute
+
+    # Get the list of other attributes that invalidate this one.
+    # If any of those attributes is chaning on a file, then reset our attribute for the file.
+    #
+    # Result::
+    # * Array<Symbol>: List of dependent attributes
+    def invalidated_on_change_of
+      []
+    end
+
+  end
+
+end

data/lib/fs_cache/attributes/crc.rb

@@ -0,0 +1,105 @@
+require 'zlib'
+
+class FsCache
+
+  module Attributes
+
+    # CRC attribute. Can be:
+    # * String: Internal CRC computed from files by blocks
+    class Crc < Attribute
+
+      # Size of blocks to compute CRCs in bytes. Changing this value will invalidate previously computed CRCs.
+      CRC_BLOCK_SIZE = 32 * 1024 * 1024 # 32 MB
+
+      # Get the attribute for a given file
+      #
+      # Parameters::
+      # * *file* (String): File to get the attribute for
+      # Result::
+      # * Object: Corresponding attribute value
+      def attribute_for(file)
+        blocks_crc = ''
+        File.open(file, 'rb') do |file_io|
+          buffer = nil
+          while (buffer = file_io.read(CRC_BLOCK_SIZE))
+            blocks_crc << Zlib.crc32(buffer, 0).to_s(16).upcase
+          end
+        end
+        Zlib.crc32(blocks_crc, 0).to_s(16).upcase
+      end
+
+      # Get the list of other attributes that invalidate this one.
+      # If any of those attributes is chaning on a file, then reset our attribute for the file.
+      #
+      # Result::
+      # * Array<Symbol>: List of dependent attributes
+      def invalidated_on_change_of
+        [:size]
+      end
+
+      # Add helpers to the cache
+      module Helpers
+
+        # Provide info on the differences between 2 directories.
+        #
+        # Parameters::
+        # * *dir1* (String): First directory
+        # * *dir2* (String): Second directory
+        # Result::
+        # * Hash<Symbol,Object>: Difference between the 2 directories (dir2 - dir1):
+        #   * *same* (Array<String>): Same files
+        #   * *renamed* (Array<[String,String]>): Renamed files (from dir1 to dir2: [file_base1, file_base2])
+        #   * *added* (Array<String>): Added files
+        #   * *deleted* (Array<String>): Deleted files
+        #   * *different* (Array<String>): Different files
+        def diff_dirs(dir1, dir2)
+          files1 = Hash[files_in(dir1).map { |file| [file, "#{dir1}/#{file}"] }]
+          files2 = Hash[files_in(dir2).map { |file| [file, "#{dir2}/#{file}"] }]
+          same = []
+          different = []
+          renamed = []
+          # First process files having the same names
+          files1.delete_if do |file_base1, file1|
+            if files2.key?(file_base1)
+              # A file with same name exists in dir2
+              if crc_for(files2[file_base1]) == crc_for(file1)
+                same << file_base1
+              else
+                different << file_base1
+              end
+              files2.delete(file_base1)
+              true
+            else
+              false
+            end
+          end
+          # Then process files having the same CRC among the remaining ones
+          files1.delete_if do |file_base1, file1|
+            crc1 = crc_for(file1)
+            found_file_base2, _found_file2 = files2.find { |_file_base2, file2| crc_for(file2) == crc1 }
+            if found_file_base2.nil?
+              false
+            else
+              renamed << [file_base1, found_file_base2]
+              files2.delete(found_file_base2)
+              true
+            end
+          end
+          remaining_files1 = files1.keys
+          remaining_files2 = files2.keys
+          {
+            same: same,
+            renamed: renamed,
+            added: remaining_files2 - remaining_files1,
+            deleted: remaining_files1 - remaining_files2,
+            different: different
+          }
+        end
+
+      end
+
+    end
+
+  end
+
+end

data/lib/fs_cache/attributes/size.rb

@@ -0,0 +1,38 @@
+class FsCache
+
+  module Attributes
+
+    # Size attribute. Can be:
+    # * Integer: File size
+    class Size < Attribute
+
+      # Get the attribute for a given file
+      #
+      # Parameters::
+      # * *file* (String): File to get the attribute for
+      # Result::
+      # * Object: Corresponding attribute value
+      def attribute_for(file)
+        File.stat(file).size
+      end
+
+      # Add helpers to the cache
+      module Helpers
+
+        # Is a file empty?
+        #
+        # Parameters::
+        # * *file* (String): File name
+        # Result::
+        # * String: Is the file empty?
+        def empty?(file)
+          size_for(file) == 0
+        end
+
+      end
+
+    end
+
+  end
+
+end

metadata

@@ -0,0 +1,64 @@
+--- !ruby/object:Gem::Specification
+name: fs_cache
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Muriel Salvan
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2019-07-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: progressbar
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+description: 
+email:
+- muriel@x-aeon.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/fs_cache.rb
+- lib/fs_cache/attribute.rb
+- lib/fs_cache/attributes/crc.rb
+- lib/fs_cache/attributes/size.rb
+homepage: http://x-aeon.com
+licenses:
+- BSD-3-Clause
+metadata:
+  homepage_uri: http://x-aeon.com
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.7.6
+signing_key: 
+specification_version: 4
+summary: Simple file system caching to perform huge and repetitive accesses to files,
+  directories and various files' content analysis
+test_files: []