file-digests 0.0.35 → 0.0.40

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 42e870b2a0e4e711e84a2748d09d8a378fc6964d78706d573ab22737d0f2519e
-  data.tar.gz: e14bef4ccd63fdc07950a257654c6594b793eabf44bbdf26cae59e621838e5c3
+  metadata.gz: a9d040599aee9aeb62234557b2a92edec36265e4439965e4c9d00a2c9afa117a
+  data.tar.gz: 992fe8a843afe761537a3c48b1ecde27ccdf8101be58dd4b456572c59232141b
 SHA512:
-  metadata.gz: b6b9eb636258ba06b29bd21736b477c0d225ba119ad3b3ccb82612c941bf892109876aa4e8da7402f580f4167f84000a1b2b4ae91e568ac50f223325fab856ef
-  data.tar.gz: afcdffefe0dcb346afcdf37ce6f3f523d4862a45a455838d8b7cbf603cead2c8fadbac185640088c79bd7e8c412b5da932bf301d90dd4337171b675a65f34388
+  metadata.gz: 1e15aa584690f8062a51cbb4785c9716e610258fba77859cb2b50642154701ffa1651312be1f6455921ee435ed7df5ed00bcc36f12def4009ac7edcc4aebb93e
+  data.tar.gz: 5c94ae4165677af0be7b790a7a92ffc9af7751f0dd2661bf238d06b02569ef4ec746908721c7d16781d11a431048759e1490f9e180ea9c2e647f5e585a3ebbe0

data/lib/file-digests.rb

@@ -139,15 +139,29 @@ class FileDigests
   end
 
   def initialize_paths files_path, digest_database_path
+    @start_time_filename_string = Time.now.strftime("%Y-%m-%d %H-%M-%S")
     @files_path = cleanup_path(files_path || ".")
+    raise "ERROR: Files path must be a readable directory" unless (File.directory?(@files_path) && File.readable?(@files_path))
+    @files_path = realpath_with_disk @files_path
 
-    raise "Files path must be a readable directory" unless (File.directory?(@files_path) && File.readable?(@files_path))
+    @error_log_path = @files_path + "file-digests errors #{@start_time_filename_string}.txt"
+    @missing_files_path = @files_path + "file-digests missing files #{@start_time_filename_string}.txt"
 
     @digest_database_path = digest_database_path ? cleanup_path(digest_database_path) : @files_path
     @digest_database_path += ".file-digests.sqlite" if File.directory?(@digest_database_path)
     ensure_dir_exist @digest_database_path.dirname
-
-    @digest_database_files = ["#{@digest_database_path}", "#{@digest_database_path}-wal", "#{@digest_database_path}-shm"]
+    @digest_database_path = realdirpath_with_disk @digest_database_path
+    
+    @digest_database_files = [
+      "#{@digest_database_path}",
+      "#{@digest_database_path}-wal",
+      "#{@digest_database_path}-shm"
+    ]
+
+    @skip_files = @digest_database_files + [
+      @error_log_path.to_s,
+      @missing_files_path.to_s
+    ]
 
     if @options[:verbose]
       puts "Target directory: #{@files_path}"
@@ -273,7 +287,7 @@ class FileDigests
       perhaps_transaction(@new_digest_algorithm, :exclusive) do
         @counters = {good: 0, updated: 0, renamed: 0, likely_damaged: 0, exceptions: 0}
 
-        walk_files do |filename|
+        walk_files(@files_path.to_s) do |filename|
           process_file filename
         end
 
@@ -286,7 +300,7 @@ class FileDigests
           if any_exceptions?
             STDERR.puts "Due to previously occurred errors, missing files will not removed from the database."
           else
-            print_missing_files
+            report_missing_files
             if !@options[:test_only] && (@options[:auto] || confirm("Remove missing files from the database"))
               nested_transaction do
                 puts "Removing missing files..." if @options[:verbose]
@@ -340,9 +354,11 @@ class FileDigests
   private
 
   def process_file filename
-    return if File.symlink? filename
+    perhaps_nt_filename = perhaps_nt_path filename
+
+    return if File.symlink? perhaps_nt_filename
 
-    stat = File.stat filename
+    stat = File.stat perhaps_nt_filename
 
     return if stat.blockdev?
     return if stat.chardev?
@@ -352,23 +368,19 @@ class FileDigests
 
     raise "File is not readable" unless stat.readable?
 
-    if @digest_database_files.include?(filename)
-      puts "SKIPPING DATABASE FILE: #{filename}" if @options[:verbose]
+    if @skip_files.include?(filename)
+      puts "SKIPPING FILE: #{filename}" if @options[:verbose]
       return
     end
 
     normalized_filename = filename.delete_prefix("#{@files_path.to_s}/").encode("utf-8", universal_newline: true).unicode_normalize(:nfkc)
     mtime_string = time_to_database stat.mtime
-    digest, new_digest = get_file_digest(filename)
+    digest, new_digest = get_file_digest(perhaps_nt_filename)
 
     nested_transaction do
       new_digests_insert(normalized_filename, new_digest) if new_digest
       process_file_indeed normalized_filename, mtime_string, digest
     end
-
-  rescue => exception
-    @counters[:exceptions] += 1
-    print_file_exception exception, filename
   end
 
   def process_file_indeed filename, mtime, digest
@@ -394,7 +406,7 @@ class FileDigests
     else
       if found["mtime"] == mtime && !@options[:accept_fate] # Digest is different and mtime is the same
         @counters[:likely_damaged] += 1
-        STDERR.puts "LIKELY DAMAGED: #{filename}"
+        error_text "LIKELY DAMAGED: #{filename}"
       else
         @counters[:updated] += 1
         puts "UPDATED#{" (FATE ACCEPTED)" if found["mtime"] == mtime && @options[:accept_fate]}: #{filename}" unless @options[:quiet]
@@ -424,10 +436,19 @@ class FileDigests
     @counters[:renamed] = @db.changes
   end
 
-  def print_missing_files
+  def report_missing_files
     puts "\nMISSING FILES:"
+    write_missing_files STDOUT
+    if missing_files_count > 256
+      File.open(@missing_files_path, "a") do |f|
+        write_missing_files f
+      end
+    end
+  end
+
+  def write_missing_files dest
     missing_files_select_all_filenames.each do |record|
-      puts record["filename"]
+      dest.puts record["filename"]
     end
   end
 
@@ -530,7 +551,7 @@ class FileDigests
   def check_if_database_is_at_certain_version target_version
     current_version = get_metadata("database_version")
     if current_version != target_version
-      STDERR.puts "This version of file-digests (#{FileDigests::VERSION || "unknown"}) is only compartible with the database version #{target_version}. Current database version is #{current_version}. To use this database, please install appropriate version if file-digest."
+      STDERR.puts "ERROR: This version of file-digests (#{FileDigests::VERSION || "unknown"}) is only compartible with the database version #{target_version}. Current database version is #{current_version}. To use this database, please install appropriate version if file-digest."
       raise "Incompatible database version"
     end
   end
@@ -542,6 +563,22 @@ class FileDigests
 
   # Filesystem-related helpers
 
+  def realpath_with_disk path
+    path = path.realpath
+    if Gem.win_platform? && path.to_s[0] == "/"
+      return Pathname(Dir.pwd[0, 2] + path.to_s)
+    end
+    path
+  end
+
+  def realdirpath_with_disk path
+    path = path.realdirpath
+    if Gem.win_platform? && path.to_s[0] == "/"
+      return Pathname(Dir.pwd[0, 2] + path.to_s)
+    end
+    path
+  end
+
   def patch_path_string path
     Gem.win_platform? ? path.gsub(/\\/, "/") : path
   end
@@ -560,10 +597,32 @@ class FileDigests
     end
   end
 
-  def walk_files
-    puts "Gathering the list of files..." if @options[:verbose]
-    Dir.glob(@files_path + "**" + "*", File::FNM_DOTMATCH) do |filename|
-      yield filename
+  def walk_files(path, &block)
+    Dir.each_child(path, encoding: "UTF-8") do |item|
+      item = "#{path}#{File::SEPARATOR}#{item}"
+      begin
+        item_perhaps_nt_path = perhaps_nt_path item
+        
+        unless File.symlink? item_perhaps_nt_path
+          if File.directory?(item_perhaps_nt_path)
+            raise "Directory is not readable" unless File.readable?(item_perhaps_nt_path)
+            walk_files(item, &block)
+          else
+            yield item
+          end
+        end
+      rescue => exception
+        @counters[:exceptions] += 1
+        report_file_exception exception, item
+      end
+    end
+  end
+
+  def perhaps_nt_path path
+    if Gem.win_platform?
+      "\\??\\#{path.gsub(/\//,"\\")}"
+    else
+      path
     end
   end
 
@@ -611,17 +670,31 @@ class FileDigests
     puts "Elapsed time: #{elapsed.to_i / 3600}h #{(elapsed.to_i % 3600) / 60}m #{"%.3f" % (elapsed % 60)}s" unless @options[:quiet]
   end
 
-  def print_file_exception exception, filename
-    STDERR.print "EXCEPTION: #{exception.message}, processing file: "
+  def report_file_exception exception, filename
+    write_file_exception STDERR, exception, filename
+    File.open(@error_log_path, "a") do |f|
+      write_file_exception f, exception, filename
+    end
+  end
+
+  def write_file_exception dest, exception, filename
+    dest.print "ERROR: #{exception.message}, processing file: "
     begin
-      STDERR.print filename.encode("utf-8", universal_newline: true)
+      dest.print filename.encode("utf-8", universal_newline: true)
     rescue
-      STDERR.print "(Unable to encode file name to utf-8) "
-      STDERR.print filename
+      dest.print "(Unable to encode file name to utf-8) "
+      dest.print filename
+    end
+    dest.print "\n"
+    dest.flush
+    exception.backtrace.each { |line| dest.puts "  " + line }
+  end
+
+  def error_text text
+    STDERR.puts text
+    File.open(@error_log_path, "a") do |f|
+      f.puts text
     end
-    STDERR.print "\n"
-    STDERR.flush
-    exception.backtrace.each { |line| STDERR.puts "  " + line }
   end
 
   def print_counters

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: file-digests
 version: !ruby/object:Gem::Version
-  version: 0.0.35
+  version: 0.0.40
 platform: ruby
 authors:
 - Stanislav Senotrusov
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-10-17 00:00:00.000000000 Z
+date: 2020-10-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: openssl