file-digests 0.0.36 → 0.0.41

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 624f7ec80a0cb15be2bc07a3dd592c33c7a178c22fb7f59ef326760d17cc564f
-  data.tar.gz: 92bb88c58bc152ad01649b0688861ea16af6801d2d6625c238b96e9ff00aaed4
+  metadata.gz: 911caad1e008b6365ad3f0043e992793a9d77ff13ce71b0da004608a8261398b
+  data.tar.gz: f98a696b52e1549694d6ff80f9b3efe894b8ad9dc79c7bc7da1a01a74800fb77
 SHA512:
-  metadata.gz: 9dbca39af63f07ddfe4d92618d0ff5e0813fa30561d42b629caa918f664f1149f8ad895e8555137b0f462eb4eac942adda07efed03859d17cb635a8d9bb25b57
-  data.tar.gz: 788d37c5d82e8892dfcfb6f4a8532f484f7cf17c5f937e8a64b09af902fe3a904ed034fc88d403852bbd8819a9d11ab61a672e2d6f3114420ed7777ea9db2961
+  metadata.gz: 9004f1053569a94f58a4f90279ae1c86597a72baa0451ea62bf0481650f3e049d5a4dff6c513e093abf808cf603ec6e9d4ad017a143c16d8ed16007912797b6c
+  data.tar.gz: fa65d38a9fe7dde9cc29f5a389d6f1ee52e4593637b0f67a2173821180f8aeeea9bddddf6bbf95ecc7a9ada47364869f4ef087e90b405ef1d7329d00f9c39375

data/lib/file-digests.rb

@@ -1,3 +1,5 @@
+# encoding: UTF-8
+
 #  Copyright 2020 Stanislav Senotrusov <stan@senotrusov.com>
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
@@ -15,10 +17,8 @@
 require "date"
 require "digest"
 require "fileutils"
-require "find"
 require "openssl"
 require "optparse"
-require "pathname"
 require "set"
 require "sqlite3"
 
@@ -139,16 +139,106 @@ class FileDigests
     puts "Using #{@digest_algorithm} digest algorithm" if @options[:verbose]
   end
 
+  def perform_check
+    measure_time do
+      perhaps_transaction(@new_digest_algorithm, :exclusive) do
+        @counters = {good: 0, updated: 0, renamed: 0, likely_damaged: 0, exceptions: 0}
+
+        walk_files(@files_path) do |filename|
+          process_file filename
+        end
+
+        nested_transaction do
+          puts "Tracking renames..." if @options[:verbose]
+          track_renames
+        end
+
+        if any_missing_files?
+          if any_exceptions?
+            STDERR.puts "Due to previously occurred errors, missing files will not removed from the database."
+          else
+            report_missing_files
+            if !@options[:test_only] && (@options[:auto] || confirm("Remove missing files from the database"))
+              nested_transaction do
+                puts "Removing missing files..." if @options[:verbose]
+                remove_missing_files
+              end
+            end
+          end
+        end
+
+        if @new_digest_algorithm && !@options[:test_only]
+          if any_missing_files? || any_likely_damaged? || any_exceptions?
+            STDERR.puts "ERROR: New digest algorithm will not be in effect until there are files that are missing, likely damaged, or processed with an exception."
+          else
+            puts "Updating database to a new digest algorithm..." if @options[:verbose]
+            digests_update_digests_to_new_digests
+            set_metadata "digest_algorithm", @new_digest_algorithm
+            puts "Transition to a new digest algorithm complete: #{@new_digest_algorithm}"
+          end
+        end
+
+        if any_likely_damaged? || any_exceptions?
+          STDERR.puts "PLEASE REVIEW ERRORS THAT WERE OCCURRED!"
+          STDERR.puts "A list of errors is also saved in a file: #{@error_log_path}"
+        end
+
+        set_metadata(@options[:test_only] ? "latest_test_only_check_time" : "latest_complete_check_time", time_to_database(Time.now))
+
+        print_counters
+      end
+      
+      puts "Performing database maintenance..." if @options[:verbose]
+      execute "PRAGMA optimize"
+      execute "VACUUM"
+      execute "PRAGMA wal_checkpoint(TRUNCATE)"
+
+      hide_database_files
+    end
+  end
+
+  def show_duplicates
+    current_digest = nil
+    digests_select_duplicates.each do |found|
+      if current_digest != found["digest"]
+        puts "" if current_digest
+        current_digest = found["digest"]
+        puts "#{found["digest"]}:"
+      end
+      puts "  #{found["filename"]}"
+    end
+  end
+
+  private
+
   def initialize_paths files_path, digest_database_path
-    @files_path = cleanup_path(files_path || ".")
+    @files_path = realpath(files_path || ".")
+    
+    unless File.directory?(@files_path) && File.readable?(@files_path)
+      raise "ERROR: Files path must be a readable directory" 
+    end
 
-    raise "Files path must be a readable directory" unless (File.directory?(@files_path) && File.readable?(@files_path))
+    @start_time_filename_string = Time.now.strftime("%Y-%m-%d %H-%M-%S")
+   
+    @error_log_path = "#{@files_path}#{File::SEPARATOR}file-digests errors #{@start_time_filename_string}.txt"
+    @missing_files_path = "#{@files_path}#{File::SEPARATOR}file-digests missing files #{@start_time_filename_string}.txt"
 
-    @digest_database_path = digest_database_path ? cleanup_path(digest_database_path) : @files_path
-    @digest_database_path += ".file-digests.sqlite" if File.directory?(@digest_database_path)
-    ensure_dir_exist @digest_database_path.dirname
+    @digest_database_path = digest_database_path ? realdirpath(digest_database_path) : @files_path
 
-    @digest_database_files = ["#{@digest_database_path}", "#{@digest_database_path}-wal", "#{@digest_database_path}-shm"]
+    if File.directory?(@digest_database_path)
+      @digest_database_path += "#{File::SEPARATOR}.file-digests.sqlite"
+    end
+
+    @digest_database_files = [
+      @digest_database_path,
+      "#{@digest_database_path}-wal",
+      "#{@digest_database_path}-shm"
+    ]
+
+    @skip_files = @digest_database_files + [
+      @error_log_path,
+      @missing_files_path
+    ]
 
     if @options[:verbose]
       puts "Target directory: #{@files_path}"
@@ -157,7 +247,7 @@ class FileDigests
   end
 
   def initialize_database
-    @db = SQLite3::Database.new @digest_database_path.to_s
+    @db = SQLite3::Database.new @digest_database_path
     @db.results_as_hash = true
     @db.busy_timeout = 5000
 
@@ -216,7 +306,7 @@ class FileDigests
       # Convert database from 1st to 2nd version
       unless get_metadata("digest_algorithm")
         if get_metadata("database_version") == "1"
-          if File.exist?(@digest_database_path.dirname + ".file-digests.sha512")
+          if File.exist?("#{File.dirname(@digest_database_path)}#{File::SEPARATOR}.file-digests.sha512")
             set_metadata("digest_algorithm", "SHA512")
           else
             set_metadata("digest_algorithm", "SHA256")
@@ -269,81 +359,84 @@ class FileDigests
     prepare_method :digests_update_digests_to_new_digests, "INSERT INTO digests (filename, digest, digest_check_time) SELECT filename, digest, false FROM new_digests WHERE true ON CONFLICT (filename) DO UPDATE SET digest=excluded.digest"
   end
 
-  def perform_check
-    measure_time do
-      perhaps_transaction(@new_digest_algorithm, :exclusive) do
-        @counters = {good: 0, updated: 0, renamed: 0, likely_damaged: 0, exceptions: 0}
 
-        walk_files do |filename|
-          process_file filename
-        end
+  # Files
 
-        nested_transaction do
-          puts "Tracking renames..." if @options[:verbose]
-          track_renames
-        end
+  def realpath path
+    realxpath path, :realpath
+  end
 
-        if any_missing_files?
-          if any_exceptions?
-            STDERR.puts "Due to previously occurred errors, missing files will not removed from the database."
-          else
-            print_missing_files
-            if !@options[:test_only] && (@options[:auto] || confirm("Remove missing files from the database"))
-              nested_transaction do
-                puts "Removing missing files..." if @options[:verbose]
-                remove_missing_files
-              end
-            end
-          end
-        end
+  def realdirpath path
+    realxpath path, :realdirpath
+  end
 
-        if @new_digest_algorithm && !@options[:test_only]
-          if any_missing_files? || any_likely_damaged? || any_exceptions?
-            STDERR.puts "ERROR: New digest algorithm will not be in effect until there are files that are missing, likely damaged, or processed with an exception."
-          else
-            puts "Updating database to a new digest algorithm..." if @options[:verbose]
-            digests_update_digests_to_new_digests
-            set_metadata "digest_algorithm", @new_digest_algorithm
-            puts "Transition to a new digest algorithm complete: #{@new_digest_algorithm}"
-          end
-        end
+  def realxpath path, method_name
+    path = path.encode("utf-8")
 
-        if any_likely_damaged? || any_exceptions?
-          STDERR.puts "PLEASE REVIEW ERRORS THAT WERE OCCURRED!"
-        end
+    if Gem.win_platform?
+      path = path.gsub(/\\/, "/")
+    end
 
-        set_metadata(@options[:test_only] ? "latest_test_only_check_time" : "latest_complete_check_time", time_to_database(Time.now))
+    path = File.send(method_name, path).encode("utf-8")
 
-        print_counters
-      end
-      
-      puts "Performing database maintenance..." if @options[:verbose]
-      execute "PRAGMA optimize"
-      execute "VACUUM"
-      execute "PRAGMA wal_checkpoint(TRUNCATE)"
+    if Gem.win_platform? && path[0] == "/"
+      path = Dir.pwd[0, 2].encode("utf-8") + path
+    end
 
-      hide_database_files
+    path
+  end
+
+  def perhaps_nt_path path
+    if Gem.win_platform?
+      "\\??\\#{path.gsub(/\//,"\\")}"
+    else
+      path
     end
   end
 
-  def show_duplicates
-    current_digest = nil
-    digests_select_duplicates.each do |found|
-      if current_digest != found["digest"]
-        puts "" if current_digest
-        current_digest = found["digest"]
-        puts "#{found["digest"]}:"
+  def get_file_digest filename
+    File.open(filename, "rb") do |io|
+      digest = OpenSSL::Digest.new(@digest_algorithm)
+      new_digest = OpenSSL::Digest.new(@new_digest_algorithm) if @new_digest_algorithm
+
+      buffer = ""
+      while io.read(409600, buffer) # 409600 seems like a sweet spot
+        digest.update(buffer)
+        new_digest.update(buffer) if @new_digest_algorithm
       end
-      puts "  #{found["filename"]}"
+      return [digest.hexdigest, (new_digest.hexdigest if @new_digest_algorithm)]
     end
   end
 
-  private
+  def walk_files(path, &block)
+    Dir.each_child(path, encoding: "UTF-8") do |item|
+      item = "#{path}#{File::SEPARATOR}#{item.encode("utf-8")}"
+      begin
+        item_perhaps_nt_path = perhaps_nt_path item
+        
+        unless File.symlink? item_perhaps_nt_path
+          if File.directory?(item_perhaps_nt_path)
+            raise "Directory is not readable" unless File.readable?(item_perhaps_nt_path)
+            walk_files(item, &block)
+          else
+            yield item
+          end
+        end
+
+      rescue => exception
+        @counters[:exceptions] += 1
+        report_file_exception exception, item
+      end
+    end
+  end
 
   def process_file filename
-    return if File.symlink? filename
+    perhaps_nt_filename = perhaps_nt_path filename
+
+    # this is checked in the walk_files
+    # return if File.symlink? perhaps_nt_filename
 
-    stat = File.stat filename
+    stat = File.stat perhaps_nt_filename
 
     return if stat.blockdev?
     return if stat.chardev?
@@ -353,23 +446,19 @@ class FileDigests
 
     raise "File is not readable" unless stat.readable?
 
-    if @digest_database_files.include?(filename)
-      puts "SKIPPING DATABASE FILE: #{filename}" if @options[:verbose]
+    if @skip_files.include?(filename)
+      puts "SKIPPING FILE: #{filename}" if @options[:verbose]
       return
     end
 
-    normalized_filename = filename.delete_prefix("#{@files_path.to_s}/").encode("utf-8", universal_newline: true).unicode_normalize(:nfkc)
+    normalized_filename = filename.delete_prefix("#{@files_path}#{File::SEPARATOR}").encode("utf-8", universal_newline: true).unicode_normalize(:nfkc)
     mtime_string = time_to_database stat.mtime
-    digest, new_digest = get_file_digest(filename)
+    digest, new_digest = get_file_digest(perhaps_nt_filename)
 
     nested_transaction do
       new_digests_insert(normalized_filename, new_digest) if new_digest
       process_file_indeed normalized_filename, mtime_string, digest
     end
-
-  rescue => exception
-    @counters[:exceptions] += 1
-    print_file_exception exception, filename
   end
 
   def process_file_indeed filename, mtime, digest
@@ -395,7 +484,7 @@ class FileDigests
     else
       if found["mtime"] == mtime && !@options[:accept_fate] # Digest is different and mtime is the same
         @counters[:likely_damaged] += 1
-        STDERR.puts "LIKELY DAMAGED: #{filename}"
+        error_text "LIKELY DAMAGED: #{filename}"
       else
         @counters[:updated] += 1
         puts "UPDATED#{" (FATE ACCEPTED)" if found["mtime"] == mtime && @options[:accept_fate]}: #{filename}" unless @options[:quiet]
@@ -425,10 +514,20 @@ class FileDigests
     @counters[:renamed] = @db.changes
   end
 
-  def print_missing_files
+  def report_missing_files
     puts "\nMISSING FILES:"
+    write_missing_files STDOUT
+    if missing_files_count > 256
+      File.open(@missing_files_path, "a") do |f|
+        write_missing_files f
+      end
+      puts "\n(A list of missing files is also saved in a file: #{@missing_files_path})"
+    end
+  end
+
+  def write_missing_files dest
     missing_files_select_all_filenames.each do |record|
-      puts record["filename"]
+      dest.puts record["filename"]
     end
   end
 
@@ -446,6 +545,17 @@ class FileDigests
   end
 
 
+  # Runtime state helpers
+
+  def any_exceptions?
+    @counters[:exceptions] > 0
+  end
+
+  def any_likely_damaged?
+    @counters[:likely_damaged] > 0
+  end
+
+
   # Database helpers
 
   def execute *args, &block
@@ -531,7 +641,7 @@ class FileDigests
   def check_if_database_is_at_certain_version target_version
     current_version = get_metadata("database_version")
     if current_version != target_version
-      STDERR.puts "This version of file-digests (#{FileDigests::VERSION || "unknown"}) is only compartible with the database version #{target_version}. Current database version is #{current_version}. To use this database, please install appropriate version if file-digest."
+      STDERR.puts "ERROR: This version of file-digests (#{FileDigests::VERSION || "unknown"}) is only compartible with the database version #{target_version}. Current database version is #{current_version}. To use this database, please install appropriate version if file-digest."
       raise "Incompatible database version"
     end
   end
@@ -541,57 +651,6 @@ class FileDigests
   end
 
 
-  # Filesystem-related helpers
-
-  def patch_path_string path
-    Gem.win_platform? ? path.gsub(/\\/, "/") : path
-  end
-
-  def cleanup_path path
-    Pathname.new(patch_path_string(path)).cleanpath
-  end
-
-  def ensure_dir_exist path
-    if File.exist?(path)
-      unless File.directory?(path)
-        raise "#{path} is not a directory"
-      end
-    else
-      FileUtils.mkdir_p path
-    end
-  end
-
-  def walk_files
-    Find.find(@files_path) do |path|
-      yield path
-    end
-  end
-
-  def get_file_digest filename
-    File.open(filename, "rb") do |io|
-      digest = OpenSSL::Digest.new(@digest_algorithm)
-      new_digest = OpenSSL::Digest.new(@new_digest_algorithm) if @new_digest_algorithm
-
-      buffer = ""
-      while io.read(409600, buffer) # 409600 seems like a sweet spot
-        digest.update(buffer)
-        new_digest.update(buffer) if @new_digest_algorithm
-      end
-      return [digest.hexdigest, (new_digest.hexdigest if @new_digest_algorithm)]
-    end
-  end
-
-
-  # Runtime state helpers
-
-  def any_exceptions?
-    @counters[:exceptions] > 0
-  end
-
-  def any_likely_damaged?
-    @counters[:likely_damaged] > 0
-  end
-
   # UI helpers
 
   def confirm text
@@ -611,17 +670,31 @@ class FileDigests
     puts "Elapsed time: #{elapsed.to_i / 3600}h #{(elapsed.to_i % 3600) / 60}m #{"%.3f" % (elapsed % 60)}s" unless @options[:quiet]
   end
 
-  def print_file_exception exception, filename
-    STDERR.print "EXCEPTION: #{exception.message}, processing file: "
+  def report_file_exception exception, filename
+    write_file_exception STDERR, exception, filename
+    File.open(@error_log_path, "a") do |f|
+      write_file_exception f, exception, filename
+    end
+  end
+
+  def write_file_exception dest, exception, filename
+    dest.print "ERROR: #{exception.message}, processing file: "
     begin
-      STDERR.print filename.encode("utf-8", universal_newline: true)
+      dest.print filename.encode("utf-8", universal_newline: true)
     rescue
-      STDERR.print "(Unable to encode file name to utf-8) "
-      STDERR.print filename
+      dest.print "(Unable to encode file name to utf-8) "
+      dest.print filename
+    end
+    dest.print "\n"
+    dest.flush
+    exception.backtrace.each { |line| dest.puts "  " + line }
+  end
+
+  def error_text text
+    STDERR.puts text
+    File.open(@error_log_path, "a") do |f|
+      f.puts text
     end
-    STDERR.print "\n"
-    STDERR.flush
-    exception.backtrace.each { |line| STDERR.puts "  " + line }
   end
 
   def print_counters

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: file-digests
 version: !ruby/object:Gem::Version
-  version: 0.0.36
+  version: 0.0.41
 platform: ruby
 authors:
 - Stanislav Senotrusov
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-10-17 00:00:00.000000000 Z
+date: 2020-10-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: openssl