file-digests 0.0.37 → 0.0.42

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 27e992f2a4849569d6c87e53807ebdc53676b3e47ca2c1efd2799927fd16d0c7
-  data.tar.gz: 9e292709b7978d906b0423a980cc72b23f8a44c665f070d71f30953ccdc59256
+  metadata.gz: c914ef250d4e173e31498c81865f4b43da56f91f81f2f16325f573747758c20b
+  data.tar.gz: 55bec69638ec367ca346de5b1d72194daa0bab75972186e0dd3ca4da0a0ceb9e
 SHA512:
-  metadata.gz: c80f844d16255d9437c8dd012eff010df23f5a14d87336bf987b34e913a12fa0932fd85e18760884c63d10f882041afecf77b0b354555cef62aadc98de5ba091
-  data.tar.gz: f5f5c8309b8921f034edeb36643cab34dff52e4091cd5508462294a48e79002dfdbdf09a126988449d1775d207651579ca19ffb7cd169038fe71067dabb5b8af
+  metadata.gz: 3963d53565a261db7b7bad2d8d662aa8fe407b92ddca9264afd10f6f9451f6aa1cfeec2825a503fc6b01f18cc85a672e4cf031109a2279e3c2d74d5c74c7cdb2
+  data.tar.gz: e6be01d8d1887010387ba5206bcb5937dac395fa5ea05c546076f76998d4603030ba5e781f43c2414e99f20bc241fcc3446721e7b9dca668af0fe7e6cff7fe46

data/lib/file-digests.rb

@@ -1,3 +1,5 @@
+# encoding: UTF-8
+
 #  Copyright 2020 Stanislav Senotrusov <stan@senotrusov.com>
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
@@ -17,7 +19,6 @@ require "digest"
 require "fileutils"
 require "openssl"
 require "optparse"
-require "pathname"
 require "set"
 require "sqlite3"
 
@@ -112,6 +113,7 @@ class FileDigests
 
     file_digests = self.new ARGV[0], ARGV[1], options
     file_digests.send(options[:action] || :perform_check)
+    file_digests.close_database
   end
 
   def initialize files_path, digest_database_path, options = {}
@@ -138,16 +140,110 @@ class FileDigests
     puts "Using #{@digest_algorithm} digest algorithm" if @options[:verbose]
   end
 
+  def perform_check
+    measure_time do
+      perhaps_transaction(@new_digest_algorithm, :exclusive) do
+        @counters = {good: 0, updated: 0, renamed: 0, likely_damaged: 0, exceptions: 0}
+
+        walk_files(@files_path) do |filename|
+          process_file filename
+        end
+
+        nested_transaction do
+          puts "Tracking renames..." if @options[:verbose]
+          track_renames
+        end
+
+        if any_missing_files?
+          if any_exceptions?
+            STDERR.puts "Due to previously occurred errors, missing files will not removed from the database."
+          else
+            report_missing_files
+            if !@options[:test_only] && (@options[:auto] || confirm("Remove missing files from the database"))
+              nested_transaction do
+                puts "Removing missing files..." if @options[:verbose]
+                remove_missing_files
+              end
+            end
+          end
+        end
+
+        if @new_digest_algorithm && !@options[:test_only]
+          if any_missing_files? || any_likely_damaged? || any_exceptions?
+            STDERR.puts "ERROR: New digest algorithm will not be in effect until there are files that are missing, likely damaged, or processed with an exception."
+          else
+            puts "Updating database to a new digest algorithm..." if @options[:verbose]
+            digests_update_digests_to_new_digests
+            set_metadata "digest_algorithm", @new_digest_algorithm
+            puts "Transition to a new digest algorithm complete: #{@new_digest_algorithm}"
+          end
+        end
+
+        if any_likely_damaged? || any_exceptions?
+          STDERR.puts "PLEASE REVIEW ERRORS THAT WERE OCCURRED!"
+          STDERR.puts "A list of errors is also saved in a file: #{@error_log_path}"
+        end
+
+        set_metadata(@options[:test_only] ? "latest_test_only_check_time" : "latest_complete_check_time", time_to_database(Time.now))
+
+        print_counters
+      end
+
+      puts "Performing database maintenance..." if @options[:verbose]
+      execute "PRAGMA optimize"
+      execute "VACUUM"
+      execute "PRAGMA wal_checkpoint(TRUNCATE)"
+    end
+  end
+
+  def show_duplicates
+    current_digest = nil
+    digests_select_duplicates.each do |found|
+      if current_digest != found["digest"]
+        puts "" if current_digest
+        current_digest = found["digest"]
+        puts "#{found["digest"]}:"
+      end
+      puts "  #{found["filename"]}"
+    end
+  end
+
+  def close_database
+    @statements.each(&:close)
+    @db.close
+    hide_database_files
+  end
+
+  private
+
   def initialize_paths files_path, digest_database_path
-    @files_path = cleanup_path(files_path || ".")
+    @files_path = realpath(files_path || ".")
+
+    unless File.directory?(@files_path) && File.readable?(@files_path)
+      raise "ERROR: Files path must be a readable directory"
+    end
 
-    raise "Files path must be a readable directory" unless (File.directory?(@files_path) && File.readable?(@files_path))
+    @start_time_filename_string = Time.now.strftime("%Y-%m-%d %H-%M-%S")
 
-    @digest_database_path = digest_database_path ? cleanup_path(digest_database_path) : @files_path
-    @digest_database_path += ".file-digests.sqlite" if File.directory?(@digest_database_path)
-    ensure_dir_exist @digest_database_path.dirname
+    @error_log_path = "#{@files_path}#{File::SEPARATOR}file-digests errors #{@start_time_filename_string}.txt"
+    @missing_files_path = "#{@files_path}#{File::SEPARATOR}file-digests missing files #{@start_time_filename_string}.txt"
+
+    @digest_database_path = digest_database_path ? realdirpath(digest_database_path) : @files_path
+
+    if File.directory?(@digest_database_path)
+      @digest_database_path += "#{File::SEPARATOR}.file-digests.sqlite"
+    end
 
-    @digest_database_files = ["#{@digest_database_path}", "#{@digest_database_path}-wal", "#{@digest_database_path}-shm"]
+    @digest_database_files = [
+      @digest_database_path,
+      "#{@digest_database_path}-wal",
+      "#{@digest_database_path}-shm"
+    ]
+
+    @skip_files = @digest_database_files + [
+      @error_log_path,
+      @missing_files_path
+    ]
 
     if @options[:verbose]
       puts "Target directory: #{@files_path}"
@@ -156,9 +252,10 @@ class FileDigests
   end
 
   def initialize_database
-    @db = SQLite3::Database.new @digest_database_path.to_s
+    @db = SQLite3::Database.new @digest_database_path
     @db.results_as_hash = true
     @db.busy_timeout = 5000
+    @statements = []
 
     execute "PRAGMA encoding = 'UTF-8'"
     execute "PRAGMA locking_mode = 'EXCLUSIVE'"
@@ -215,7 +312,7 @@ class FileDigests
       # Convert database from 1st to 2nd version
       unless get_metadata("digest_algorithm")
         if get_metadata("database_version") == "1"
-          if File.exist?(@digest_database_path.dirname + ".file-digests.sha512")
+          if File.exist?("#{File.dirname(@digest_database_path)}#{File::SEPARATOR}.file-digests.sha512")
             set_metadata("digest_algorithm", "SHA512")
           else
             set_metadata("digest_algorithm", "SHA256")
@@ -228,7 +325,7 @@ class FileDigests
         execute "CREATE INDEX digests_digest ON digests(digest)"
         set_metadata "database_version", "3"
       end
-  
+
       check_if_database_is_at_certain_version "3"
 
       create_temporary_tables
@@ -268,81 +365,83 @@ class FileDigests
     prepare_method :digests_update_digests_to_new_digests, "INSERT INTO digests (filename, digest, digest_check_time) SELECT filename, digest, false FROM new_digests WHERE true ON CONFLICT (filename) DO UPDATE SET digest=excluded.digest"
   end
 
-  def perform_check
-    measure_time do
-      perhaps_transaction(@new_digest_algorithm, :exclusive) do
-        @counters = {good: 0, updated: 0, renamed: 0, likely_damaged: 0, exceptions: 0}
+  # Files
 
-        walk_files do |filename|
-          process_file filename
-        end
+  def realpath path
+    realxpath path, :realpath
+  end
 
-        nested_transaction do
-          puts "Tracking renames..." if @options[:verbose]
-          track_renames
-        end
+  def realdirpath path
+    realxpath path, :realdirpath
+  end
 
-        if any_missing_files?
-          if any_exceptions?
-            STDERR.puts "Due to previously occurred errors, missing files will not removed from the database."
-          else
-            print_missing_files
-            if !@options[:test_only] && (@options[:auto] || confirm("Remove missing files from the database"))
-              nested_transaction do
-                puts "Removing missing files..." if @options[:verbose]
-                remove_missing_files
-              end
-            end
-          end
-        end
+  def realxpath path, method_name
+    path = path.encode("utf-8")
 
-        if @new_digest_algorithm && !@options[:test_only]
-          if any_missing_files? || any_likely_damaged? || any_exceptions?
-            STDERR.puts "ERROR: New digest algorithm will not be in effect until there are files that are missing, likely damaged, or processed with an exception."
-          else
-            puts "Updating database to a new digest algorithm..." if @options[:verbose]
-            digests_update_digests_to_new_digests
-            set_metadata "digest_algorithm", @new_digest_algorithm
-            puts "Transition to a new digest algorithm complete: #{@new_digest_algorithm}"
-          end
-        end
+    if Gem.win_platform?
+      path = path.gsub(/\\/, "/")
+    end
 
-        if any_likely_damaged? || any_exceptions?
-          STDERR.puts "PLEASE REVIEW ERRORS THAT WERE OCCURRED!"
-        end
+    path = File.send(method_name, path).encode("utf-8")
 
-        set_metadata(@options[:test_only] ? "latest_test_only_check_time" : "latest_complete_check_time", time_to_database(Time.now))
+    if Gem.win_platform? && path[0] == "/"
+      path = Dir.pwd[0, 2].encode("utf-8") + path
+    end
 
-        print_counters
-      end
-      
-      puts "Performing database maintenance..." if @options[:verbose]
-      execute "PRAGMA optimize"
-      execute "VACUUM"
-      execute "PRAGMA wal_checkpoint(TRUNCATE)"
+    path
+  end
 
-      hide_database_files
+  def perhaps_nt_path path
+    if Gem.win_platform?
+      "\\??\\#{path.gsub(/\//,"\\")}"
+    else
+      path
     end
   end
 
-  def show_duplicates
-    current_digest = nil
-    digests_select_duplicates.each do |found|
-      if current_digest != found["digest"]
-        puts "" if current_digest
-        current_digest = found["digest"]
-        puts "#{found["digest"]}:"
+  def get_file_digest filename
+    File.open(filename, "rb") do |io|
+      digest = OpenSSL::Digest.new(@digest_algorithm)
+      new_digest = OpenSSL::Digest.new(@new_digest_algorithm) if @new_digest_algorithm
+
+      buffer = ""
+      while io.read(409600, buffer) # 409600 seems like a sweet spot
+        digest.update(buffer)
+        new_digest.update(buffer) if @new_digest_algorithm
       end
-      puts "  #{found["filename"]}"
+      return [digest.hexdigest, (new_digest.hexdigest if @new_digest_algorithm)]
     end
   end
 
-  private
+  def walk_files(path, &block)
+    Dir.each_child(path, encoding: "UTF-8") do |item|
+      item = "#{path}#{File::SEPARATOR}#{item.encode("utf-8")}"
+      begin
+        item_perhaps_nt_path = perhaps_nt_path item
+
+        unless File.symlink? item_perhaps_nt_path
+          if File.directory?(item_perhaps_nt_path)
+            raise "Directory is not readable" unless File.readable?(item_perhaps_nt_path)
+            walk_files(item, &block)
+          else
+            yield item
+          end
+        end
+
+      rescue => exception
+        @counters[:exceptions] += 1
+        report_file_exception exception, item
+      end
+    end
+  end
 
   def process_file filename
-    return if File.symlink? filename
+    perhaps_nt_filename = perhaps_nt_path filename
+
+    # this is checked in the walk_files
+    # return if File.symlink? perhaps_nt_filename
 
-    stat = File.stat filename
+    stat = File.stat perhaps_nt_filename
 
     return if stat.blockdev?
     return if stat.chardev?
@@ -352,23 +451,19 @@ class FileDigests
 
     raise "File is not readable" unless stat.readable?
 
-    if @digest_database_files.include?(filename)
-      puts "SKIPPING DATABASE FILE: #{filename}" if @options[:verbose]
+    if @skip_files.include?(filename)
+      puts "SKIPPING FILE: #{filename}" if @options[:verbose]
       return
     end
 
-    normalized_filename = filename.delete_prefix("#{@files_path.to_s}/").encode("utf-8", universal_newline: true).unicode_normalize(:nfkc)
+    normalized_filename = filename.delete_prefix("#{@files_path}#{File::SEPARATOR}").encode("utf-8", universal_newline: true).unicode_normalize(:nfkc)
     mtime_string = time_to_database stat.mtime
-    digest, new_digest = get_file_digest(filename)
+    digest, new_digest = get_file_digest(perhaps_nt_filename)
 
     nested_transaction do
       new_digests_insert(normalized_filename, new_digest) if new_digest
       process_file_indeed normalized_filename, mtime_string, digest
     end
-
-  rescue => exception
-    @counters[:exceptions] += 1
-    print_file_exception exception, filename
   end
 
   def process_file_indeed filename, mtime, digest
@@ -394,7 +489,7 @@ class FileDigests
     else
       if found["mtime"] == mtime && !@options[:accept_fate] # Digest is different and mtime is the same
         @counters[:likely_damaged] += 1
-        STDERR.puts "LIKELY DAMAGED: #{filename}"
+        error_text "LIKELY DAMAGED: #{filename}"
       else
         @counters[:updated] += 1
         puts "UPDATED#{" (FATE ACCEPTED)" if found["mtime"] == mtime && @options[:accept_fate]}: #{filename}" unless @options[:quiet]
@@ -424,10 +519,20 @@ class FileDigests
     @counters[:renamed] = @db.changes
   end
 
-  def print_missing_files
+  def report_missing_files
     puts "\nMISSING FILES:"
+    write_missing_files STDOUT
+    if missing_files_count > 256
+      File.open(@missing_files_path, "a") do |f|
+        write_missing_files f
+      end
+      puts "\n(A list of missing files is also saved in a file: #{@missing_files_path})"
+    end
+  end
+
+  def write_missing_files dest
     missing_files_select_all_filenames.each do |record|
-      puts record["filename"]
+      dest.puts record["filename"]
     end
   end
 
@@ -445,6 +550,17 @@ class FileDigests
   end
 
 
+  # Runtime state helpers
+
+  def any_exceptions?
+    @counters[:exceptions] > 0
+  end
+
+  def any_likely_damaged?
+    @counters[:likely_damaged] > 0
+  end
+
+
   # Database helpers
 
   def execute *args, &block
@@ -485,7 +601,10 @@ class FileDigests
   def prepare_method name, query
     variable = "@#{name}"
 
-    instance_variable_set(variable, @db.prepare(query))
+    statement = @db.prepare(query)
+    @statements.push(statement)
+
+    instance_variable_set(variable, statement)
 
     define_singleton_method name do |*args, &block|
       instance_variable_get(variable).execute(*args, &block)
@@ -530,7 +649,7 @@ class FileDigests
   def check_if_database_is_at_certain_version target_version
     current_version = get_metadata("database_version")
     if current_version != target_version
-      STDERR.puts "This version of file-digests (#{FileDigests::VERSION || "unknown"}) is only compartible with the database version #{target_version}. Current database version is #{current_version}. To use this database, please install appropriate version if file-digest."
+      STDERR.puts "ERROR: This version of file-digests (#{FileDigests::VERSION || "unknown"}) is only compartible with the database version #{target_version}. Current database version is #{current_version}. To use this database, please install appropriate version if file-digest."
       raise "Incompatible database version"
     end
   end
@@ -540,58 +659,6 @@ class FileDigests
   end
 
 
-  # Filesystem-related helpers
-
-  def patch_path_string path
-    Gem.win_platform? ? path.gsub(/\\/, "/") : path
-  end
-
-  def cleanup_path path
-    Pathname.new(patch_path_string(path)).cleanpath
-  end
-
-  def ensure_dir_exist path
-    if File.exist?(path)
-      unless File.directory?(path)
-        raise "#{path} is not a directory"
-      end
-    else
-      FileUtils.mkdir_p path
-    end
-  end
-
-  def walk_files
-    puts "Gathering the list of files..." if @options[:verbose]
-    Dir.glob(@files_path + "**" + "*", File::FNM_DOTMATCH) do |filename|
-      yield filename
-    end
-  end
-
-  def get_file_digest filename
-    File.open(filename, "rb") do |io|
-      digest = OpenSSL::Digest.new(@digest_algorithm)
-      new_digest = OpenSSL::Digest.new(@new_digest_algorithm) if @new_digest_algorithm
-
-      buffer = ""
-      while io.read(409600, buffer) # 409600 seems like a sweet spot
-        digest.update(buffer)
-        new_digest.update(buffer) if @new_digest_algorithm
-      end
-      return [digest.hexdigest, (new_digest.hexdigest if @new_digest_algorithm)]
-    end
-  end
-
-
-  # Runtime state helpers
-
-  def any_exceptions?
-    @counters[:exceptions] > 0
-  end
-
-  def any_likely_damaged?
-    @counters[:likely_damaged] > 0
-  end
-
   # UI helpers
 
   def confirm text
@@ -611,17 +678,31 @@ class FileDigests
     puts "Elapsed time: #{elapsed.to_i / 3600}h #{(elapsed.to_i % 3600) / 60}m #{"%.3f" % (elapsed % 60)}s" unless @options[:quiet]
   end
 
-  def print_file_exception exception, filename
-    STDERR.print "EXCEPTION: #{exception.message}, processing file: "
+  def report_file_exception exception, filename
+    write_file_exception STDERR, exception, filename
+    File.open(@error_log_path, "a") do |f|
+      write_file_exception f, exception, filename
+    end
+  end
+
+  def write_file_exception dest, exception, filename
+    dest.print "ERROR: #{exception.message}, processing file: "
     begin
-      STDERR.print filename.encode("utf-8", universal_newline: true)
+      dest.print filename.encode("utf-8", universal_newline: true)
     rescue
-      STDERR.print "(Unable to encode file name to utf-8) "
-      STDERR.print filename
+      dest.print "(Unable to encode file name to utf-8) "
+      dest.print filename
+    end
+    dest.print "\n"
+    dest.flush
+    exception.backtrace.each { |line| dest.puts "  " + line }
+  end
+
+  def error_text text
+    STDERR.puts text
+    File.open(@error_log_path, "a") do |f|
+      f.puts text
     end
-    STDERR.print "\n"
-    STDERR.flush
-    exception.backtrace.each { |line| STDERR.puts "  " + line }
   end
 
   def print_counters

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: file-digests
 version: !ruby/object:Gem::Version
-  version: 0.0.37
+  version: 0.0.42
 platform: ruby
 authors:
 - Stanislav Senotrusov
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-10-17 00:00:00.000000000 Z
+date: 2021-04-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: openssl
@@ -51,7 +51,7 @@ homepage: https://github.com/senotrusov/file-digests
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -66,8 +66,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key: 
+rubygems_version: 3.1.4
+signing_key:
 specification_version: 4
 summary: file-digests
 test_files: []