file-digests 0.0.24 → 0.0.29

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1816d453ddc057c7a8eef4a0c81f63699f9ba10fdee404488fa8dc945bac791d
-  data.tar.gz: ae5baf350ac64577f0807b81231f045d875eff6351c78e559df46379ad8d6b1e
+  metadata.gz: cab5bc80dc7949984501c3068bf99f44f82564774733e84b1cd6810fd0cb6a05
+  data.tar.gz: 97a4220b83a08408345b21a60256e55bbbe0a5fb30e6c97efa05a4faac42d356
 SHA512:
-  metadata.gz: d76b3e25d709e17e7260c0ab7836f016107c7335461f7d8ab6a74b391067fbfb1691abd6286fa3894aa432b9af229ba417972ba08c5d1524dd06d06c8dffe653
-  data.tar.gz: 9e39f4ae5d19268c986fac6d6d3bc47e2be66bb60857be608c033670ccd4b7ba3f0c5a8242a18e9ebaf161b8f5b230c040f931405e32a5a53d840e250c46fe9f
+  metadata.gz: 5eaae0823ccb95bb3db6245e0863ff37aaa6f14f599bd92d040c8085e753aaa8d3be7903d1039a433bded9011bdc73c6a0d42e3e10d1e2c52eb81b6b791a5bc1
+  data.tar.gz: 97f7fa2ac8605fc1775570accf44a93c73e67a11699b7e98bfa9687f9e2c0219ad1890c1a293f52ca901079e602f6cf6863b5f57de174dd39e36ec338d6597f7

data/lib/file-digests.rb

@@ -1,19 +1,21 @@
-require 'date'
-require 'digest'
-require 'fileutils'
-require 'openssl'
-require 'optparse'
-require 'pathname'
-require 'set'
-require 'sqlite3'
+require "date"
+require "digest"
+require "fileutils"
+require "openssl"
+require "optparse"
+require "pathname"
+require "set"
+require "sqlite3"
 
 class FileDigests
   DIGEST_ALGORITHMS=["BLAKE2b512", "SHA3-256", "SHA512-256"]
+  LEGACY_DIGEST_ALGORITHMS = ["SHA512", "SHA256"]
 
   def self.canonical_digest_algorithm_name(string)
     if string
-      index = DIGEST_ALGORITHMS.map(&:downcase).index(string.downcase)
-      index && DIGEST_ALGORITHMS[index]
+      algorithms = DIGEST_ALGORITHMS + LEGACY_DIGEST_ALGORITHMS
+      index = algorithms.map(&:downcase).index(string.downcase)
+      index && algorithms[index]
     end
   end
 
@@ -27,55 +29,60 @@ class FileDigests
 
   def self.parse_cli_options
     options = {}
-    
+
     OptionParser.new do |opts|
       opts.banner = [
         "Usage: file-digests [options] [path/to/directory] [path/to/database_file]",
         "       By default the current directory will be operated upon, and the database file will be placed to the current directory as well.",
         "       Should you wish to check current directory but place the database elsewhere, you could provide \".\" as a first argument, and the path to a database_file as a second."
       ].join "\n"
-    
-      opts.on("-a", "--auto", "Do not ask for any confirmation") do
+
+      opts.on("-a", "--auto", "Do not ask for any confirmation.") do
         options[:auto] = true
       end
 
       opts.on(
-        '--digest=DIGEST',
+        "-d", "--digest DIGEST",
         'Select a digest algorithm to use. Default is "BLAKE2b512".',
         'You might also consider to use slower "SHA512-256" or even more slower "SHA3-256".',
         "#{digest_algorithms_list_text}.",
-        'You only need to specify an algorithm on the first run, your choice will be saved to a database.',
-        'Any time later you could specify a new algorithm to change the current one.',
-        'Transition to a new algorithm will only occur if all files pass the check by digests which were stored using the old one.'
+        "You only need to specify an algorithm on the first run, your choice will be saved to a database.",
+        "Any time later you could specify a new algorithm to change the current one.",
+        "Transition to a new algorithm will only occur if all files pass the check by digests which were stored using the old one."
       ) do |value|
         digest_algorithm = canonical_digest_algorithm_name(value)
-        unless digest_algorithm
-          STDERR.puts "ERROR: #{digest_algorithms_list_text}" 
+        unless DIGEST_ALGORITHMS.include?(digest_algorithm)
+          STDERR.puts "ERROR: #{digest_algorithms_list_text}"
           exit 1
         end
         options[:digest_algorithm] = digest_algorithm
       end
 
-      opts.on("-d", "--duplicates", "Show the list of duplicate files, based on the information out of the database") do
-        options[:action] = :show_duplicates
+      opts.on("-f", "--accept-fate", "Accept the current state of files that are likely damaged and update their digest data.") do
+        options[:accept_fate] = true
       end
 
-      opts.on("-t", "--test", "Perform only the test, do not modify the digest database") do
-        options[:test_only] = true
+      opts.on("-h", "--help", "Prints this help.") do
+        puts opts
+        exit
       end
 
-      opts.on("-q", "--quiet", "Less verbose output, stil report any found issues") do
+      opts.on("-p", "--duplicates", "Show the list of duplicate files, based on the information out of the database.") do
+        options[:action] = :show_duplicates
+      end
+
+      opts.on("-q", "--quiet", "Less verbose output, stil report any found issues.") do
         options[:quiet] = true
       end
 
-      opts.on("-v", "--verbose", "More verbose output") do
-        options[:verbose] = true
+      opts.on("-t", "--test", "Perform only the test, do not modify the digest database.") do
+        options[:test_only] = true
       end
 
-      opts.on("-h", "--help", "Prints this help") do
-        puts opts
-        exit
+      opts.on("-v", "--verbose", "More verbose output.") do
+        options[:verbose] = true
       end
+
     end.parse!
     options
   end
@@ -93,15 +100,20 @@ class FileDigests
     initialize_paths files_path, digest_database_path
     initialize_database
 
-    if @digest_algorithm = canonical_digest_algorithm_name(get_metadata("digest_algorithm"))
-      if @options[:digest_algorithm] && @options[:digest_algorithm] != @digest_algorithm
-        @new_digest_algorithm = @options[:digest_algorithm]
+    @db.transaction(:exclusive) do
+      if db_digest_algorithm = get_metadata("digest_algorithm")
+        if @digest_algorithm = canonical_digest_algorithm_name(db_digest_algorithm)
+          if @options[:digest_algorithm] && @options[:digest_algorithm] != @digest_algorithm
+            @new_digest_algorithm = @options[:digest_algorithm]
+          end
+        else
+          raise "Database contains data for unsupported digest algorithm: #{db_digest_algorithm}"
+        end
+      else
+        @digest_algorithm = (@options[:digest_algorithm] || "BLAKE2b512")
+        set_metadata "digest_algorithm", @digest_algorithm
       end
-    else
-      @digest_algorithm = (@options[:digest_algorithm] || "BLAKE2b512")
-      set_metadata "digest_algorithm", @digest_algorithm
     end
-
     puts "Using #{@digest_algorithm} digest algorithm" if @options[:verbose]
   end
 
@@ -111,8 +123,10 @@ class FileDigests
     raise "Files path must be a readable directory" unless (File.directory?(@files_path) && File.readable?(@files_path))
 
     @digest_database_path = digest_database_path ? cleanup_path(digest_database_path) : @files_path
-    @digest_database_path += '.file-digests.sqlite' if File.directory?(@digest_database_path)
-    ensure_dir_exists @digest_database_path.dirname
+    @digest_database_path += ".file-digests.sqlite" if File.directory?(@digest_database_path)
+    ensure_dir_exist @digest_database_path.dirname
+
+    @digest_database_files = ["#{@digest_database_path}", "#{@digest_database_path}-wal", "#{@digest_database_path}-shm"]
 
     if @options[:verbose]
       puts "Target directory: #{@files_path}"
@@ -123,14 +137,17 @@ class FileDigests
   def initialize_database
     @db = SQLite3::Database.new @digest_database_path.to_s
     @db.results_as_hash = true
+    @db.busy_timeout = 5000
 
     file_digests_gem_version = Gem.loaded_specs["file-digests"]&.version&.to_s
 
-    execute 'PRAGMA encoding = "UTF-8"'
-    execute 'PRAGMA journal_mode = "WAL"'
-    execute 'PRAGMA synchronous = "NORMAL"'
-    execute 'PRAGMA locking_mode = "EXCLUSIVE"'
-    execute 'PRAGMA cache_size = "5000"'
+    execute "PRAGMA encoding = 'UTF-8'"
+    execute "PRAGMA locking_mode = 'EXCLUSIVE'"
+    execute "PRAGMA journal_mode = 'WAL'"
+    execute "PRAGMA synchronous = 'NORMAL'"
+    execute "PRAGMA cache_size = '5000'"
+
+    integrity_check
 
     @db.transaction(:exclusive) do
       metadata_table_was_created = false
@@ -181,7 +198,7 @@ class FileDigests
       # Convert database from 1st to 2nd version
       unless get_metadata("digest_algorithm")
         if get_metadata("database_version") == "1"
-          if File.exist?(@digest_database_path.dirname + '.file-digests.sha512')
+          if File.exist?(@digest_database_path.dirname + ".file-digests.sha512")
             set_metadata("digest_algorithm", "SHA512")
           else
             set_metadata("digest_algorithm", "SHA256")
@@ -191,7 +208,7 @@ class FileDigests
       end
 
       if get_metadata("database_version") != "2"
-        STDERR.puts "This version of file-digests is only compartible with the database version 2. Current database version is #{get_metadata("database_version")}. To use this database, please install appropriate version if file-digest."
+        STDERR.puts "This version of file-digests (#{file_digests_gem_version || "unknown"}) is only compartible with the database version 2. Current database version is #{get_metadata("database_version")}. To use this database, please install appropriate version if file-digest."
         raise "Incompatible database version"
       end
     end
@@ -211,15 +228,19 @@ class FileDigests
         end
       end
 
-      track_renames
-      
+      nested_transaction do
+        track_renames
+      end
+
       if any_missing_files?
         if any_exceptions?
           STDERR.puts "Due to previously occurred errors, database cleanup from missing files will be skipped this time."
         else
           print_missing_files
           if !@options[:test_only] && (@options[:auto] || confirm("Remove missing files from the database"))
-            remove_missing_files
+            nested_transaction do
+              remove_missing_files
+            end
           end
         end
       end
@@ -232,6 +253,7 @@ class FileDigests
             update_digest_to_new_digest new_digest, old_digest
           end
           set_metadata "digest_algorithm", @new_digest_algorithm
+          puts "Transition to a new digest algorithm complete: #{@new_digest_algorithm}"
         end
       end
 
@@ -241,6 +263,12 @@ class FileDigests
 
       set_metadata(@options[:test_only] ? "latest_test_only_check_time" : "latest_complete_check_time", time_to_database(Time.now))
 
+      execute "PRAGMA optimize"
+      execute "VACUUM"
+      execute "PRAGMA wal_checkpoint(TRUNCATE)"
+
+      hide_database_files
+
       print_counters
     end
   end
@@ -248,12 +276,12 @@ class FileDigests
   def show_duplicates
     current_digest = nil
     query_duplicates.each do |found|
-      if current_digest != found['digest']
+      if current_digest != found["digest"]
         puts "" if current_digest
-        current_digest = found['digest']
-        puts "#{found['digest']}:"
+        current_digest = found["digest"]
+        puts "#{found["digest"]}:"
       end
-      puts "  #{found['filename']}"
+      puts "  #{found["filename"]}"
     end
   end
 
@@ -272,17 +300,18 @@ class FileDigests
 
     raise "File is not readable" unless stat.readable?
 
-    if filename == "#{@digest_database_path}" ||
-       filename == "#{@digest_database_path}-wal" ||
-       filename == "#{@digest_database_path}-shm"
+    if @digest_database_files.include?(filename)
       puts "SKIPPING DATABASE FILE: #{filename}" if @options[:verbose]
       return
     end
 
-    normalized_filename = filename.delete_prefix("#{@files_path.to_s}/").encode('utf-8', universal_newline: true).unicode_normalize(:nfkc)
+    normalized_filename = filename.delete_prefix("#{@files_path.to_s}/").encode("utf-8", universal_newline: true).unicode_normalize(:nfkc)
     mtime_string = time_to_database stat.mtime
+    digest = get_file_digest(filename)
 
-    process_file_indeed normalized_filename, mtime_string, get_file_digest(filename)
+    nested_transaction do
+      process_file_indeed normalized_filename, mtime_string, digest
+    end
 
   rescue => exception
     @counters[:exceptions] += 1
@@ -299,25 +328,25 @@ class FileDigests
 
   def process_previously_seen_file found, filename, mtime, digest
     @missing_files.delete(filename)
-    if found['digest'] == digest
+    if found["digest"] == digest
       @counters[:good] += 1
       puts "GOOD: #{filename}" if @options[:verbose]
       unless @options[:test_only]
-        if found['mtime'] == mtime
-          touch_digest_check_time found['id']
+        if found["mtime"] == mtime
+          touch_digest_check_time found["id"]
         else
-          update_mtime mtime, found['id']
+          update_mtime mtime, found["id"]
         end
       end
     else
-      if found['mtime'] == mtime # Digest is different and mtime is the same
+      if found["mtime"] == mtime && !@options[:accept_fate] # Digest is different and mtime is the same
         @counters[:likely_damaged] += 1
         STDERR.puts "LIKELY DAMAGED: #{filename}"
       else
         @counters[:updated] += 1
-        puts "UPDATED: #{filename}" unless @options[:quiet]
+        puts "UPDATED#{" (FATE ACCEPTED)" if found["mtime"] == mtime && @options[:accept_fate]}: #{filename}" unless @options[:quiet]
         unless @options[:test_only]
-          update_mtime_and_digest mtime, digest, found['id']
+          update_mtime_and_digest mtime, digest, found["id"]
         end
       end
     end
@@ -355,12 +384,10 @@ class FileDigests
   end
 
   def remove_missing_files
-    nested_transaction do
-      @missing_files.each do |filename, digest|
-        delete_by_filename filename
-      end
-      @missing_files = {}
+    @missing_files.each do |filename, digest|
+      delete_by_filename filename
     end
+    @missing_files = {}
   end
 
 
@@ -370,7 +397,13 @@ class FileDigests
     @db.execute *args, &block
   end
 
-  def nested_transaction(mode)
+  def integrity_check
+    if execute("PRAGMA integrity_check")&.first&.fetch("integrity_check") != "ok"
+      raise "Database integrity check failed"
+    end
+  end
+
+  def nested_transaction(mode = :deferred)
     if @db.transaction_active?
       yield
     else
@@ -380,9 +413,9 @@ class FileDigests
     end
   end
 
-  def perhaps_transaction(condition, mode)
+  def perhaps_transaction(condition, mode = :deferred)
     if condition
-      @db.transaction(mode) do
+      nested_transaction(mode) do
         yield
       end
     else
@@ -391,7 +424,7 @@ class FileDigests
   end
 
   def table_exist? table_name
-    execute("SELECT name FROM sqlite_master WHERE type='table' AND name = '#{table_name}'").length == 1
+    execute("SELECT name FROM sqlite_master WHERE type='table' AND name = ?", table_name).length == 1
   end
 
   def prepare_method name, query
@@ -426,21 +459,31 @@ class FileDigests
   end
 
   def time_to_database time
-    time.utc.strftime('%Y-%m-%d %H:%M:%S')
+    time.utc.strftime("%Y-%m-%d %H:%M:%S")
+  end
+
+  def hide_database_files
+    if Gem.win_platform?
+      @digest_database_files.each do |file|
+        if File.exist?(file)
+          system "attrib", "+H", file, exception: true
+        end
+      end
+    end
   end
 
 
   # Filesystem-related helpers
 
   def patch_path_string path
-    Gem.win_platform? ? path.gsub(/\\/, '/') : path
+    Gem.win_platform? ? path.gsub(/\\/, "/") : path
   end
 
   def cleanup_path path
     Pathname.new(patch_path_string(path)).cleanpath
   end
 
-  def ensure_dir_exists path
+  def ensure_dir_exist path
     if File.exist?(path)
       unless File.directory?(path)
         raise "#{path} is not a directory"
@@ -451,13 +494,13 @@ class FileDigests
   end
 
   def walk_files
-    Dir.glob(@files_path + '**' + '*', File::FNM_DOTMATCH) do |filename|
+    Dir.glob(@files_path + "**" + "*", File::FNM_DOTMATCH) do |filename|
       yield filename
     end
   end
 
   def get_file_digest filename
-    File.open(filename, 'rb') do |io|
+    File.open(filename, "rb") do |io|
       digest = OpenSSL::Digest.new(@digest_algorithm)
       new_digest = OpenSSL::Digest.new(@new_digest_algorithm) if @new_digest_algorithm
 
@@ -499,13 +542,13 @@ class FileDigests
     start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
     yield
     elapsed = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start)
-    puts "Elapsed time: #{elapsed.to_i / 3600}h #{(elapsed.to_i % 3600) / 60}m #{'%.3f' % (elapsed % 60)}s" unless @options[:quiet]
+    puts "Elapsed time: #{elapsed.to_i / 3600}h #{(elapsed.to_i % 3600) / 60}m #{"%.3f" % (elapsed % 60)}s" unless @options[:quiet]
   end
 
   def print_file_exception exception, filename
     STDERR.print "EXCEPTION: #{exception.message}, processing file: "
     begin
-      STDERR.print filename.encode('utf-8', universal_newline: true)
+      STDERR.print filename.encode("utf-8", universal_newline: true)
     rescue
       STDERR.print "(Unable to encode file name to utf-8) "
       STDERR.print filename

metadata

@@ -1,43 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: file-digests
 version: !ruby/object:Gem::Version
-  version: 0.0.24
+  version: 0.0.29
 platform: ruby
 authors:
 - Stanislav Senotrusov
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-10-12 00:00:00.000000000 Z
+date: 2020-10-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: sqlite3
+  name: openssl
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '2.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '2.1'
 - !ruby/object:Gem::Dependency
-  name: openssl
+  name: sqlite3
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: '1.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: '1.3'
 description: Calculate file digests and check for the possible file corruption
 email: stan@senotrusov.com
 executables: