file-digests 0.0.23 → 0.0.28

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a2d9f3f948e0853b515442c1b480326373309320cb91eb6aa68a5844b0d2be4e
-  data.tar.gz: 804b6f40e5475286a6cee0ea49adb646e3fd7367e2fc73d4a105b45137ce747a
+  metadata.gz: 406b02c22923ae98c45dc92f2836a99dffa6dc8b2343ff62b5b5957a4a154bdc
+  data.tar.gz: 5898cbc3826818da8c3fa5cf16a334bcb4627f95dc2635a297cc9c7d81d25dfe
 SHA512:
-  metadata.gz: f785ab8fe3d91bee0a59bc33a46d6c1790f1095d9ea54eb2b1a8aafe43189ec1e7c6c1353312db547d36d21aed871f7bb985638499b20719f17112fecc8d8ac0
-  data.tar.gz: bafc97b9a8e37b3dc4bcd3bfdf534849f3e9aa33e3a17c17da30044c51bf254f53694643b3fb9be94d4bac8800177549d454012ad2371d7e68f7d78b36e61fb1
+  metadata.gz: '021494ba0a65daada30e55c63e489c60c018ae3b189409be200635948132f0ed5118d217388e81d8097bc991cf10fea2ce09c4a3a7c148c1fdaa8f66e6b8e074'
+  data.tar.gz: 3a28808aa979157a2597b5ea56ad006576d28f35678cbaa6e13d62be20aa37066a9d4684d48f5ed2990b24c42acb2ee8cd2d0f14f4f9b12f893c5b3946d5b76e

data/lib/file-digests.rb

@@ -1,19 +1,21 @@
-require 'date'
-require 'digest'
-require 'fileutils'
-require 'openssl'
-require 'optparse'
-require 'pathname'
-require 'set'
-require 'sqlite3'
+require "date"
+require "digest"
+require "fileutils"
+require "openssl"
+require "optparse"
+require "pathname"
+require "set"
+require "sqlite3"
 
 class FileDigests
   DIGEST_ALGORITHMS=["BLAKE2b512", "SHA3-256", "SHA512-256"]
+  LEGACY_DIGEST_ALGORITHMS = ["SHA512", "SHA256"]
 
   def self.canonical_digest_algorithm_name(string)
     if string
-      index = DIGEST_ALGORITHMS.map(&:downcase).index(string.downcase)
-      index && DIGEST_ALGORITHMS[index]
+      algorithms = DIGEST_ALGORITHMS + LEGACY_DIGEST_ALGORITHMS
+      index = algorithms.map(&:downcase).index(string.downcase)
+      index && algorithms[index]
     end
   end
 
@@ -27,55 +29,60 @@ class FileDigests
 
   def self.parse_cli_options
     options = {}
-    
+
     OptionParser.new do |opts|
       opts.banner = [
         "Usage: file-digests [options] [path/to/directory] [path/to/database_file]",
         "       By default the current directory will be operated upon, and the database file will be placed to the current directory as well.",
         "       Should you wish to check current directory but place the database elsewhere, you could provide \".\" as a first argument, and the path to a database_file as a second."
       ].join "\n"
-    
-      opts.on("-a", "--auto", "Do not ask for any confirmation") do
+
+      opts.on("-a", "--auto", "Do not ask for any confirmation.") do
         options[:auto] = true
       end
 
       opts.on(
-        '--digest=DIGEST',
+        "-d", "--digest DIGEST",
         'Select a digest algorithm to use. Default is "BLAKE2b512".',
         'You might also consider to use slower "SHA512-256" or even more slower "SHA3-256".',
         "#{digest_algorithms_list_text}.",
-        'You only need to specify an algorithm on the first run, your choice will be saved to a database.',
-        'Any time later you could specify a new algorithm to change the current one.',
-        'Transition to a new algorithm will only occur if all files pass the check by digests which were stored using the old one.'
+        "You only need to specify an algorithm on the first run, your choice will be saved to a database.",
+        "Any time later you could specify a new algorithm to change the current one.",
+        "Transition to a new algorithm will only occur if all files pass the check by digests which were stored using the old one."
       ) do |value|
         digest_algorithm = canonical_digest_algorithm_name(value)
-        unless digest_algorithm
-          STDERR.puts "ERROR: #{digest_algorithms_list_text}" 
+        unless DIGEST_ALGORITHMS.include?(digest_algorithm)
+          STDERR.puts "ERROR: #{digest_algorithms_list_text}"
           exit 1
         end
         options[:digest_algorithm] = digest_algorithm
       end
 
-      opts.on("-d", "--duplicates", "Show the list of duplicate files, based on the information out of the database") do
-        options[:action] = :show_duplicates
+      opts.on("-f", "--accept-fate", "Accept the current state of files that are likely damaged and update their digest data.") do
+        options[:accept_fate] = true
       end
 
-      opts.on("-t", "--test", "Perform only the test, do not modify the digest database") do
-        options[:test_only] = true
+      opts.on("-h", "--help", "Prints this help.") do
+        puts opts
+        exit
+      end
+
+      opts.on("-p", "--duplicates", "Show the list of duplicate files, based on the information out of the database.") do
+        options[:action] = :show_duplicates
       end
 
-      opts.on("-q", "--quiet", "Less verbose output, stil report any found issues") do
+      opts.on("-q", "--quiet", "Less verbose output, stil report any found issues.") do
         options[:quiet] = true
       end
 
-      opts.on("-v", "--verbose", "More verbose output") do
-        options[:verbose] = true
+      opts.on("-t", "--test", "Perform only the test, do not modify the digest database.") do
+        options[:test_only] = true
       end
 
-      opts.on("-h", "--help", "Prints this help") do
-        puts opts
-        exit
+      opts.on("-v", "--verbose", "More verbose output.") do
+        options[:verbose] = true
       end
+
     end.parse!
     options
   end
@@ -93,15 +100,20 @@ class FileDigests
     initialize_paths files_path, digest_database_path
     initialize_database
 
-    if @digest_algorithm = canonical_digest_algorithm_name(get_metadata("digest_algorithm"))
-      if @options[:digest_algorithm] && @options[:digest_algorithm] != @digest_algorithm
-        @new_digest_algorithm = @options[:digest_algorithm]
+    @db.transaction(:exclusive) do
+      if db_digest_algorithm = get_metadata("digest_algorithm")
+        if @digest_algorithm = canonical_digest_algorithm_name(db_digest_algorithm)
+          if @options[:digest_algorithm] && @options[:digest_algorithm] != @digest_algorithm
+            @new_digest_algorithm = @options[:digest_algorithm]
+          end
+        else
+          raise "Database contains data for unsupported digest algorithm: #{db_digest_algorithm}"
+        end
+      else
+        @digest_algorithm = (@options[:digest_algorithm] || "BLAKE2b512")
+        set_metadata "digest_algorithm", @digest_algorithm
       end
-    else
-      @digest_algorithm = (@options[:digest_algorithm] || "BLAKE2b512")
-      set_metadata "digest_algorithm", @digest_algorithm
     end
-
     puts "Using #{@digest_algorithm} digest algorithm" if @options[:verbose]
   end
 
@@ -111,7 +123,7 @@ class FileDigests
     raise "Files path must be a readable directory" unless (File.directory?(@files_path) && File.readable?(@files_path))
 
     @digest_database_path = digest_database_path ? cleanup_path(digest_database_path) : @files_path
-    @digest_database_path += '.file-digests.sqlite' if File.directory?(@digest_database_path)
+    @digest_database_path += ".file-digests.sqlite" if File.directory?(@digest_database_path)
     ensure_dir_exists @digest_database_path.dirname
 
     if @options[:verbose]
@@ -123,14 +135,17 @@ class FileDigests
   def initialize_database
     @db = SQLite3::Database.new @digest_database_path.to_s
     @db.results_as_hash = true
+    @db.busy_timeout = 5000
 
     file_digests_gem_version = Gem.loaded_specs["file-digests"]&.version&.to_s
 
-    execute 'PRAGMA encoding = "UTF-8"'
-    execute 'PRAGMA journal_mode = "WAL"'
-    execute 'PRAGMA synchronous = "NORMAL"'
-    execute 'PRAGMA locking_mode = "EXCLUSIVE"'
-    execute 'PRAGMA cache_size = "5000"'
+    execute "PRAGMA encoding = 'UTF-8'"
+    execute "PRAGMA locking_mode = 'EXCLUSIVE'"
+    execute "PRAGMA journal_mode = 'WAL'"
+    execute "PRAGMA synchronous = 'NORMAL'"
+    execute "PRAGMA cache_size = '5000'"
+
+    integrity_check
 
     @db.transaction(:exclusive) do
       metadata_table_was_created = false
@@ -181,7 +196,7 @@ class FileDigests
       # Convert database from 1st to 2nd version
       unless get_metadata("digest_algorithm")
         if get_metadata("database_version") == "1"
-          if File.exist?(@digest_database_path.dirname + '.file-digests.sha512')
+          if File.exist?(@digest_database_path.dirname + ".file-digests.sha512")
             set_metadata("digest_algorithm", "SHA512")
           else
             set_metadata("digest_algorithm", "SHA256")
@@ -190,6 +205,10 @@ class FileDigests
         end
       end
 
+      if get_metadata("database_version") != "2"
+        STDERR.puts "This version of file-digests (#{file_digests_gem_version || "unknown"}) is only compartible with the database version 2. Current database version is #{get_metadata("database_version")}. To use this database, please install appropriate version if file-digest."
+        raise "Incompatible database version"
+      end
     end
   end
 
@@ -207,15 +226,19 @@ class FileDigests
         end
       end
 
-      track_renames
-      
+      nested_transaction do
+        track_renames
+      end
+
       if any_missing_files?
         if any_exceptions?
           STDERR.puts "Due to previously occurred errors, database cleanup from missing files will be skipped this time."
         else
           print_missing_files
           if !@options[:test_only] && (@options[:auto] || confirm("Remove missing files from the database"))
-            remove_missing_files
+            nested_transaction do
+              remove_missing_files
+            end
           end
         end
       end
@@ -228,6 +251,7 @@ class FileDigests
             update_digest_to_new_digest new_digest, old_digest
           end
           set_metadata "digest_algorithm", @new_digest_algorithm
+          puts "Transition to a new digest algorithm complete: #{@new_digest_algorithm}"
         end
       end
 
@@ -237,6 +261,10 @@ class FileDigests
 
       set_metadata(@options[:test_only] ? "latest_test_only_check_time" : "latest_complete_check_time", time_to_database(Time.now))
 
+      execute "PRAGMA optimize"
+      execute "VACUUM"
+      execute "PRAGMA wal_checkpoint(TRUNCATE)"
+
       print_counters
     end
   end
@@ -244,12 +272,12 @@ class FileDigests
   def show_duplicates
     current_digest = nil
     query_duplicates.each do |found|
-      if current_digest != found['digest']
+      if current_digest != found["digest"]
         puts "" if current_digest
-        current_digest = found['digest']
-        puts "#{found['digest']}:"
+        current_digest = found["digest"]
+        puts "#{found["digest"]}:"
       end
-      puts "  #{found['filename']}"
+      puts "  #{found["filename"]}"
     end
   end
 
@@ -275,10 +303,13 @@ class FileDigests
       return
     end
 
-    normalized_filename = filename.delete_prefix("#{@files_path.to_s}/").encode('utf-8', universal_newline: true).unicode_normalize(:nfkc)
+    normalized_filename = filename.delete_prefix("#{@files_path.to_s}/").encode("utf-8", universal_newline: true).unicode_normalize(:nfkc)
     mtime_string = time_to_database stat.mtime
+    digest = get_file_digest(filename)
 
-    process_file_indeed normalized_filename, mtime_string, get_file_digest(filename)
+    nested_transaction do
+      process_file_indeed normalized_filename, mtime_string, digest
+    end
 
   rescue => exception
     @counters[:exceptions] += 1
@@ -295,25 +326,25 @@ class FileDigests
 
   def process_previously_seen_file found, filename, mtime, digest
     @missing_files.delete(filename)
-    if found['digest'] == digest
+    if found["digest"] == digest
       @counters[:good] += 1
       puts "GOOD: #{filename}" if @options[:verbose]
       unless @options[:test_only]
-        if found['mtime'] == mtime
-          touch_digest_check_time found['id']
+        if found["mtime"] == mtime
+          touch_digest_check_time found["id"]
         else
-          update_mtime mtime, found['id']
+          update_mtime mtime, found["id"]
         end
       end
     else
-      if found['mtime'] == mtime # Digest is different and mtime is the same
+      if found["mtime"] == mtime && !@options[:accept_fate] # Digest is different and mtime is the same
         @counters[:likely_damaged] += 1
         STDERR.puts "LIKELY DAMAGED: #{filename}"
       else
         @counters[:updated] += 1
-        puts "UPDATED: #{filename}" unless @options[:quiet]
+        puts "UPDATED#{" (FATE ACCEPTED)" if found["mtime"] == mtime && @options[:accept_fate]}: #{filename}" unless @options[:quiet]
         unless @options[:test_only]
-          update_mtime_and_digest mtime, digest, found['id']
+          update_mtime_and_digest mtime, digest, found["id"]
         end
       end
     end
@@ -351,12 +382,10 @@ class FileDigests
   end
 
   def remove_missing_files
-    nested_transaction do
-      @missing_files.each do |filename, digest|
-        delete_by_filename filename
-      end
-      @missing_files = {}
+    @missing_files.each do |filename, digest|
+      delete_by_filename filename
     end
+    @missing_files = {}
   end
 
 
@@ -366,7 +395,13 @@ class FileDigests
     @db.execute *args, &block
   end
 
-  def nested_transaction(mode)
+  def integrity_check
+    if execute("PRAGMA integrity_check")&.first&.fetch("integrity_check") != "ok"
+      raise "Database integrity check failed"
+    end
+  end
+
+  def nested_transaction(mode = :deferred)
     if @db.transaction_active?
       yield
     else
@@ -376,9 +411,9 @@ class FileDigests
     end
   end
 
-  def perhaps_transaction(condition, mode)
+  def perhaps_transaction(condition, mode = :deferred)
     if condition
-      @db.transaction(mode) do
+      nested_transaction(mode) do
         yield
       end
     else
@@ -387,7 +422,7 @@ class FileDigests
   end
 
   def table_exist? table_name
-    execute("SELECT name FROM sqlite_master WHERE type='table' AND name = '#{table_name}'").length == 1
+    execute("SELECT name FROM sqlite_master WHERE type='table' AND name = ?", table_name).length == 1
   end
 
   def prepare_method name, query
@@ -422,14 +457,14 @@ class FileDigests
   end
 
   def time_to_database time
-    time.utc.strftime('%Y-%m-%d %H:%M:%S')
+    time.utc.strftime("%Y-%m-%d %H:%M:%S")
   end
 
 
   # Filesystem-related helpers
 
   def patch_path_string path
-    Gem.win_platform? ? path.gsub(/\\/, '/') : path
+    Gem.win_platform? ? path.gsub(/\\/, "/") : path
   end
 
   def cleanup_path path
@@ -447,13 +482,13 @@ class FileDigests
   end
 
   def walk_files
-    Dir.glob(@files_path + '**' + '*', File::FNM_DOTMATCH) do |filename|
+    Dir.glob(@files_path + "**" + "*", File::FNM_DOTMATCH) do |filename|
       yield filename
     end
   end
 
   def get_file_digest filename
-    File.open(filename, 'rb') do |io|
+    File.open(filename, "rb") do |io|
       digest = OpenSSL::Digest.new(@digest_algorithm)
       new_digest = OpenSSL::Digest.new(@new_digest_algorithm) if @new_digest_algorithm
 
@@ -495,13 +530,13 @@ class FileDigests
     start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
     yield
     elapsed = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start)
-    puts "Elapsed time: #{elapsed.to_i / 3600}h #{(elapsed.to_i % 3600) / 60}m #{'%.3f' % (elapsed % 60)}s" unless @options[:quiet]
+    puts "Elapsed time: #{elapsed.to_i / 3600}h #{(elapsed.to_i % 3600) / 60}m #{"%.3f" % (elapsed % 60)}s" unless @options[:quiet]
   end
 
   def print_file_exception exception, filename
     STDERR.print "EXCEPTION: #{exception.message}, processing file: "
     begin
-      STDERR.print filename.encode('utf-8', universal_newline: true)
+      STDERR.print filename.encode("utf-8", universal_newline: true)
     rescue
       STDERR.print "(Unable to encode file name to utf-8) "
       STDERR.print filename

metadata

@@ -1,43 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: file-digests
 version: !ruby/object:Gem::Version
-  version: 0.0.23
+  version: 0.0.28
 platform: ruby
 authors:
 - Stanislav Senotrusov
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-10-12 00:00:00.000000000 Z
+date: 2020-10-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: sqlite3
+  name: openssl
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: '2.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: '2.1'
 - !ruby/object:Gem::Dependency
-  name: openssl
+  name: sqlite3
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.1.0
+        version: '1.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.1.0
+        version: '1.3'
 description: Calculate file digests and check for the possible file corruption
 email: stan@senotrusov.com
 executables: