file-digests 0.0.22 → 0.0.27

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f08714e86e275eb74108da01c667d403134366e6d7e9c5e8b08278d8c8842ba8
-  data.tar.gz: 57802538edae099807ede5460a07d6fd7ea6a5d0810f40d306db8f330435631d
+  metadata.gz: 63e300c17abcf4035c957c9e9c45b8d677b2f47172919efd758467ff4da7f51e
+  data.tar.gz: dbee998de8f9957d8b69a4afbbca54ca39ce8ca2cd4d9ee743998ee7bdd5f3c2
 SHA512:
-  metadata.gz: 4c642c5aaf06d903114aabc65ec13ae3c5eb04c807b167c7eb1f3c63e8f56144ff3d93964fdc55cbcb33fad743cad635cfa61284171c94ed3b7a9a42c3efbca6
-  data.tar.gz: eebb6b444c6d0921f638200a27f97af5e9026ab0cf21b7b43b3e58f6f13ccc6323f482f93594ae993637a18b7611514469848307defe3796f412dd453c08b932
+  metadata.gz: 66e5d0eb877617acf92b6c7bdada2c77a262d1484933dc44b7e3df548a3fd58fb0a0aa4460c368aaac785360375a650badeb148253d919a77d9882daa5b31201
+  data.tar.gz: 4444e166dbe2d71ac240cebf69992c57f846648c046696c89f575e174b7b92e3be92256d85ed8538f80581877a6e5e5e23a30ffc47a0d141762abc5d09a67e4b

data/lib/file-digests.rb

@@ -1,19 +1,21 @@
-require 'date'
-require 'digest'
-require 'fileutils'
-require 'openssl'
-require 'optparse'
-require 'pathname'
-require 'set'
-require 'sqlite3'
+require "date"
+require "digest"
+require "fileutils"
+require "openssl"
+require "optparse"
+require "pathname"
+require "set"
+require "sqlite3"
 
 class FileDigests
   DIGEST_ALGORITHMS=["BLAKE2b512", "SHA3-256", "SHA512-256"]
+  LEGACY_DIGEST_ALGORITHMS = ["SHA512", "SHA256"]
 
   def self.canonical_digest_algorithm_name(string)
     if string
-      index = DIGEST_ALGORITHMS.map(&:downcase).index(string.downcase)
-      index && DIGEST_ALGORITHMS[index]
+      algorithms = DIGEST_ALGORITHMS + LEGACY_DIGEST_ALGORITHMS
+      index = algorithms.map(&:downcase).index(string.downcase)
+      index && algorithms[index]
     end
   end
 
@@ -27,55 +29,60 @@ class FileDigests
 
   def self.parse_cli_options
     options = {}
-    
+
     OptionParser.new do |opts|
       opts.banner = [
         "Usage: file-digests [options] [path/to/directory] [path/to/database_file]",
         "       By default the current directory will be operated upon, and the database file will be placed to the current directory as well.",
         "       Should you wish to check current directory but place the database elsewhere, you could provide \".\" as a first argument, and the path to a database_file as a second."
       ].join "\n"
-    
-      opts.on("-a", "--auto", "Do not ask for any confirmation") do
+
+      opts.on("-a", "--auto", "Do not ask for any confirmation.") do
         options[:auto] = true
       end
 
       opts.on(
-        '--digest=DIGEST',
+        "-d", "--digest DIGEST",
         'Select a digest algorithm to use. Default is "BLAKE2b512".',
         'You might also consider to use slower "SHA512-256" or even more slower "SHA3-256".',
         "#{digest_algorithms_list_text}.",
-        'You only need to specify an algorithm on the first run, your choice will be saved to a database.',
-        'Any time later you could specify a new algorithm to change the current one.',
-        'Transition to a new algorithm will only occur if all files pass the check by digests which were stored using the old one.'
+        "You only need to specify an algorithm on the first run, your choice will be saved to a database.",
+        "Any time later you could specify a new algorithm to change the current one.",
+        "Transition to a new algorithm will only occur if all files pass the check by digests which were stored using the old one."
       ) do |value|
         digest_algorithm = canonical_digest_algorithm_name(value)
-        unless digest_algorithm
-          STDERR.puts "ERROR: #{digest_algorithms_list_text}" 
+        unless DIGEST_ALGORITHMS.include?(digest_algorithm)
+          STDERR.puts "ERROR: #{digest_algorithms_list_text}"
           exit 1
         end
         options[:digest_algorithm] = digest_algorithm
       end
 
-      opts.on("-d", "--duplicates", "Show the list of duplicate files, based on the information out of the database") do
-        options[:action] = :show_duplicates
+      opts.on("-f", "--accept-fate", "Accept the current state of files that are likely damaged and update their digest data.") do
+        options[:accept_fate] = true
       end
 
-      opts.on("-t", "--test", "Perform only the test, do not modify the digest database") do
-        options[:test_only] = true
+      opts.on("-h", "--help", "Prints this help.") do
+        puts opts
+        exit
+      end
+
+      opts.on("-p", "--duplicates", "Show the list of duplicate files, based on the information out of the database.") do
+        options[:action] = :show_duplicates
       end
 
-      opts.on("-q", "--quiet", "Less verbose output, stil report any found issues") do
+      opts.on("-q", "--quiet", "Less verbose output, stil report any found issues.") do
         options[:quiet] = true
       end
 
-      opts.on("-v", "--verbose", "More verbose output") do
-        options[:verbose] = true
+      opts.on("-t", "--test", "Perform only the test, do not modify the digest database.") do
+        options[:test_only] = true
       end
 
-      opts.on("-h", "--help", "Prints this help") do
-        puts opts
-        exit
+      opts.on("-v", "--verbose", "More verbose output.") do
+        options[:verbose] = true
       end
+
     end.parse!
     options
   end
@@ -93,13 +100,15 @@ class FileDigests
     initialize_paths files_path, digest_database_path
     initialize_database
 
-    if @digest_algorithm = canonical_digest_algorithm_name(get_metadata("digest_algorithm"))
-      if @options[:digest_algorithm] && @options[:digest_algorithm] != @digest_algorithm
-        @new_digest_algorithm = @options[:digest_algorithm]
+    @db.transaction(:exclusive) do
+      if @digest_algorithm = canonical_digest_algorithm_name(get_metadata("digest_algorithm"))
+        if @options[:digest_algorithm] && @options[:digest_algorithm] != @digest_algorithm
+          @new_digest_algorithm = @options[:digest_algorithm]
+        end
+      else
+        @digest_algorithm = (@options[:digest_algorithm] || "BLAKE2b512")
+        set_metadata "digest_algorithm", @digest_algorithm
       end
-    else
-      @digest_algorithm = (@options[:digest_algorithm] || "BLAKE2b512")
-      set_metadata "digest_algorithm", @digest_algorithm
     end
 
     puts "Using #{@digest_algorithm} digest algorithm" if @options[:verbose]
@@ -111,7 +120,7 @@ class FileDigests
     raise "Files path must be a readable directory" unless (File.directory?(@files_path) && File.readable?(@files_path))
 
     @digest_database_path = digest_database_path ? cleanup_path(digest_database_path) : @files_path
-    @digest_database_path += '.file-digests.sqlite' if File.directory?(@digest_database_path)
+    @digest_database_path += ".file-digests.sqlite" if File.directory?(@digest_database_path)
     ensure_dir_exists @digest_database_path.dirname
 
     if @options[:verbose]
@@ -123,27 +132,33 @@ class FileDigests
   def initialize_database
     @db = SQLite3::Database.new @digest_database_path.to_s
     @db.results_as_hash = true
+    @db.busy_timeout = 5000
 
     file_digests_gem_version = Gem.loaded_specs["file-digests"]&.version&.to_s
 
-    execute 'PRAGMA encoding = "UTF-8"'
-    execute 'PRAGMA journal_mode = "WAL"'
-    execute 'PRAGMA synchronous = "NORMAL"'
-    execute 'PRAGMA locking_mode = "EXCLUSIVE"'
-    execute 'PRAGMA cache_size = "5000"'
+    execute "PRAGMA encoding = 'UTF-8'"
+    execute "PRAGMA locking_mode = 'EXCLUSIVE'"
+    execute "PRAGMA journal_mode = 'WAL'"
+    execute "PRAGMA synchronous = 'NORMAL'"
+    execute "PRAGMA cache_size = '5000'"
+
+    integrity_check
 
     @db.transaction(:exclusive) do
+      metadata_table_was_created = false
       unless table_exist?("metadata")
         execute "CREATE TABLE metadata (
           key TEXT NOT NULL PRIMARY KEY,
           value TEXT)"
         execute "CREATE UNIQUE INDEX metadata_key ON metadata(key)"
-        set_metadata("metadata_table_created_by_gem_version", file_digests_gem_version) if file_digests_gem_version
+        metadata_table_was_created = true
       end
 
       prepare_method :set_metadata_query, "INSERT INTO metadata (key, value) VALUES (?, ?) ON CONFLICT (key) DO UPDATE SET value=excluded.value"
       prepare_method :get_metadata_query, "SELECT value FROM metadata WHERE key = ?"
 
+      set_metadata("metadata_table_created_by_gem_version", file_digests_gem_version) if file_digests_gem_version && metadata_table_was_created
+
       # Heuristic to detect database version 1 (metadata was not stored back then)
       unless get_metadata("database_version")
         if table_exist?("digests")
@@ -178,7 +193,7 @@ class FileDigests
       # Convert database from 1st to 2nd version
       unless get_metadata("digest_algorithm")
         if get_metadata("database_version") == "1"
-          if File.exist?(@digest_database_path.dirname + '.file-digests.sha512')
+          if File.exist?(@digest_database_path.dirname + ".file-digests.sha512")
             set_metadata("digest_algorithm", "SHA512")
           else
             set_metadata("digest_algorithm", "SHA256")
@@ -187,6 +202,10 @@ class FileDigests
         end
       end
 
+      if get_metadata("database_version") != "2"
+        STDERR.puts "This version of file-digests (#{file_digests_gem_version || "unknown"}) is only compartible with the database version 2. Current database version is #{get_metadata("database_version")}. To use this database, please install appropriate version if file-digest."
+        raise "Incompatible database version"
+      end
     end
   end
 
@@ -204,15 +223,19 @@ class FileDigests
         end
       end
 
-      track_renames
-      
+      nested_transaction do
+        track_renames
+      end
+
       if any_missing_files?
         if any_exceptions?
           STDERR.puts "Due to previously occurred errors, database cleanup from missing files will be skipped this time."
         else
           print_missing_files
           if !@options[:test_only] && (@options[:auto] || confirm("Remove missing files from the database"))
-            remove_missing_files
+            nested_transaction do
+              remove_missing_files
+            end
           end
         end
       end
@@ -225,6 +248,7 @@ class FileDigests
             update_digest_to_new_digest new_digest, old_digest
           end
           set_metadata "digest_algorithm", @new_digest_algorithm
+          puts "Transition to a new digest algorithm complete: #{@new_digest_algorithm}"
         end
       end
 
@@ -234,6 +258,10 @@ class FileDigests
 
       set_metadata(@options[:test_only] ? "latest_test_only_check_time" : "latest_complete_check_time", time_to_database(Time.now))
 
+      execute "PRAGMA optimize"
+      execute "VACUUM"
+      execute "PRAGMA wal_checkpoint(TRUNCATE)"
+
       print_counters
     end
   end
@@ -241,12 +269,12 @@ class FileDigests
   def show_duplicates
     current_digest = nil
     query_duplicates.each do |found|
-      if current_digest != found['digest']
+      if current_digest != found["digest"]
         puts "" if current_digest
-        current_digest = found['digest']
-        puts "#{found['digest']}:"
+        current_digest = found["digest"]
+        puts "#{found["digest"]}:"
       end
-      puts "  #{found['filename']}"
+      puts "  #{found["filename"]}"
     end
   end
 
@@ -272,10 +300,13 @@ class FileDigests
       return
     end
 
-    normalized_filename = filename.delete_prefix("#{@files_path.to_s}/").encode('utf-8', universal_newline: true).unicode_normalize(:nfkc)
+    normalized_filename = filename.delete_prefix("#{@files_path.to_s}/").encode("utf-8", universal_newline: true).unicode_normalize(:nfkc)
     mtime_string = time_to_database stat.mtime
+    digest = get_file_digest(filename)
 
-    process_file_indeed normalized_filename, mtime_string, get_file_digest(filename)
+    nested_transaction do
+      process_file_indeed normalized_filename, mtime_string, digest
+    end
 
   rescue => exception
     @counters[:exceptions] += 1
@@ -292,25 +323,25 @@ class FileDigests
 
   def process_previously_seen_file found, filename, mtime, digest
     @missing_files.delete(filename)
-    if found['digest'] == digest
+    if found["digest"] == digest
       @counters[:good] += 1
       puts "GOOD: #{filename}" if @options[:verbose]
       unless @options[:test_only]
-        if found['mtime'] == mtime
-          touch_digest_check_time found['id']
+        if found["mtime"] == mtime
+          touch_digest_check_time found["id"]
         else
-          update_mtime mtime, found['id']
+          update_mtime mtime, found["id"]
         end
       end
     else
-      if found['mtime'] == mtime # Digest is different and mtime is the same
+      if found["mtime"] == mtime && !@options[:accept_fate] # Digest is different and mtime is the same
         @counters[:likely_damaged] += 1
         STDERR.puts "LIKELY DAMAGED: #{filename}"
       else
         @counters[:updated] += 1
-        puts "UPDATED: #{filename}" unless @options[:quiet]
+        puts "UPDATED#{" (FATE ACCEPTED)" if found["mtime"] == mtime && @options[:accept_fate]}: #{filename}" unless @options[:quiet]
         unless @options[:test_only]
-          update_mtime_and_digest mtime, digest, found['id']
+          update_mtime_and_digest mtime, digest, found["id"]
         end
       end
     end
@@ -348,12 +379,10 @@ class FileDigests
   end
 
   def remove_missing_files
-    nested_transaction do
-      @missing_files.each do |filename, digest|
-        delete_by_filename filename
-      end
-      @missing_files = {}
+    @missing_files.each do |filename, digest|
+      delete_by_filename filename
     end
+    @missing_files = {}
   end
 
 
@@ -363,7 +392,13 @@ class FileDigests
     @db.execute *args, &block
   end
 
-  def nested_transaction(mode)
+  def integrity_check
+    if execute("PRAGMA integrity_check")&.first&.fetch("integrity_check") != "ok"
+      raise "Database integrity check failed"
+    end
+  end
+
+  def nested_transaction(mode = :deferred)
     if @db.transaction_active?
       yield
     else
@@ -373,9 +408,9 @@ class FileDigests
     end
   end
 
-  def perhaps_transaction(condition, mode)
+  def perhaps_transaction(condition, mode = :deferred)
     if condition
-      @db.transaction(mode) do
+      nested_transaction(mode) do
         yield
       end
     else
@@ -384,7 +419,7 @@ class FileDigests
   end
 
   def table_exist? table_name
-    execute("SELECT name FROM sqlite_master WHERE type='table' AND name = '#{table_name}'").length == 1
+    execute("SELECT name FROM sqlite_master WHERE type='table' AND name = ?", table_name).length == 1
   end
 
   def prepare_method name, query
@@ -419,14 +454,14 @@ class FileDigests
   end
 
   def time_to_database time
-    time.utc.strftime('%Y-%m-%d %H:%M:%S')
+    time.utc.strftime("%Y-%m-%d %H:%M:%S")
   end
 
 
   # Filesystem-related helpers
 
   def patch_path_string path
-    Gem.win_platform? ? path.gsub(/\\/, '/') : path
+    Gem.win_platform? ? path.gsub(/\\/, "/") : path
   end
 
   def cleanup_path path
@@ -444,13 +479,13 @@ class FileDigests
   end
 
   def walk_files
-    Dir.glob(@files_path + '**' + '*', File::FNM_DOTMATCH) do |filename|
+    Dir.glob(@files_path + "**" + "*", File::FNM_DOTMATCH) do |filename|
       yield filename
     end
   end
 
   def get_file_digest filename
-    File.open(filename, 'rb') do |io|
+    File.open(filename, "rb") do |io|
       digest = OpenSSL::Digest.new(@digest_algorithm)
       new_digest = OpenSSL::Digest.new(@new_digest_algorithm) if @new_digest_algorithm
 
@@ -492,13 +527,13 @@ class FileDigests
     start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
     yield
     elapsed = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start)
-    puts "Elapsed time: #{elapsed.to_i / 3600}h #{(elapsed.to_i % 3600) / 60}m #{'%.3f' % (elapsed % 60)}s" unless @options[:quiet]
+    puts "Elapsed time: #{elapsed.to_i / 3600}h #{(elapsed.to_i % 3600) / 60}m #{"%.3f" % (elapsed % 60)}s" unless @options[:quiet]
   end
 
   def print_file_exception exception, filename
     STDERR.print "EXCEPTION: #{exception.message}, processing file: "
     begin
-      STDERR.print filename.encode('utf-8', universal_newline: true)
+      STDERR.print filename.encode("utf-8", universal_newline: true)
     rescue
       STDERR.print "(Unable to encode file name to utf-8) "
       STDERR.print filename

metadata

@@ -1,43 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: file-digests
 version: !ruby/object:Gem::Version
-  version: 0.0.22
+  version: 0.0.27
 platform: ruby
 authors:
 - Stanislav Senotrusov
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-10-12 00:00:00.000000000 Z
+date: 2020-10-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: sqlite3
+  name: openssl
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: '2.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 1.3.0
+        version: '2.1'
 - !ruby/object:Gem::Dependency
-  name: openssl
+  name: sqlite3
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.1.0
+        version: '1.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.1.0
+        version: '1.3'
 description: Calculate file digests and check for the possible file corruption
 email: stan@senotrusov.com
 executables: