file-digests 0.0.26 → 0.0.27

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 912b66387453e26ebd7280d7f2dbd599838098a6bcc4da70d1eabe19737ba172
-  data.tar.gz: 9c02352223aff8d6489892df533606bd512de3357dfb59a82513dc04aa1eddfe
+  metadata.gz: 63e300c17abcf4035c957c9e9c45b8d677b2f47172919efd758467ff4da7f51e
+  data.tar.gz: dbee998de8f9957d8b69a4afbbca54ca39ce8ca2cd4d9ee743998ee7bdd5f3c2
 SHA512:
-  metadata.gz: '0380c5acf3750632a3fe49f63d93dbc7533da1389f3e8cab41addc8909f71552a280aec7e3e783e9ae20071bc94af3d70021098b1fbfba96a3e1f78967479c34'
-  data.tar.gz: 4b0a05a27dcb61ee405dc06688753d5eb162e3fdf513fa45aee9a1e2d946fd4697b4f2dbb45f1af57f33ee787c366665144a999c161976cd81992902bfe90fb5
+  metadata.gz: 66e5d0eb877617acf92b6c7bdada2c77a262d1484933dc44b7e3df548a3fd58fb0a0aa4460c368aaac785360375a650badeb148253d919a77d9882daa5b31201
+  data.tar.gz: 4444e166dbe2d71ac240cebf69992c57f846648c046696c89f575e174b7b92e3be92256d85ed8538f80581877a6e5e5e23a30ffc47a0d141762abc5d09a67e4b

data/lib/file-digests.rb

@@ -1,11 +1,11 @@
-require 'date'
-require 'digest'
-require 'fileutils'
-require 'openssl'
-require 'optparse'
-require 'pathname'
-require 'set'
-require 'sqlite3'
+require "date"
+require "digest"
+require "fileutils"
+require "openssl"
+require "optparse"
+require "pathname"
+require "set"
+require "sqlite3"
 
 class FileDigests
   DIGEST_ALGORITHMS=["BLAKE2b512", "SHA3-256", "SHA512-256"]
@@ -37,22 +37,18 @@ class FileDigests
         "       Should you wish to check current directory but place the database elsewhere, you could provide \".\" as a first argument, and the path to a database_file as a second."
       ].join "\n"
 
-      opts.on("-a", "--auto", "Do not ask for any confirmation") do
+      opts.on("-a", "--auto", "Do not ask for any confirmation.") do
         options[:auto] = true
       end
 
-      opts.on("--accept-fate", "Accept the current state of files that are likely damaged and update their digest data") do
-        options[:accept_fate] = true
-      end
-
       opts.on(
-        '--digest=DIGEST',
+        "-d", "--digest DIGEST",
         'Select a digest algorithm to use. Default is "BLAKE2b512".',
         'You might also consider to use slower "SHA512-256" or even more slower "SHA3-256".',
         "#{digest_algorithms_list_text}.",
-        'You only need to specify an algorithm on the first run, your choice will be saved to a database.',
-        'Any time later you could specify a new algorithm to change the current one.',
-        'Transition to a new algorithm will only occur if all files pass the check by digests which were stored using the old one.'
+        "You only need to specify an algorithm on the first run, your choice will be saved to a database.",
+        "Any time later you could specify a new algorithm to change the current one.",
+        "Transition to a new algorithm will only occur if all files pass the check by digests which were stored using the old one."
       ) do |value|
         digest_algorithm = canonical_digest_algorithm_name(value)
         unless DIGEST_ALGORITHMS.include?(digest_algorithm)
@@ -62,26 +58,31 @@ class FileDigests
         options[:digest_algorithm] = digest_algorithm
       end
 
-      opts.on("-d", "--duplicates", "Show the list of duplicate files, based on the information out of the database") do
-        options[:action] = :show_duplicates
+      opts.on("-f", "--accept-fate", "Accept the current state of files that are likely damaged and update their digest data.") do
+        options[:accept_fate] = true
       end
 
-      opts.on("-t", "--test", "Perform only the test, do not modify the digest database") do
-        options[:test_only] = true
+      opts.on("-h", "--help", "Prints this help.") do
+        puts opts
+        exit
       end
 
-      opts.on("-q", "--quiet", "Less verbose output, stil report any found issues") do
+      opts.on("-p", "--duplicates", "Show the list of duplicate files, based on the information out of the database.") do
+        options[:action] = :show_duplicates
+      end
+
+      opts.on("-q", "--quiet", "Less verbose output, stil report any found issues.") do
         options[:quiet] = true
       end
 
-      opts.on("-v", "--verbose", "More verbose output") do
-        options[:verbose] = true
+      opts.on("-t", "--test", "Perform only the test, do not modify the digest database.") do
+        options[:test_only] = true
       end
 
-      opts.on("-h", "--help", "Prints this help") do
-        puts opts
-        exit
+      opts.on("-v", "--verbose", "More verbose output.") do
+        options[:verbose] = true
       end
+
     end.parse!
     options
   end
@@ -99,13 +100,15 @@ class FileDigests
     initialize_paths files_path, digest_database_path
     initialize_database
 
-    if @digest_algorithm = canonical_digest_algorithm_name(get_metadata("digest_algorithm"))
-      if @options[:digest_algorithm] && @options[:digest_algorithm] != @digest_algorithm
-        @new_digest_algorithm = @options[:digest_algorithm]
+    @db.transaction(:exclusive) do
+      if @digest_algorithm = canonical_digest_algorithm_name(get_metadata("digest_algorithm"))
+        if @options[:digest_algorithm] && @options[:digest_algorithm] != @digest_algorithm
+          @new_digest_algorithm = @options[:digest_algorithm]
+        end
+      else
+        @digest_algorithm = (@options[:digest_algorithm] || "BLAKE2b512")
+        set_metadata "digest_algorithm", @digest_algorithm
       end
-    else
-      @digest_algorithm = (@options[:digest_algorithm] || "BLAKE2b512")
-      set_metadata "digest_algorithm", @digest_algorithm
     end
 
     puts "Using #{@digest_algorithm} digest algorithm" if @options[:verbose]
@@ -117,7 +120,7 @@ class FileDigests
     raise "Files path must be a readable directory" unless (File.directory?(@files_path) && File.readable?(@files_path))
 
     @digest_database_path = digest_database_path ? cleanup_path(digest_database_path) : @files_path
-    @digest_database_path += '.file-digests.sqlite' if File.directory?(@digest_database_path)
+    @digest_database_path += ".file-digests.sqlite" if File.directory?(@digest_database_path)
     ensure_dir_exists @digest_database_path.dirname
 
     if @options[:verbose]
@@ -129,14 +132,17 @@ class FileDigests
   def initialize_database
     @db = SQLite3::Database.new @digest_database_path.to_s
     @db.results_as_hash = true
+    @db.busy_timeout = 5000
 
     file_digests_gem_version = Gem.loaded_specs["file-digests"]&.version&.to_s
 
-    execute 'PRAGMA encoding = "UTF-8"'
-    execute 'PRAGMA journal_mode = "WAL"'
-    execute 'PRAGMA synchronous = "NORMAL"'
-    execute 'PRAGMA locking_mode = "EXCLUSIVE"'
-    execute 'PRAGMA cache_size = "5000"'
+    execute "PRAGMA encoding = 'UTF-8'"
+    execute "PRAGMA locking_mode = 'EXCLUSIVE'"
+    execute "PRAGMA journal_mode = 'WAL'"
+    execute "PRAGMA synchronous = 'NORMAL'"
+    execute "PRAGMA cache_size = '5000'"
+
+    integrity_check
 
     @db.transaction(:exclusive) do
       metadata_table_was_created = false
@@ -187,7 +193,7 @@ class FileDigests
       # Convert database from 1st to 2nd version
       unless get_metadata("digest_algorithm")
         if get_metadata("database_version") == "1"
-          if File.exist?(@digest_database_path.dirname + '.file-digests.sha512')
+          if File.exist?(@digest_database_path.dirname + ".file-digests.sha512")
             set_metadata("digest_algorithm", "SHA512")
           else
             set_metadata("digest_algorithm", "SHA256")
@@ -197,7 +203,7 @@ class FileDigests
       end
 
       if get_metadata("database_version") != "2"
-        STDERR.puts "This version of file-digests (#{file_digests_gem_version || 'unknown'}) is only compartible with the database version 2. Current database version is #{get_metadata("database_version")}. To use this database, please install appropriate version if file-digest."
+        STDERR.puts "This version of file-digests (#{file_digests_gem_version || "unknown"}) is only compartible with the database version 2. Current database version is #{get_metadata("database_version")}. To use this database, please install appropriate version if file-digest."
         raise "Incompatible database version"
       end
     end
@@ -217,7 +223,9 @@ class FileDigests
         end
       end
 
-      track_renames
+      nested_transaction do
+        track_renames
+      end
 
       if any_missing_files?
         if any_exceptions?
@@ -225,7 +233,9 @@ class FileDigests
         else
           print_missing_files
           if !@options[:test_only] && (@options[:auto] || confirm("Remove missing files from the database"))
-            remove_missing_files
+            nested_transaction do
+              remove_missing_files
+            end
           end
         end
       end
@@ -248,6 +258,10 @@ class FileDigests
 
       set_metadata(@options[:test_only] ? "latest_test_only_check_time" : "latest_complete_check_time", time_to_database(Time.now))
 
+      execute "PRAGMA optimize"
+      execute "VACUUM"
+      execute "PRAGMA wal_checkpoint(TRUNCATE)"
+
       print_counters
     end
   end
@@ -255,12 +269,12 @@ class FileDigests
   def show_duplicates
     current_digest = nil
     query_duplicates.each do |found|
-      if current_digest != found['digest']
+      if current_digest != found["digest"]
         puts "" if current_digest
-        current_digest = found['digest']
-        puts "#{found['digest']}:"
+        current_digest = found["digest"]
+        puts "#{found["digest"]}:"
       end
-      puts "  #{found['filename']}"
+      puts "  #{found["filename"]}"
     end
   end
 
@@ -286,10 +300,13 @@ class FileDigests
       return
     end
 
-    normalized_filename = filename.delete_prefix("#{@files_path.to_s}/").encode('utf-8', universal_newline: true).unicode_normalize(:nfkc)
+    normalized_filename = filename.delete_prefix("#{@files_path.to_s}/").encode("utf-8", universal_newline: true).unicode_normalize(:nfkc)
     mtime_string = time_to_database stat.mtime
+    digest = get_file_digest(filename)
 
-    process_file_indeed normalized_filename, mtime_string, get_file_digest(filename)
+    nested_transaction do
+      process_file_indeed normalized_filename, mtime_string, digest
+    end
 
   rescue => exception
     @counters[:exceptions] += 1
@@ -306,25 +323,25 @@ class FileDigests
 
   def process_previously_seen_file found, filename, mtime, digest
     @missing_files.delete(filename)
-    if found['digest'] == digest
+    if found["digest"] == digest
       @counters[:good] += 1
       puts "GOOD: #{filename}" if @options[:verbose]
       unless @options[:test_only]
-        if found['mtime'] == mtime
-          touch_digest_check_time found['id']
+        if found["mtime"] == mtime
+          touch_digest_check_time found["id"]
         else
-          update_mtime mtime, found['id']
+          update_mtime mtime, found["id"]
         end
       end
     else
-      if found['mtime'] == mtime && !@options[:accept_fate] # Digest is different and mtime is the same
+      if found["mtime"] == mtime && !@options[:accept_fate] # Digest is different and mtime is the same
         @counters[:likely_damaged] += 1
         STDERR.puts "LIKELY DAMAGED: #{filename}"
       else
         @counters[:updated] += 1
-        puts "UPDATED: #{filename}" unless @options[:quiet]
+        puts "UPDATED#{" (FATE ACCEPTED)" if found["mtime"] == mtime && @options[:accept_fate]}: #{filename}" unless @options[:quiet]
         unless @options[:test_only]
-          update_mtime_and_digest mtime, digest, found['id']
+          update_mtime_and_digest mtime, digest, found["id"]
         end
       end
     end
@@ -362,12 +379,10 @@ class FileDigests
   end
 
   def remove_missing_files
-    nested_transaction do
-      @missing_files.each do |filename, digest|
-        delete_by_filename filename
-      end
-      @missing_files = {}
+    @missing_files.each do |filename, digest|
+      delete_by_filename filename
     end
+    @missing_files = {}
   end
 
 
@@ -377,6 +392,12 @@ class FileDigests
     @db.execute *args, &block
   end
 
+  def integrity_check
+    if execute("PRAGMA integrity_check")&.first&.fetch("integrity_check") != "ok"
+      raise "Database integrity check failed"
+    end
+  end
+
   def nested_transaction(mode = :deferred)
     if @db.transaction_active?
       yield
@@ -398,7 +419,7 @@ class FileDigests
   end
 
   def table_exist? table_name
-    execute("SELECT name FROM sqlite_master WHERE type='table' AND name = '#{table_name}'").length == 1
+    execute("SELECT name FROM sqlite_master WHERE type='table' AND name = ?", table_name).length == 1
   end
 
   def prepare_method name, query
@@ -433,14 +454,14 @@ class FileDigests
   end
 
   def time_to_database time
-    time.utc.strftime('%Y-%m-%d %H:%M:%S')
+    time.utc.strftime("%Y-%m-%d %H:%M:%S")
   end
 
 
   # Filesystem-related helpers
 
   def patch_path_string path
-    Gem.win_platform? ? path.gsub(/\\/, '/') : path
+    Gem.win_platform? ? path.gsub(/\\/, "/") : path
   end
 
   def cleanup_path path
@@ -458,13 +479,13 @@ class FileDigests
   end
 
   def walk_files
-    Dir.glob(@files_path + '**' + '*', File::FNM_DOTMATCH) do |filename|
+    Dir.glob(@files_path + "**" + "*", File::FNM_DOTMATCH) do |filename|
       yield filename
     end
   end
 
   def get_file_digest filename
-    File.open(filename, 'rb') do |io|
+    File.open(filename, "rb") do |io|
       digest = OpenSSL::Digest.new(@digest_algorithm)
       new_digest = OpenSSL::Digest.new(@new_digest_algorithm) if @new_digest_algorithm
 
@@ -506,13 +527,13 @@ class FileDigests
     start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
     yield
     elapsed = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start)
-    puts "Elapsed time: #{elapsed.to_i / 3600}h #{(elapsed.to_i % 3600) / 60}m #{'%.3f' % (elapsed % 60)}s" unless @options[:quiet]
+    puts "Elapsed time: #{elapsed.to_i / 3600}h #{(elapsed.to_i % 3600) / 60}m #{"%.3f" % (elapsed % 60)}s" unless @options[:quiet]
   end
 
   def print_file_exception exception, filename
     STDERR.print "EXCEPTION: #{exception.message}, processing file: "
     begin
-      STDERR.print filename.encode('utf-8', universal_newline: true)
+      STDERR.print filename.encode("utf-8", universal_newline: true)
     rescue
       STDERR.print "(Unable to encode file name to utf-8) "
       STDERR.print filename

metadata

@@ -1,43 +1,43 @@
 --- !ruby/object:Gem::Specification
 name: file-digests
 version: !ruby/object:Gem::Version
-  version: 0.0.26
+  version: 0.0.27
 platform: ruby
 authors:
 - Stanislav Senotrusov
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-10-12 00:00:00.000000000 Z
+date: 2020-10-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: sqlite3
+  name: openssl
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '2.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '2.1'
 - !ruby/object:Gem::Dependency
-  name: openssl
+  name: sqlite3
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: '1.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: '1.3'
 description: Calculate file digests and check for the possible file corruption
 email: stan@senotrusov.com
 executables: