file-digests 0.0.30 → 0.0.35

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 99b58aede8267994cc69da0ac1fd2e35d6661fd666257e9f3dfcde5054b3b6f8
-  data.tar.gz: 4b733dcf4be1f14b4a08931d15d0531c18e161f3d6d3af3e3025b36266f67e9d
+  metadata.gz: 42e870b2a0e4e711e84a2748d09d8a378fc6964d78706d573ab22737d0f2519e
+  data.tar.gz: e14bef4ccd63fdc07950a257654c6594b793eabf44bbdf26cae59e621838e5c3
 SHA512:
-  metadata.gz: 9cf4c3df2b8f206b54689f2506de474ca3b021c572869765caf6e1424dac73a7c6dca3b67918ba5cb88f712353febe9aebe5fc40f7da24f6dd4019300b052436
-  data.tar.gz: cb0904f141e8861c9923647214c92bb8ce022952c53e8a96cee9819a43f4192bf550b47c927cc0ea8f5a16f2c58c3013561555c2afde9c093dd6cd7997f25c94
+  metadata.gz: b6b9eb636258ba06b29bd21736b477c0d225ba119ad3b3ccb82612c941bf892109876aa4e8da7402f580f4167f84000a1b2b4ae91e568ac50f223325fab856ef
+  data.tar.gz: afcdffefe0dcb346afcdf37ce6f3f523d4862a45a455838d8b7cbf603cead2c8fadbac185640088c79bd7e8c412b5da932bf301d90dd4337171b675a65f34388

data/bin/file-digests

@@ -1,5 +1,19 @@
 #!/usr/bin/env ruby
 
+#  Copyright 2020 Stanislav Senotrusov <stan@senotrusov.com>
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
 require 'file-digests'
 
 FileDigests.run_cli_utility

data/lib/file-digests.rb

@@ -1,3 +1,17 @@
+#  Copyright 2020 Stanislav Senotrusov <stan@senotrusov.com>
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
 require "date"
 require "digest"
 require "fileutils"
@@ -8,7 +22,8 @@ require "set"
 require "sqlite3"
 
 class FileDigests
-  DIGEST_ALGORITHMS=["BLAKE2b512", "SHA3-256", "SHA512-256"]
+  VERSION = Gem.loaded_specs["file-digests"]&.version&.to_s
+  DIGEST_ALGORITHMS = ["BLAKE2b512", "SHA3-256", "SHA512-256"]
   LEGACY_DIGEST_ALGORITHMS = ["SHA512", "SHA256"]
 
   def self.canonical_digest_algorithm_name(string)
@@ -75,7 +90,12 @@ class FileDigests
         options[:quiet] = true
       end
 
-      opts.on("-t", "--test", "Perform only the test, do not modify the digest database.") do
+      opts.on(
+        "-t", "--test",
+        "Perform a test to verify directory contents.",
+        "Compare actual files with the stored digests, check if any files are missing.",
+        "Digest database will not be modified."
+      ) do
         options[:test_only] = true
       end
 
@@ -96,6 +116,7 @@ class FileDigests
 
   def initialize files_path, digest_database_path, options = {}
     @options = options
+    @user_input_wait_time = 0
 
     initialize_paths files_path, digest_database_path
     initialize_database
@@ -139,8 +160,6 @@ class FileDigests
     @db.results_as_hash = true
     @db.busy_timeout = 5000
 
-    file_digests_gem_version = Gem.loaded_specs["file-digests"]&.version&.to_s
-
     execute "PRAGMA encoding = 'UTF-8'"
     execute "PRAGMA locking_mode = 'EXCLUSIVE'"
     execute "PRAGMA journal_mode = 'WAL'"
@@ -155,14 +174,13 @@ class FileDigests
         execute "CREATE TABLE metadata (
           key TEXT NOT NULL PRIMARY KEY,
           value TEXT)"
-        execute "CREATE UNIQUE INDEX metadata_key ON metadata(key)"
         metadata_table_was_created = true
       end
 
       prepare_method :set_metadata_query, "INSERT INTO metadata (key, value) VALUES (?, ?) ON CONFLICT (key) DO UPDATE SET value=excluded.value"
       prepare_method :get_metadata_query, "SELECT value FROM metadata WHERE key = ?"
 
-      set_metadata("metadata_table_created_by_gem_version", file_digests_gem_version) if file_digests_gem_version && metadata_table_was_created
+      set_metadata("metadata_table_created_by_gem_version", FileDigests::VERSION) if FileDigests::VERSION && metadata_table_was_created
 
       # Heuristic to detect database version 1 (metadata was not stored back then)
       unless get_metadata("database_version")
@@ -179,20 +197,19 @@ class FileDigests
           digest TEXT NOT NULL,
           digest_check_time TEXT NOT NULL)"
         execute "CREATE UNIQUE INDEX digests_filename ON digests(filename)"
-        set_metadata("digests_table_created_by_gem_version", file_digests_gem_version) if file_digests_gem_version
+        execute "CREATE INDEX digests_digest ON digests(digest)"
+        set_metadata("digests_table_created_by_gem_version", FileDigests::VERSION) if FileDigests::VERSION
       end
 
-      prepare_method :insert, "INSERT INTO digests (filename, mtime, digest, digest_check_time) VALUES (?, ?, ?, datetime('now'))"
-      prepare_method :find_by_filename_query, "SELECT id, mtime, digest FROM digests WHERE filename = ?"
-      prepare_method :touch_digest_check_time, "UPDATE digests SET digest_check_time = datetime('now') WHERE id = ?"
-      prepare_method :update_mtime_and_digest, "UPDATE digests SET mtime = ?, digest = ?, digest_check_time = datetime('now') WHERE id = ?"
-      prepare_method :update_mtime, "UPDATE digests SET mtime = ?, digest_check_time = datetime('now') WHERE id = ?"
-      prepare_method :delete_by_filename, "DELETE FROM digests WHERE filename = ?"
-      prepare_method :query_duplicates, "SELECT digest, filename FROM digests WHERE digest IN (SELECT digest FROM digests GROUP BY digest HAVING count(*) > 1) ORDER BY digest, filename;"
-      prepare_method :update_digest_to_new_digest, "UPDATE digests SET digest = ? WHERE digest = ?"
+      prepare_method :digests_insert, "INSERT INTO digests (filename, mtime, digest, digest_check_time) VALUES (?, ?, ?, datetime('now'))"
+      prepare_method :digests_find_by_filename_query, "SELECT id, mtime, digest FROM digests WHERE filename = ?"
+      prepare_method :digests_touch_check_time, "UPDATE digests SET digest_check_time = datetime('now') WHERE id = ?"
+      prepare_method :digests_update_mtime_and_digest, "UPDATE digests SET mtime = ?, digest = ?, digest_check_time = datetime('now') WHERE id = ?"
+      prepare_method :digests_update_mtime, "UPDATE digests SET mtime = ?, digest_check_time = datetime('now') WHERE id = ?"
+      prepare_method :digests_select_duplicates, "SELECT digest, filename FROM digests WHERE digest IN (SELECT digest FROM digests GROUP BY digest HAVING count(*) > 1) ORDER BY digest, filename;"
 
       unless get_metadata("database_version")
-        set_metadata "database_version", "2"
+        set_metadata "database_version", "3"
       end
 
       # Convert database from 1st to 2nd version
@@ -207,74 +224,110 @@ class FileDigests
         end
       end
 
-      if get_metadata("database_version") != "2"
-        STDERR.puts "This version of file-digests (#{file_digests_gem_version || "unknown"}) is only compartible with the database version 2. Current database version is #{get_metadata("database_version")}. To use this database, please install appropriate version if file-digest."
-        raise "Incompatible database version"
+      if get_metadata("database_version") == "2"
+        execute "CREATE INDEX digests_digest ON digests(digest)"
+        set_metadata "database_version", "3"
       end
+  
+      check_if_database_is_at_certain_version "3"
+
+      create_temporary_tables
     end
   end
 
-  def perform_check
-    perhaps_transaction(@new_digest_algorithm, :exclusive) do
-      @counters = {good: 0, updated: 0, new: 0, renamed: 0, likely_damaged: 0, exceptions: 0}
-      @new_files = {}
-      @new_digests = {}
+  def create_temporary_tables
+    execute "CREATE TEMPORARY TABLE new_files (
+      filename TEXT NOT NULL PRIMARY KEY,
+      digest TEXT NOT NULL)"
+    execute "CREATE INDEX new_files_digest ON new_files(digest)"
+
+    prepare_method :new_files_insert, "INSERT INTO new_files (filename, digest) VALUES (?, ?)"
+    prepare_method :new_files_count_query, "SELECT count(*) FROM new_files"
+
+    execute "CREATE TEMPORARY TABLE missing_files (
+      filename TEXT NOT NULL PRIMARY KEY,
+      digest TEXT NOT NULL)"
+    execute "CREATE INDEX missing_files_digest ON missing_files(digest)"
+
+    execute "INSERT INTO missing_files (filename, digest) SELECT filename, digest FROM digests"
+
+    prepare_method :missing_files_delete, "DELETE FROM missing_files WHERE filename = ?"
+    prepare_method :missing_files_delete_renamed_files, "DELETE FROM missing_files WHERE digest IN (SELECT digest FROM new_files)"
+    prepare_method :missing_files_select_all_filenames, "SELECT filename FROM missing_files ORDER BY filename"
+    prepare_method :missing_files_delete_all, "DELETE FROM missing_files"
+    prepare_method :missing_files_count_query, "SELECT count(*) FROM missing_files"
+
+    prepare_method :digests_delete_renamed_files, "DELETE FROM digests WHERE filename IN (SELECT filename FROM missing_files WHERE digest IN (SELECT digest FROM new_files))"
+    prepare_method :digests_delete_all_missing_files, "DELETE FROM digests WHERE filename IN (SELECT filename FROM missing_files)"
+
+    execute "CREATE TEMPORARY TABLE new_digests (
+      filename TEXT NOT NULL PRIMARY KEY,
+      digest TEXT NOT NULL)"
+
+    prepare_method :new_digests_insert, "INSERT INTO new_digests (filename, digest) VALUES (?, ?)"
+    prepare_method :digests_update_digests_to_new_digests, "INSERT INTO digests (filename, digest, digest_check_time) SELECT filename, digest, false FROM new_digests WHERE true ON CONFLICT (filename) DO UPDATE SET digest=excluded.digest"
+  end
 
-      @missing_files = Hash[@db.prepare("SELECT filename, digest FROM digests").execute!]
+  def perform_check
+    measure_time do
+      perhaps_transaction(@new_digest_algorithm, :exclusive) do
+        @counters = {good: 0, updated: 0, renamed: 0, likely_damaged: 0, exceptions: 0}
 
-      measure_time do
         walk_files do |filename|
           process_file filename
         end
-      end
 
-      nested_transaction do
-        track_renames
-      end
+        nested_transaction do
+          puts "Tracking renames..." if @options[:verbose]
+          track_renames
+        end
 
-      if any_missing_files?
-        if any_exceptions?
-          STDERR.puts "Due to previously occurred errors, database cleanup from missing files will be skipped this time."
-        else
-          print_missing_files
-          if !@options[:test_only] && (@options[:auto] || confirm("Remove missing files from the database"))
-            nested_transaction do
-              remove_missing_files
+        if any_missing_files?
+          if any_exceptions?
+            STDERR.puts "Due to previously occurred errors, missing files will not removed from the database."
+          else
+            print_missing_files
+            if !@options[:test_only] && (@options[:auto] || confirm("Remove missing files from the database"))
+              nested_transaction do
+                puts "Removing missing files..." if @options[:verbose]
+                remove_missing_files
+              end
             end
           end
         end
-      end
 
-      if @new_digest_algorithm && !@options[:test_only]
-        if any_missing_files? || any_likely_damaged? || any_exceptions?
-          STDERR.puts "ERROR: New digest algorithm will not be in effect until there are files that are missing, likely damaged, or processed with an exception."
-        else
-          @new_digests.each do |old_digest, new_digest|
-            update_digest_to_new_digest new_digest, old_digest
+        if @new_digest_algorithm && !@options[:test_only]
+          if any_missing_files? || any_likely_damaged? || any_exceptions?
+            STDERR.puts "ERROR: New digest algorithm will not be in effect until there are files that are missing, likely damaged, or processed with an exception."
+          else
+            puts "Updating database to a new digest algorithm..." if @options[:verbose]
+            digests_update_digests_to_new_digests
+            set_metadata "digest_algorithm", @new_digest_algorithm
+            puts "Transition to a new digest algorithm complete: #{@new_digest_algorithm}"
           end
-          set_metadata "digest_algorithm", @new_digest_algorithm
-          puts "Transition to a new digest algorithm complete: #{@new_digest_algorithm}"
         end
-      end
 
-      if any_likely_damaged? || any_exceptions?
-        STDERR.puts "PLEASE REVIEW ERRORS THAT WERE OCCURRED!"
-      end
+        if any_likely_damaged? || any_exceptions?
+          STDERR.puts "PLEASE REVIEW ERRORS THAT WERE OCCURRED!"
+        end
 
-      set_metadata(@options[:test_only] ? "latest_test_only_check_time" : "latest_complete_check_time", time_to_database(Time.now))
+        set_metadata(@options[:test_only] ? "latest_test_only_check_time" : "latest_complete_check_time", time_to_database(Time.now))
 
-      print_counters
-    end
-    execute "PRAGMA optimize"
-    execute "VACUUM"
-    execute "PRAGMA wal_checkpoint(TRUNCATE)"
+        print_counters
+      end
+      
+      puts "Performing database maintenance..." if @options[:verbose]
+      execute "PRAGMA optimize"
+      execute "VACUUM"
+      execute "PRAGMA wal_checkpoint(TRUNCATE)"
 
-    hide_database_files
+      hide_database_files
+    end
   end
 
   def show_duplicates
     current_digest = nil
-    query_duplicates.each do |found|
+    digests_select_duplicates.each do |found|
       if current_digest != found["digest"]
         puts "" if current_digest
         current_digest = found["digest"]
@@ -306,9 +359,10 @@ class FileDigests
 
     normalized_filename = filename.delete_prefix("#{@files_path.to_s}/").encode("utf-8", universal_newline: true).unicode_normalize(:nfkc)
     mtime_string = time_to_database stat.mtime
-    digest = get_file_digest(filename)
+    digest, new_digest = get_file_digest(filename)
 
     nested_transaction do
+      new_digests_insert(normalized_filename, new_digest) if new_digest
       process_file_indeed normalized_filename, mtime_string, digest
     end
 
@@ -326,15 +380,15 @@ class FileDigests
   end
 
   def process_previously_seen_file found, filename, mtime, digest
-    @missing_files.delete(filename)
+    missing_files_delete filename
     if found["digest"] == digest
       @counters[:good] += 1
       puts "GOOD: #{filename}" if @options[:verbose]
       unless @options[:test_only]
         if found["mtime"] == mtime
-          touch_digest_check_time found["id"]
+          digests_touch_check_time found["id"]
         else
-          update_mtime mtime, found["id"]
+          digests_update_mtime mtime, found["id"]
         end
       end
     else
@@ -345,18 +399,17 @@ class FileDigests
         @counters[:updated] += 1
         puts "UPDATED#{" (FATE ACCEPTED)" if found["mtime"] == mtime && @options[:accept_fate]}: #{filename}" unless @options[:quiet]
         unless @options[:test_only]
-          update_mtime_and_digest mtime, digest, found["id"]
+          digests_update_mtime_and_digest mtime, digest, found["id"]
         end
       end
     end
   end
 
   def process_new_file filename, mtime, digest
-    @counters[:new] += 1
     puts "NEW: #{filename}" unless @options[:quiet]
+    new_files_insert filename, digest
     unless @options[:test_only]
-      @new_files[filename] = digest
-      insert filename, mtime, digest
+      digests_insert filename, mtime, digest
     end
   end
 
@@ -364,29 +417,31 @@ class FileDigests
   # Renames and missing files
 
   def track_renames
-    @missing_files.delete_if do |filename, digest|
-      if @new_files.value?(digest)
-        @counters[:renamed] += 1
-        unless @options[:test_only]
-          delete_by_filename filename
-        end
-        true
-      end
+    unless @options[:test_only]
+      digests_delete_renamed_files
     end
+    missing_files_delete_renamed_files
+    @counters[:renamed] = @db.changes
   end
 
   def print_missing_files
     puts "\nMISSING FILES:"
-    @missing_files.sort.to_h.each do |filename, digest|
-      puts filename
+    missing_files_select_all_filenames.each do |record|
+      puts record["filename"]
     end
   end
 
   def remove_missing_files
-    @missing_files.each do |filename, digest|
-      delete_by_filename filename
-    end
-    @missing_files = {}
+    digests_delete_all_missing_files
+    missing_files_delete_all
+  end
+
+  def missing_files_count
+    missing_files_count_query!&.first&.first
+  end
+
+  def any_missing_files?
+    missing_files_count > 0
   end
 
 
@@ -397,6 +452,7 @@ class FileDigests
   end
 
   def integrity_check
+    puts "Checking database integrity..." if @options[:verbose]
     if execute("PRAGMA integrity_check")&.first&.fetch("integrity_check") != "ok"
       raise "Database integrity check failed"
     end
@@ -451,7 +507,7 @@ class FileDigests
   end
 
   def find_by_filename filename
-    result = find_by_filename_query filename
+    result = digests_find_by_filename_query filename
     found = result.next
     raise "Multiple records found" if result.next
     found
@@ -471,6 +527,18 @@ class FileDigests
     end
   end
 
+  def check_if_database_is_at_certain_version target_version
+    current_version = get_metadata("database_version")
+    if current_version != target_version
+      STDERR.puts "This version of file-digests (#{FileDigests::VERSION || "unknown"}) is only compartible with the database version #{target_version}. Current database version is #{current_version}. To use this database, please install appropriate version if file-digest."
+      raise "Incompatible database version"
+    end
+  end
+
+  def new_files_count
+    new_files_count_query!&.first&.first
+  end
+
 
   # Filesystem-related helpers
 
@@ -493,6 +561,7 @@ class FileDigests
   end
 
   def walk_files
+    puts "Gathering the list of files..." if @options[:verbose]
     Dir.glob(@files_path + "**" + "*", File::FNM_DOTMATCH) do |filename|
       yield filename
     end
@@ -508,18 +577,13 @@ class FileDigests
         digest.update(buffer)
         new_digest.update(buffer) if @new_digest_algorithm
       end
-      @new_digests[digest.hexdigest] = new_digest.hexdigest if @new_digest_algorithm
-      return digest.hexdigest
+      return [digest.hexdigest, (new_digest.hexdigest if @new_digest_algorithm)]
     end
   end
 
 
   # Runtime state helpers
 
-  def any_missing_files?
-    @missing_files.length > 0
-  end
-
   def any_exceptions?
     @counters[:exceptions] > 0
   end
@@ -533,14 +597,17 @@ class FileDigests
   def confirm text
     if STDIN.tty? && STDOUT.tty?
       puts "#{text} (y/n)?"
-      STDIN.gets.strip.downcase == "y"
+      start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      result = (STDIN.gets.strip.downcase == "y")
+      @user_input_wait_time += (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start)
+      result
     end
   end
 
   def measure_time
     start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
     yield
-    elapsed = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start)
+    elapsed = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start) - @user_input_wait_time
     puts "Elapsed time: #{elapsed.to_i / 3600}h #{(elapsed.to_i % 3600) / 60}m #{"%.3f" % (elapsed % 60)}s" unless @options[:quiet]
   end
 
@@ -558,11 +625,14 @@ class FileDigests
   end
 
   def print_counters
+    missing_files_count_result = missing_files_count
+    new_files_count_result = new_files_count - @counters[:renamed]
+
     puts "#{@counters[:good]} file(s) passes digest check" if @counters[:good] > 0
     puts "#{@counters[:updated]} file(s) are updated" if @counters[:updated] > 0
-    puts "#{@counters[:new]} file(s) are new" if @counters[:new] > 0
+    puts "#{new_files_count_result} file(s) are new" if new_files_count_result > 0
     puts "#{@counters[:renamed]} file(s) are renamed" if @counters[:renamed] > 0
-    puts "#{@missing_files.length} file(s) are missing" if @missing_files.length > 0
+    puts "#{missing_files_count_result} file(s) are missing" if missing_files_count_result > 0
     puts "#{@counters[:likely_damaged]} file(s) are likely damaged (!)" if @counters[:likely_damaged] > 0
     puts "#{@counters[:exceptions]} file(s) had exceptions occured during processing (!)" if @counters[:exceptions] > 0
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: file-digests
 version: !ruby/object:Gem::Version
-  version: 0.0.30
+  version: 0.0.35
 platform: ruby
 authors:
 - Stanislav Senotrusov
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-10-15 00:00:00.000000000 Z
+date: 2020-10-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: openssl