file-digests 0.0.29 → 0.0.34

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cab5bc80dc7949984501c3068bf99f44f82564774733e84b1cd6810fd0cb6a05
-  data.tar.gz: 97a4220b83a08408345b21a60256e55bbbe0a5fb30e6c97efa05a4faac42d356
+  metadata.gz: dd11bc930ad2146b6e9bbecd44479df1817c8262197a184f19d5db03296d16c1
+  data.tar.gz: 60a3741b3b6d7cfa714991bac995176398e4834a53106751b55d537a9e9ac901
 SHA512:
-  metadata.gz: 5eaae0823ccb95bb3db6245e0863ff37aaa6f14f599bd92d040c8085e753aaa8d3be7903d1039a433bded9011bdc73c6a0d42e3e10d1e2c52eb81b6b791a5bc1
-  data.tar.gz: 97f7fa2ac8605fc1775570accf44a93c73e67a11699b7e98bfa9687f9e2c0219ad1890c1a293f52ca901079e602f6cf6863b5f57de174dd39e36ec338d6597f7
+  metadata.gz: 2cfbe4ac1169e5b48c9bb0c1ee3200fa060e38bc770200d14a127f98ab3967f83476f35130f4bd9d8427bbe7cac07c33b43e14c086e5897921c7383f0748cbfe
+  data.tar.gz: 9a5830dc67eb127ff39a0fffda53193d5ad43cfe25405e56d7bd211475af69192f404433c02f2c83d0c3f79dd4a56e4a02f469dbaad52abf6ab2027d8f9894bf

data/bin/file-digests

@@ -1,5 +1,19 @@
 #!/usr/bin/env ruby
 
+#  Copyright 2020 Stanislav Senotrusov <stan@senotrusov.com>
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
 require 'file-digests'
 
 FileDigests.run_cli_utility

data/lib/file-digests.rb

@@ -1,3 +1,17 @@
+#  Copyright 2020 Stanislav Senotrusov <stan@senotrusov.com>
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
 require "date"
 require "digest"
 require "fileutils"
@@ -8,7 +22,8 @@ require "set"
 require "sqlite3"
 
 class FileDigests
-  DIGEST_ALGORITHMS=["BLAKE2b512", "SHA3-256", "SHA512-256"]
+  VERSION = Gem.loaded_specs["file-digests"]&.version&.to_s
+  DIGEST_ALGORITHMS = ["BLAKE2b512", "SHA3-256", "SHA512-256"]
   LEGACY_DIGEST_ALGORITHMS = ["SHA512", "SHA256"]
 
   def self.canonical_digest_algorithm_name(string)
@@ -75,7 +90,12 @@ class FileDigests
         options[:quiet] = true
       end
 
-      opts.on("-t", "--test", "Perform only the test, do not modify the digest database.") do
+      opts.on(
+        "-t", "--test",
+        "Perform a test to verify directory contents.",
+        "Compare actual files with the stored digests, check if any files are missing.",
+        "Digest database will not be modified."
+      ) do
         options[:test_only] = true
       end
 
@@ -96,6 +116,7 @@ class FileDigests
 
   def initialize files_path, digest_database_path, options = {}
     @options = options
+    @user_input_wait_time = 0
 
     initialize_paths files_path, digest_database_path
     initialize_database
@@ -139,8 +160,6 @@ class FileDigests
     @db.results_as_hash = true
     @db.busy_timeout = 5000
 
-    file_digests_gem_version = Gem.loaded_specs["file-digests"]&.version&.to_s
-
     execute "PRAGMA encoding = 'UTF-8'"
     execute "PRAGMA locking_mode = 'EXCLUSIVE'"
     execute "PRAGMA journal_mode = 'WAL'"
@@ -155,14 +174,13 @@ class FileDigests
         execute "CREATE TABLE metadata (
           key TEXT NOT NULL PRIMARY KEY,
           value TEXT)"
-        execute "CREATE UNIQUE INDEX metadata_key ON metadata(key)"
         metadata_table_was_created = true
       end
 
       prepare_method :set_metadata_query, "INSERT INTO metadata (key, value) VALUES (?, ?) ON CONFLICT (key) DO UPDATE SET value=excluded.value"
       prepare_method :get_metadata_query, "SELECT value FROM metadata WHERE key = ?"
 
-      set_metadata("metadata_table_created_by_gem_version", file_digests_gem_version) if file_digests_gem_version && metadata_table_was_created
+      set_metadata("metadata_table_created_by_gem_version", FileDigests::VERSION) if FileDigests::VERSION && metadata_table_was_created
 
       # Heuristic to detect database version 1 (metadata was not stored back then)
       unless get_metadata("database_version")
@@ -179,20 +197,19 @@ class FileDigests
           digest TEXT NOT NULL,
           digest_check_time TEXT NOT NULL)"
         execute "CREATE UNIQUE INDEX digests_filename ON digests(filename)"
-        set_metadata("digests_table_created_by_gem_version", file_digests_gem_version) if file_digests_gem_version
+        execute "CREATE INDEX digests_digest ON digests(digest)"
+        set_metadata("digests_table_created_by_gem_version", FileDigests::VERSION) if FileDigests::VERSION
       end
 
-      prepare_method :insert, "INSERT INTO digests (filename, mtime, digest, digest_check_time) VALUES (?, ?, ?, datetime('now'))"
-      prepare_method :find_by_filename_query, "SELECT id, mtime, digest FROM digests WHERE filename = ?"
-      prepare_method :touch_digest_check_time, "UPDATE digests SET digest_check_time = datetime('now') WHERE id = ?"
-      prepare_method :update_mtime_and_digest, "UPDATE digests SET mtime = ?, digest = ?, digest_check_time = datetime('now') WHERE id = ?"
-      prepare_method :update_mtime, "UPDATE digests SET mtime = ?, digest_check_time = datetime('now') WHERE id = ?"
-      prepare_method :delete_by_filename, "DELETE FROM digests WHERE filename = ?"
-      prepare_method :query_duplicates, "SELECT digest, filename FROM digests WHERE digest IN (SELECT digest FROM digests GROUP BY digest HAVING count(*) > 1) ORDER BY digest, filename;"
-      prepare_method :update_digest_to_new_digest, "UPDATE digests SET digest = ? WHERE digest = ?"
+      prepare_method :digests_insert, "INSERT INTO digests (filename, mtime, digest, digest_check_time) VALUES (?, ?, ?, datetime('now'))"
+      prepare_method :digests_find_by_filename_query, "SELECT id, mtime, digest FROM digests WHERE filename = ?"
+      prepare_method :digests_touch_check_time, "UPDATE digests SET digest_check_time = datetime('now') WHERE id = ?"
+      prepare_method :digests_update_mtime_and_digest, "UPDATE digests SET mtime = ?, digest = ?, digest_check_time = datetime('now') WHERE id = ?"
+      prepare_method :digests_update_mtime, "UPDATE digests SET mtime = ?, digest_check_time = datetime('now') WHERE id = ?"
+      prepare_method :digests_select_duplicates, "SELECT digest, filename FROM digests WHERE digest IN (SELECT digest FROM digests GROUP BY digest HAVING count(*) > 1) ORDER BY digest, filename;"
 
       unless get_metadata("database_version")
-        set_metadata "database_version", "2"
+        set_metadata "database_version", "3"
       end
 
       # Convert database from 1st to 2nd version
@@ -207,75 +224,110 @@ class FileDigests
         end
       end
 
-      if get_metadata("database_version") != "2"
-        STDERR.puts "This version of file-digests (#{file_digests_gem_version || "unknown"}) is only compartible with the database version 2. Current database version is #{get_metadata("database_version")}. To use this database, please install appropriate version if file-digest."
-        raise "Incompatible database version"
+      if get_metadata("database_version") == "2"
+        execute "CREATE INDEX digests_digest ON digests(digest)"
+        set_metadata "database_version", "3"
       end
+  
+      check_if_database_is_at_certain_version "3"
+
+      create_temporary_tables
     end
   end
 
-  def perform_check
-    perhaps_transaction(@new_digest_algorithm, :exclusive) do
-      @counters = {good: 0, updated: 0, new: 0, renamed: 0, likely_damaged: 0, exceptions: 0}
-      @new_files = {}
-      @new_digests = {}
+  def create_temporary_tables
+    execute "CREATE TEMPORARY TABLE new_files (
+      filename TEXT NOT NULL PRIMARY KEY,
+      digest TEXT NOT NULL)"
+    execute "CREATE INDEX new_files_digest ON new_files(digest)"
+
+    prepare_method :new_files_insert, "INSERT INTO new_files (filename, digest) VALUES (?, ?)"
+    prepare_method :new_files_count_query, "SELECT count(*) FROM new_files"
 
-      @missing_files = Hash[@db.prepare("SELECT filename, digest FROM digests").execute!]
+    execute "CREATE TEMPORARY TABLE missing_files (
+      filename TEXT NOT NULL PRIMARY KEY,
+      digest TEXT NOT NULL)"
+    execute "CREATE INDEX missing_files_digest ON missing_files(digest)"
+
+    execute "INSERT INTO missing_files (filename, digest) SELECT filename, digest FROM digests"
+
+    prepare_method :missing_files_delete, "DELETE FROM missing_files WHERE filename = ?"
+    prepare_method :missing_files_delete_renamed_files, "DELETE FROM missing_files WHERE digest IN (SELECT digest FROM new_files)"
+    prepare_method :missing_files_select_all_filenames, "SELECT filename FROM missing_files ORDER BY filename"
+    prepare_method :missing_files_delete_all, "DELETE FROM missing_files"
+    prepare_method :missing_files_count_query, "SELECT count(*) FROM missing_files"
+
+    prepare_method :digests_delete_renamed_files, "DELETE FROM digests WHERE filename IN (SELECT filename FROM missing_files WHERE digest IN (SELECT digest FROM new_files))"
+    prepare_method :digests_delete_all_missing_files, "DELETE FROM digests WHERE filename IN (SELECT filename FROM missing_files)"
+
+    execute "CREATE TEMPORARY TABLE new_digests (
+      filename TEXT NOT NULL PRIMARY KEY,
+      digest TEXT NOT NULL)"
+
+    prepare_method :new_digests_insert, "INSERT INTO new_digests (filename, digest) VALUES (?, ?)"
+    prepare_method :digests_update_digests_to_new_digests, "INSERT INTO digests (filename, digest, digest_check_time) SELECT filename, digest, false FROM new_digests WHERE true ON CONFLICT (filename) DO UPDATE SET digest=excluded.digest"
+  end
+
+  def perform_check
+    measure_time do
+      perhaps_transaction(@new_digest_algorithm, :exclusive) do
+        @counters = {good: 0, updated: 0, renamed: 0, likely_damaged: 0, exceptions: 0}
 
-      measure_time do
         walk_files do |filename|
           process_file filename
         end
-      end
 
-      nested_transaction do
-        track_renames
-      end
+        nested_transaction do
+          puts "Tracking renames..." if @options[:verbose]
+          track_renames
+        end
 
-      if any_missing_files?
-        if any_exceptions?
-          STDERR.puts "Due to previously occurred errors, database cleanup from missing files will be skipped this time."
-        else
-          print_missing_files
-          if !@options[:test_only] && (@options[:auto] || confirm("Remove missing files from the database"))
-            nested_transaction do
-              remove_missing_files
+        if any_missing_files?
+          if any_exceptions?
+            STDERR.puts "Due to previously occurred errors, missing files will not removed from the database."
+          else
+            print_missing_files
+            if !@options[:test_only] && (@options[:auto] || confirm("Remove missing files from the database"))
+              nested_transaction do
+                puts "Removing missing files..." if @options[:verbose]
+                remove_missing_files
+              end
             end
           end
         end
-      end
 
-      if @new_digest_algorithm && !@options[:test_only]
-        if any_missing_files? || any_likely_damaged? || any_exceptions?
-          STDERR.puts "ERROR: New digest algorithm will not be in effect until there are files that are missing, likely damaged, or processed with an exception."
-        else
-          @new_digests.each do |old_digest, new_digest|
-            update_digest_to_new_digest new_digest, old_digest
+        if @new_digest_algorithm && !@options[:test_only]
+          if any_missing_files? || any_likely_damaged? || any_exceptions?
+            STDERR.puts "ERROR: New digest algorithm will not be in effect until there are files that are missing, likely damaged, or processed with an exception."
+          else
+            puts "Updating database to a new digest algorithm..." if @options[:verbose]
+            digests_update_digests_to_new_digests
+            set_metadata "digest_algorithm", @new_digest_algorithm
+            puts "Transition to a new digest algorithm complete: #{@new_digest_algorithm}"
           end
-          set_metadata "digest_algorithm", @new_digest_algorithm
-          puts "Transition to a new digest algorithm complete: #{@new_digest_algorithm}"
         end
-      end
 
-      if any_likely_damaged? || any_exceptions?
-        STDERR.puts "PLEASE REVIEW ERRORS THAT WERE OCCURRED!"
-      end
+        if any_likely_damaged? || any_exceptions?
+          STDERR.puts "PLEASE REVIEW ERRORS THAT WERE OCCURRED!"
+        end
 
-      set_metadata(@options[:test_only] ? "latest_test_only_check_time" : "latest_complete_check_time", time_to_database(Time.now))
+        set_metadata(@options[:test_only] ? "latest_test_only_check_time" : "latest_complete_check_time", time_to_database(Time.now))
 
+        print_counters
+      end
+      
+      puts "Performing database maintenance..." if @options[:verbose]
       execute "PRAGMA optimize"
       execute "VACUUM"
       execute "PRAGMA wal_checkpoint(TRUNCATE)"
 
       hide_database_files
-
-      print_counters
     end
   end
 
   def show_duplicates
     current_digest = nil
-    query_duplicates.each do |found|
+    digests_select_duplicates.each do |found|
       if current_digest != found["digest"]
         puts "" if current_digest
         current_digest = found["digest"]
@@ -307,9 +359,10 @@ class FileDigests
 
     normalized_filename = filename.delete_prefix("#{@files_path.to_s}/").encode("utf-8", universal_newline: true).unicode_normalize(:nfkc)
     mtime_string = time_to_database stat.mtime
-    digest = get_file_digest(filename)
+    digest, new_digest = get_file_digest(filename)
 
     nested_transaction do
+      new_digests_insert(normalized_filename, new_digest) if new_digest
       process_file_indeed normalized_filename, mtime_string, digest
     end
 
@@ -327,15 +380,15 @@ class FileDigests
   end
 
   def process_previously_seen_file found, filename, mtime, digest
-    @missing_files.delete(filename)
+    missing_files_delete filename
     if found["digest"] == digest
       @counters[:good] += 1
       puts "GOOD: #{filename}" if @options[:verbose]
       unless @options[:test_only]
         if found["mtime"] == mtime
-          touch_digest_check_time found["id"]
+          digests_touch_check_time found["id"]
         else
-          update_mtime mtime, found["id"]
+          digests_update_mtime mtime, found["id"]
         end
       end
     else
@@ -346,18 +399,17 @@ class FileDigests
         @counters[:updated] += 1
         puts "UPDATED#{" (FATE ACCEPTED)" if found["mtime"] == mtime && @options[:accept_fate]}: #{filename}" unless @options[:quiet]
         unless @options[:test_only]
-          update_mtime_and_digest mtime, digest, found["id"]
+          digests_update_mtime_and_digest mtime, digest, found["id"]
         end
       end
     end
   end
 
   def process_new_file filename, mtime, digest
-    @counters[:new] += 1
     puts "NEW: #{filename}" unless @options[:quiet]
+    new_files_insert filename, digest
     unless @options[:test_only]
-      @new_files[filename] = digest
-      insert filename, mtime, digest
+      digests_insert filename, mtime, digest
     end
   end
 
@@ -365,29 +417,31 @@ class FileDigests
   # Renames and missing files
 
   def track_renames
-    @missing_files.delete_if do |filename, digest|
-      if @new_files.value?(digest)
-        @counters[:renamed] += 1
-        unless @options[:test_only]
-          delete_by_filename filename
-        end
-        true
-      end
+    unless @options[:test_only]
+      digests_delete_renamed_files
     end
+    missing_files_delete_renamed_files
+    @counters[:renamed] = @db.changes
   end
 
   def print_missing_files
     puts "\nMISSING FILES:"
-    @missing_files.sort.to_h.each do |filename, digest|
-      puts filename
+    missing_files_select_all_filenames.each do |record|
+      puts record["filename"]
     end
   end
 
   def remove_missing_files
-    @missing_files.each do |filename, digest|
-      delete_by_filename filename
-    end
-    @missing_files = {}
+    digests_delete_all_missing_files
+    missing_files_delete_all
+  end
+
+  def missing_files_count
+    missing_files_count_query!&.first&.first
+  end
+
+  def any_missing_files?
+    missing_files_count > 0
   end
 
 
@@ -452,7 +506,7 @@ class FileDigests
   end
 
   def find_by_filename filename
-    result = find_by_filename_query filename
+    result = digests_find_by_filename_query filename
     found = result.next
     raise "Multiple records found" if result.next
     found
@@ -472,6 +526,18 @@ class FileDigests
     end
   end
 
+  def check_if_database_is_at_certain_version target_version
+    current_version = get_metadata("database_version")
+    if current_version != target_version
+      STDERR.puts "This version of file-digests (#{FileDigests::VERSION || "unknown"}) is only compartible with the database version #{target_version}. Current database version is #{current_version}. To use this database, please install appropriate version if file-digest."
+      raise "Incompatible database version"
+    end
+  end
+
+  def new_files_count
+    new_files_count_query!&.first&.first
+  end
+
 
   # Filesystem-related helpers
 
@@ -494,6 +560,7 @@ class FileDigests
   end
 
   def walk_files
+    puts "Gathering the list of files..." if @options[:verbose]
     Dir.glob(@files_path + "**" + "*", File::FNM_DOTMATCH) do |filename|
       yield filename
     end
@@ -509,18 +576,13 @@ class FileDigests
         digest.update(buffer)
         new_digest.update(buffer) if @new_digest_algorithm
       end
-      @new_digests[digest.hexdigest] = new_digest.hexdigest if @new_digest_algorithm
-      return digest.hexdigest
+      return [digest.hexdigest, (new_digest.hexdigest if @new_digest_algorithm)]
     end
   end
 
 
   # Runtime state helpers
 
-  def any_missing_files?
-    @missing_files.length > 0
-  end
-
   def any_exceptions?
     @counters[:exceptions] > 0
   end
@@ -534,14 +596,17 @@ class FileDigests
   def confirm text
     if STDIN.tty? && STDOUT.tty?
       puts "#{text} (y/n)?"
-      STDIN.gets.strip.downcase == "y"
+      start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      result = (STDIN.gets.strip.downcase == "y")
+      @user_input_wait_time += (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start)
+      result
     end
   end
 
   def measure_time
     start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
     yield
-    elapsed = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start)
+    elapsed = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start) - @user_input_wait_time
     puts "Elapsed time: #{elapsed.to_i / 3600}h #{(elapsed.to_i % 3600) / 60}m #{"%.3f" % (elapsed % 60)}s" unless @options[:quiet]
   end
 
@@ -559,11 +624,14 @@ class FileDigests
   end
 
   def print_counters
+    missing_files_count_result = missing_files_count
+    new_files_count_result = new_files_count - @counters[:renamed]
+
     puts "#{@counters[:good]} file(s) passes digest check" if @counters[:good] > 0
     puts "#{@counters[:updated]} file(s) are updated" if @counters[:updated] > 0
-    puts "#{@counters[:new]} file(s) are new" if @counters[:new] > 0
+    puts "#{new_files_count_result} file(s) are new" if new_files_count_result > 0
     puts "#{@counters[:renamed]} file(s) are renamed" if @counters[:renamed] > 0
-    puts "#{@missing_files.length} file(s) are missing" if @missing_files.length > 0
+    puts "#{missing_files_count_result} file(s) are missing" if missing_files_count_result > 0
     puts "#{@counters[:likely_damaged]} file(s) are likely damaged (!)" if @counters[:likely_damaged] > 0
     puts "#{@counters[:exceptions]} file(s) had exceptions occured during processing (!)" if @counters[:exceptions] > 0
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: file-digests
 version: !ruby/object:Gem::Version
-  version: 0.0.29
+  version: 0.0.34
 platform: ruby
 authors:
 - Stanislav Senotrusov
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-10-15 00:00:00.000000000 Z
+date: 2020-10-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: openssl