file-digests 0.0.31 → 0.0.36

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c7a7259116dbf913b55651133e3bcb5e5ed545bd42e9bb63b1e5f83fb7b98a57
-  data.tar.gz: 17915ee54fceff38c52d451db21648b15fb0a3ef4c595df7736b7e077dbbb10e
+  metadata.gz: 624f7ec80a0cb15be2bc07a3dd592c33c7a178c22fb7f59ef326760d17cc564f
+  data.tar.gz: 92bb88c58bc152ad01649b0688861ea16af6801d2d6625c238b96e9ff00aaed4
 SHA512:
-  metadata.gz: d186110615604c8a833f213cf5dee0f41e19992bc34b335f59e8126ad4dc5995a5601ec9ac0559b7fa930b58c9e8ee11a258a5392dd4f4433c21765ee1e99b1b
-  data.tar.gz: ed21957d8217ae64d8aa90aec4fd81d680bd1acbfc2115f59c09acc61510b23b28fcef7083ecd51a5ce854f52d2ec8aab4b79b53599975c18b9d71f307d19f7a
+  metadata.gz: 9dbca39af63f07ddfe4d92618d0ff5e0813fa30561d42b629caa918f664f1149f8ad895e8555137b0f462eb4eac942adda07efed03859d17cb635a8d9bb25b57
+  data.tar.gz: 788d37c5d82e8892dfcfb6f4a8532f484f7cf17c5f937e8a64b09af902fe3a904ed034fc88d403852bbd8819a9d11ab61a672e2d6f3114420ed7777ea9db2961

data/bin/file-digests

@@ -1,5 +1,19 @@
 #!/usr/bin/env ruby
 
+#  Copyright 2020 Stanislav Senotrusov <stan@senotrusov.com>
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
 require 'file-digests'
 
 FileDigests.run_cli_utility

data/lib/file-digests.rb

@@ -1,6 +1,21 @@
+#  Copyright 2020 Stanislav Senotrusov <stan@senotrusov.com>
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
 require "date"
 require "digest"
 require "fileutils"
+require "find"
 require "openssl"
 require "optparse"
 require "pathname"
@@ -8,7 +23,8 @@ require "set"
 require "sqlite3"
 
 class FileDigests
-  DIGEST_ALGORITHMS=["BLAKE2b512", "SHA3-256", "SHA512-256"]
+  VERSION = Gem.loaded_specs["file-digests"]&.version&.to_s
+  DIGEST_ALGORITHMS = ["BLAKE2b512", "SHA3-256", "SHA512-256"]
   LEGACY_DIGEST_ALGORITHMS = ["SHA512", "SHA256"]
 
   def self.canonical_digest_algorithm_name(string)
@@ -75,7 +91,12 @@ class FileDigests
         options[:quiet] = true
       end
 
-      opts.on("-t", "--test", "Perform only the test, do not modify the digest database.") do
+      opts.on(
+        "-t", "--test",
+        "Perform a test to verify directory contents.",
+        "Compare actual files with the stored digests, check if any files are missing.",
+        "Digest database will not be modified."
+      ) do
         options[:test_only] = true
       end
 
@@ -96,6 +117,7 @@ class FileDigests
 
   def initialize files_path, digest_database_path, options = {}
     @options = options
+    @user_input_wait_time = 0
 
     initialize_paths files_path, digest_database_path
     initialize_database
@@ -139,8 +161,6 @@ class FileDigests
     @db.results_as_hash = true
     @db.busy_timeout = 5000
 
-    file_digests_gem_version = Gem.loaded_specs["file-digests"]&.version&.to_s
-
     execute "PRAGMA encoding = 'UTF-8'"
     execute "PRAGMA locking_mode = 'EXCLUSIVE'"
     execute "PRAGMA journal_mode = 'WAL'"
@@ -155,14 +175,13 @@ class FileDigests
         execute "CREATE TABLE metadata (
           key TEXT NOT NULL PRIMARY KEY,
           value TEXT)"
-        execute "CREATE UNIQUE INDEX metadata_key ON metadata(key)"
         metadata_table_was_created = true
       end
 
       prepare_method :set_metadata_query, "INSERT INTO metadata (key, value) VALUES (?, ?) ON CONFLICT (key) DO UPDATE SET value=excluded.value"
       prepare_method :get_metadata_query, "SELECT value FROM metadata WHERE key = ?"
 
-      set_metadata("metadata_table_created_by_gem_version", file_digests_gem_version) if file_digests_gem_version && metadata_table_was_created
+      set_metadata("metadata_table_created_by_gem_version", FileDigests::VERSION) if FileDigests::VERSION && metadata_table_was_created
 
       # Heuristic to detect database version 1 (metadata was not stored back then)
       unless get_metadata("database_version")
@@ -179,20 +198,19 @@ class FileDigests
           digest TEXT NOT NULL,
           digest_check_time TEXT NOT NULL)"
         execute "CREATE UNIQUE INDEX digests_filename ON digests(filename)"
-        set_metadata("digests_table_created_by_gem_version", file_digests_gem_version) if file_digests_gem_version
+        execute "CREATE INDEX digests_digest ON digests(digest)"
+        set_metadata("digests_table_created_by_gem_version", FileDigests::VERSION) if FileDigests::VERSION
       end
 
-      prepare_method :insert, "INSERT INTO digests (filename, mtime, digest, digest_check_time) VALUES (?, ?, ?, datetime('now'))"
-      prepare_method :find_by_filename_query, "SELECT id, mtime, digest FROM digests WHERE filename = ?"
-      prepare_method :touch_digest_check_time, "UPDATE digests SET digest_check_time = datetime('now') WHERE id = ?"
-      prepare_method :update_mtime_and_digest, "UPDATE digests SET mtime = ?, digest = ?, digest_check_time = datetime('now') WHERE id = ?"
-      prepare_method :update_mtime, "UPDATE digests SET mtime = ?, digest_check_time = datetime('now') WHERE id = ?"
-      prepare_method :delete_by_filename, "DELETE FROM digests WHERE filename = ?"
-      prepare_method :query_duplicates, "SELECT digest, filename FROM digests WHERE digest IN (SELECT digest FROM digests GROUP BY digest HAVING count(*) > 1) ORDER BY digest, filename;"
-      prepare_method :update_digest_to_new_digest, "UPDATE digests SET digest = ? WHERE digest = ?"
+      prepare_method :digests_insert, "INSERT INTO digests (filename, mtime, digest, digest_check_time) VALUES (?, ?, ?, datetime('now'))"
+      prepare_method :digests_find_by_filename_query, "SELECT id, mtime, digest FROM digests WHERE filename = ?"
+      prepare_method :digests_touch_check_time, "UPDATE digests SET digest_check_time = datetime('now') WHERE id = ?"
+      prepare_method :digests_update_mtime_and_digest, "UPDATE digests SET mtime = ?, digest = ?, digest_check_time = datetime('now') WHERE id = ?"
+      prepare_method :digests_update_mtime, "UPDATE digests SET mtime = ?, digest_check_time = datetime('now') WHERE id = ?"
+      prepare_method :digests_select_duplicates, "SELECT digest, filename FROM digests WHERE digest IN (SELECT digest FROM digests GROUP BY digest HAVING count(*) > 1) ORDER BY digest, filename;"
 
       unless get_metadata("database_version")
-        set_metadata "database_version", "2"
+        set_metadata "database_version", "3"
       end
 
       # Convert database from 1st to 2nd version
@@ -207,79 +225,110 @@ class FileDigests
         end
       end
 
-      if get_metadata("database_version") != "2"
-        STDERR.puts "This version of file-digests (#{file_digests_gem_version || "unknown"}) is only compartible with the database version 2. Current database version is #{get_metadata("database_version")}. To use this database, please install appropriate version if file-digest."
-        raise "Incompatible database version"
+      if get_metadata("database_version") == "2"
+        execute "CREATE INDEX digests_digest ON digests(digest)"
+        set_metadata "database_version", "3"
       end
+  
+      check_if_database_is_at_certain_version "3"
+
+      create_temporary_tables
     end
   end
 
-  def perform_check
-    perhaps_transaction(@new_digest_algorithm, :exclusive) do
-      @counters = {good: 0, updated: 0, new: 0, renamed: 0, likely_damaged: 0, exceptions: 0}
-      @new_files = {}
-      @new_digests = {}
+  def create_temporary_tables
+    execute "CREATE TEMPORARY TABLE new_files (
+      filename TEXT NOT NULL PRIMARY KEY,
+      digest TEXT NOT NULL)"
+    execute "CREATE INDEX new_files_digest ON new_files(digest)"
+
+    prepare_method :new_files_insert, "INSERT INTO new_files (filename, digest) VALUES (?, ?)"
+    prepare_method :new_files_count_query, "SELECT count(*) FROM new_files"
+
+    execute "CREATE TEMPORARY TABLE missing_files (
+      filename TEXT NOT NULL PRIMARY KEY,
+      digest TEXT NOT NULL)"
+    execute "CREATE INDEX missing_files_digest ON missing_files(digest)"
+
+    execute "INSERT INTO missing_files (filename, digest) SELECT filename, digest FROM digests"
+
+    prepare_method :missing_files_delete, "DELETE FROM missing_files WHERE filename = ?"
+    prepare_method :missing_files_delete_renamed_files, "DELETE FROM missing_files WHERE digest IN (SELECT digest FROM new_files)"
+    prepare_method :missing_files_select_all_filenames, "SELECT filename FROM missing_files ORDER BY filename"
+    prepare_method :missing_files_delete_all, "DELETE FROM missing_files"
+    prepare_method :missing_files_count_query, "SELECT count(*) FROM missing_files"
+
+    prepare_method :digests_delete_renamed_files, "DELETE FROM digests WHERE filename IN (SELECT filename FROM missing_files WHERE digest IN (SELECT digest FROM new_files))"
+    prepare_method :digests_delete_all_missing_files, "DELETE FROM digests WHERE filename IN (SELECT filename FROM missing_files)"
+
+    execute "CREATE TEMPORARY TABLE new_digests (
+      filename TEXT NOT NULL PRIMARY KEY,
+      digest TEXT NOT NULL)"
+
+    prepare_method :new_digests_insert, "INSERT INTO new_digests (filename, digest) VALUES (?, ?)"
+    prepare_method :digests_update_digests_to_new_digests, "INSERT INTO digests (filename, digest, digest_check_time) SELECT filename, digest, false FROM new_digests WHERE true ON CONFLICT (filename) DO UPDATE SET digest=excluded.digest"
+  end
 
-      @missing_files = Hash[@db.prepare("SELECT filename, digest FROM digests").execute!]
+  def perform_check
+    measure_time do
+      perhaps_transaction(@new_digest_algorithm, :exclusive) do
+        @counters = {good: 0, updated: 0, renamed: 0, likely_damaged: 0, exceptions: 0}
 
-      measure_time do
         walk_files do |filename|
           process_file filename
         end
-      end
 
-      nested_transaction do
-        puts "Tracking renames..." if @options[:verbose]
-        track_renames
-      end
+        nested_transaction do
+          puts "Tracking renames..." if @options[:verbose]
+          track_renames
+        end
 
-      if any_missing_files?
-        if any_exceptions?
-          STDERR.puts "Due to previously occurred errors, database cleanup from missing files will be skipped this time."
-        else
-          print_missing_files
-          if !@options[:test_only] && (@options[:auto] || confirm("Remove missing files from the database"))
-            nested_transaction do
-              puts "Removing missing files..." if @options[:verbose]
-              remove_missing_files
+        if any_missing_files?
+          if any_exceptions?
+            STDERR.puts "Due to previously occurred errors, missing files will not removed from the database."
+          else
+            print_missing_files
+            if !@options[:test_only] && (@options[:auto] || confirm("Remove missing files from the database"))
+              nested_transaction do
+                puts "Removing missing files..." if @options[:verbose]
+                remove_missing_files
+              end
             end
           end
         end
-      end
 
-      if @new_digest_algorithm && !@options[:test_only]
-        if any_missing_files? || any_likely_damaged? || any_exceptions?
-          STDERR.puts "ERROR: New digest algorithm will not be in effect until there are files that are missing, likely damaged, or processed with an exception."
-        else
-          puts "Updating database to a new digest algorithm..." if @options[:verbose]
-          @new_digests.each do |old_digest, new_digest|
-            update_digest_to_new_digest new_digest, old_digest
+        if @new_digest_algorithm && !@options[:test_only]
+          if any_missing_files? || any_likely_damaged? || any_exceptions?
+            STDERR.puts "ERROR: New digest algorithm will not be in effect until there are files that are missing, likely damaged, or processed with an exception."
+          else
+            puts "Updating database to a new digest algorithm..." if @options[:verbose]
+            digests_update_digests_to_new_digests
+            set_metadata "digest_algorithm", @new_digest_algorithm
+            puts "Transition to a new digest algorithm complete: #{@new_digest_algorithm}"
           end
-          set_metadata "digest_algorithm", @new_digest_algorithm
-          puts "Transition to a new digest algorithm complete: #{@new_digest_algorithm}"
         end
-      end
 
-      if any_likely_damaged? || any_exceptions?
-        STDERR.puts "PLEASE REVIEW ERRORS THAT WERE OCCURRED!"
-      end
+        if any_likely_damaged? || any_exceptions?
+          STDERR.puts "PLEASE REVIEW ERRORS THAT WERE OCCURRED!"
+        end
 
-      set_metadata(@options[:test_only] ? "latest_test_only_check_time" : "latest_complete_check_time", time_to_database(Time.now))
+        set_metadata(@options[:test_only] ? "latest_test_only_check_time" : "latest_complete_check_time", time_to_database(Time.now))
 
-      print_counters
-    end
-    
-    puts "Performing database maintenance..." if @options[:verbose]
-    execute "PRAGMA optimize"
-    execute "VACUUM"
-    execute "PRAGMA wal_checkpoint(TRUNCATE)"
+        print_counters
+      end
+      
+      puts "Performing database maintenance..." if @options[:verbose]
+      execute "PRAGMA optimize"
+      execute "VACUUM"
+      execute "PRAGMA wal_checkpoint(TRUNCATE)"
 
-    hide_database_files
+      hide_database_files
+    end
   end
 
   def show_duplicates
     current_digest = nil
-    query_duplicates.each do |found|
+    digests_select_duplicates.each do |found|
       if current_digest != found["digest"]
         puts "" if current_digest
         current_digest = found["digest"]
@@ -311,9 +360,10 @@ class FileDigests
 
     normalized_filename = filename.delete_prefix("#{@files_path.to_s}/").encode("utf-8", universal_newline: true).unicode_normalize(:nfkc)
     mtime_string = time_to_database stat.mtime
-    digest = get_file_digest(filename)
+    digest, new_digest = get_file_digest(filename)
 
     nested_transaction do
+      new_digests_insert(normalized_filename, new_digest) if new_digest
       process_file_indeed normalized_filename, mtime_string, digest
     end
 
@@ -331,15 +381,15 @@ class FileDigests
   end
 
   def process_previously_seen_file found, filename, mtime, digest
-    @missing_files.delete(filename)
+    missing_files_delete filename
     if found["digest"] == digest
       @counters[:good] += 1
       puts "GOOD: #{filename}" if @options[:verbose]
       unless @options[:test_only]
         if found["mtime"] == mtime
-          touch_digest_check_time found["id"]
+          digests_touch_check_time found["id"]
         else
-          update_mtime mtime, found["id"]
+          digests_update_mtime mtime, found["id"]
         end
       end
     else
@@ -350,18 +400,17 @@ class FileDigests
         @counters[:updated] += 1
         puts "UPDATED#{" (FATE ACCEPTED)" if found["mtime"] == mtime && @options[:accept_fate]}: #{filename}" unless @options[:quiet]
         unless @options[:test_only]
-          update_mtime_and_digest mtime, digest, found["id"]
+          digests_update_mtime_and_digest mtime, digest, found["id"]
         end
       end
     end
   end
 
   def process_new_file filename, mtime, digest
-    @counters[:new] += 1
     puts "NEW: #{filename}" unless @options[:quiet]
+    new_files_insert filename, digest
     unless @options[:test_only]
-      @new_files[filename] = digest
-      insert filename, mtime, digest
+      digests_insert filename, mtime, digest
     end
   end
 
@@ -369,29 +418,31 @@ class FileDigests
   # Renames and missing files
 
   def track_renames
-    @missing_files.delete_if do |filename, digest|
-      if @new_files.value?(digest)
-        @counters[:renamed] += 1
-        unless @options[:test_only]
-          delete_by_filename filename
-        end
-        true
-      end
+    unless @options[:test_only]
+      digests_delete_renamed_files
     end
+    missing_files_delete_renamed_files
+    @counters[:renamed] = @db.changes
   end
 
   def print_missing_files
     puts "\nMISSING FILES:"
-    @missing_files.sort.to_h.each do |filename, digest|
-      puts filename
+    missing_files_select_all_filenames.each do |record|
+      puts record["filename"]
     end
   end
 
   def remove_missing_files
-    @missing_files.each do |filename, digest|
-      delete_by_filename filename
-    end
-    @missing_files = {}
+    digests_delete_all_missing_files
+    missing_files_delete_all
+  end
+
+  def missing_files_count
+    missing_files_count_query!&.first&.first
+  end
+
+  def any_missing_files?
+    missing_files_count > 0
   end
 
 
@@ -402,6 +453,7 @@ class FileDigests
   end
 
   def integrity_check
+    puts "Checking database integrity..." if @options[:verbose]
     if execute("PRAGMA integrity_check")&.first&.fetch("integrity_check") != "ok"
       raise "Database integrity check failed"
     end
@@ -456,7 +508,7 @@ class FileDigests
   end
 
   def find_by_filename filename
-    result = find_by_filename_query filename
+    result = digests_find_by_filename_query filename
     found = result.next
     raise "Multiple records found" if result.next
     found
@@ -476,6 +528,18 @@ class FileDigests
     end
   end
 
+  def check_if_database_is_at_certain_version target_version
+    current_version = get_metadata("database_version")
+    if current_version != target_version
+      STDERR.puts "This version of file-digests (#{FileDigests::VERSION || "unknown"}) is only compartible with the database version #{target_version}. Current database version is #{current_version}. To use this database, please install appropriate version if file-digest."
+      raise "Incompatible database version"
+    end
+  end
+
+  def new_files_count
+    new_files_count_query!&.first&.first
+  end
+
 
   # Filesystem-related helpers
 
@@ -498,8 +562,8 @@ class FileDigests
   end
 
   def walk_files
-    Dir.glob(@files_path + "**" + "*", File::FNM_DOTMATCH) do |filename|
-      yield filename
+    Find.find(@files_path) do |path|
+      yield path
     end
   end
 
@@ -513,18 +577,13 @@ class FileDigests
         digest.update(buffer)
         new_digest.update(buffer) if @new_digest_algorithm
       end
-      @new_digests[digest.hexdigest] = new_digest.hexdigest if @new_digest_algorithm
-      return digest.hexdigest
+      return [digest.hexdigest, (new_digest.hexdigest if @new_digest_algorithm)]
     end
   end
 
 
   # Runtime state helpers
 
-  def any_missing_files?
-    @missing_files.length > 0
-  end
-
   def any_exceptions?
     @counters[:exceptions] > 0
   end
@@ -538,14 +597,17 @@ class FileDigests
   def confirm text
     if STDIN.tty? && STDOUT.tty?
       puts "#{text} (y/n)?"
-      STDIN.gets.strip.downcase == "y"
+      start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      result = (STDIN.gets.strip.downcase == "y")
+      @user_input_wait_time += (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start)
+      result
     end
   end
 
   def measure_time
     start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
     yield
-    elapsed = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start)
+    elapsed = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start) - @user_input_wait_time
     puts "Elapsed time: #{elapsed.to_i / 3600}h #{(elapsed.to_i % 3600) / 60}m #{"%.3f" % (elapsed % 60)}s" unless @options[:quiet]
   end
 
@@ -563,11 +625,14 @@ class FileDigests
   end
 
   def print_counters
+    missing_files_count_result = missing_files_count
+    new_files_count_result = new_files_count - @counters[:renamed]
+
     puts "#{@counters[:good]} file(s) passes digest check" if @counters[:good] > 0
     puts "#{@counters[:updated]} file(s) are updated" if @counters[:updated] > 0
-    puts "#{@counters[:new]} file(s) are new" if @counters[:new] > 0
+    puts "#{new_files_count_result} file(s) are new" if new_files_count_result > 0
     puts "#{@counters[:renamed]} file(s) are renamed" if @counters[:renamed] > 0
-    puts "#{@missing_files.length} file(s) are missing" if @missing_files.length > 0
+    puts "#{missing_files_count_result} file(s) are missing" if missing_files_count_result > 0
     puts "#{@counters[:likely_damaged]} file(s) are likely damaged (!)" if @counters[:likely_damaged] > 0
     puts "#{@counters[:exceptions]} file(s) had exceptions occured during processing (!)" if @counters[:exceptions] > 0
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: file-digests
 version: !ruby/object:Gem::Version
-  version: 0.0.31
+  version: 0.0.36
 platform: ruby
 authors:
 - Stanislav Senotrusov
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-10-15 00:00:00.000000000 Z
+date: 2020-10-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: openssl