file-digests 0.0.33 → 0.0.38

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 68dea8d33894b138d5908dcf27fef7800a17af8f1441fd043876248414ad9525
-  data.tar.gz: da5c89c26fc95ddeaa6b8443193293490e71f41bc4ed21e00415c5937661833e
+  metadata.gz: 1a7193a9795eb785cd22a7663fa4dae908d3349b854be8d3613750f24ce6b711
+  data.tar.gz: 32a8172cc7115240f5954b7e4775f841ddce10102c7e48e184690d2643e7a476
 SHA512:
-  metadata.gz: 05e700dedeebde8e472e8adf6b0eedb6c3692b919b3d478a041c8e8110375daae7dd926cd9c33c46379976730f0fb0610880a57524aa44e0764fa169a4020fad
-  data.tar.gz: c0e954b6250223e53ec0823f328bd808886f9566ecf373ceaaba5da7a079ae76e18dfba3cf81bfad0741ea8dfdb87c1715c81f8af72094b77abe014fc59727ef
+  metadata.gz: 584a6fc4c10e6a33d9e55d9647abcfce913fa18b046eddf7193807408d7ac4167b1b7114aa300531426e767668841f483ad66775bf7f492ff552f290d19f6e2a
+  data.tar.gz: 877dea6a6ba8f3ab091ed6d5b0379090a60cff4477ad9d4409fa9fef9674bd39b9134682ac534d49535a21ed58382b02088fa00f83524d31add7c23919e9c0c4

data/bin/file-digests

@@ -1,5 +1,19 @@
 #!/usr/bin/env ruby
 
+#  Copyright 2020 Stanislav Senotrusov <stan@senotrusov.com>
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
 require 'file-digests'
 
 FileDigests.run_cli_utility

data/lib/file-digests.rb

@@ -1,3 +1,17 @@
+#  Copyright 2020 Stanislav Senotrusov <stan@senotrusov.com>
+#
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
 require "date"
 require "digest"
 require "fileutils"
@@ -8,7 +22,8 @@ require "set"
 require "sqlite3"
 
 class FileDigests
-  DIGEST_ALGORITHMS=["BLAKE2b512", "SHA3-256", "SHA512-256"]
+  VERSION = Gem.loaded_specs["file-digests"]&.version&.to_s
+  DIGEST_ALGORITHMS = ["BLAKE2b512", "SHA3-256", "SHA512-256"]
   LEGACY_DIGEST_ALGORITHMS = ["SHA512", "SHA256"]
 
   def self.canonical_digest_algorithm_name(string)
@@ -101,6 +116,7 @@ class FileDigests
 
   def initialize files_path, digest_database_path, options = {}
     @options = options
+    @user_input_wait_time = 0
 
     initialize_paths files_path, digest_database_path
     initialize_database
@@ -124,13 +140,13 @@ class FileDigests
 
   def initialize_paths files_path, digest_database_path
     @files_path = cleanup_path(files_path || ".")
-
     raise "Files path must be a readable directory" unless (File.directory?(@files_path) && File.readable?(@files_path))
+    @files_path = realpath_with_disk @files_path
 
     @digest_database_path = digest_database_path ? cleanup_path(digest_database_path) : @files_path
     @digest_database_path += ".file-digests.sqlite" if File.directory?(@digest_database_path)
     ensure_dir_exist @digest_database_path.dirname
-
+    @digest_database_path = realdirpath_with_disk @digest_database_path
     @digest_database_files = ["#{@digest_database_path}", "#{@digest_database_path}-wal", "#{@digest_database_path}-shm"]
 
     if @options[:verbose]
@@ -144,8 +160,6 @@ class FileDigests
     @db.results_as_hash = true
     @db.busy_timeout = 5000
 
-    file_digests_gem_version = Gem.loaded_specs["file-digests"]&.version&.to_s
-
     execute "PRAGMA encoding = 'UTF-8'"
     execute "PRAGMA locking_mode = 'EXCLUSIVE'"
     execute "PRAGMA journal_mode = 'WAL'"
@@ -160,14 +174,13 @@ class FileDigests
         execute "CREATE TABLE metadata (
           key TEXT NOT NULL PRIMARY KEY,
           value TEXT)"
-        execute "CREATE UNIQUE INDEX metadata_key ON metadata(key)"
         metadata_table_was_created = true
       end
 
       prepare_method :set_metadata_query, "INSERT INTO metadata (key, value) VALUES (?, ?) ON CONFLICT (key) DO UPDATE SET value=excluded.value"
       prepare_method :get_metadata_query, "SELECT value FROM metadata WHERE key = ?"
 
-      set_metadata("metadata_table_created_by_gem_version", file_digests_gem_version) if file_digests_gem_version && metadata_table_was_created
+      set_metadata("metadata_table_created_by_gem_version", FileDigests::VERSION) if FileDigests::VERSION && metadata_table_was_created
 
       # Heuristic to detect database version 1 (metadata was not stored back then)
       unless get_metadata("database_version")
@@ -184,20 +197,19 @@ class FileDigests
           digest TEXT NOT NULL,
           digest_check_time TEXT NOT NULL)"
         execute "CREATE UNIQUE INDEX digests_filename ON digests(filename)"
-        set_metadata("digests_table_created_by_gem_version", file_digests_gem_version) if file_digests_gem_version
+        execute "CREATE INDEX digests_digest ON digests(digest)"
+        set_metadata("digests_table_created_by_gem_version", FileDigests::VERSION) if FileDigests::VERSION
       end
 
-      prepare_method :insert, "INSERT INTO digests (filename, mtime, digest, digest_check_time) VALUES (?, ?, ?, datetime('now'))"
-      prepare_method :find_by_filename_query, "SELECT id, mtime, digest FROM digests WHERE filename = ?"
-      prepare_method :touch_digest_check_time, "UPDATE digests SET digest_check_time = datetime('now') WHERE id = ?"
-      prepare_method :update_mtime_and_digest, "UPDATE digests SET mtime = ?, digest = ?, digest_check_time = datetime('now') WHERE id = ?"
-      prepare_method :update_mtime, "UPDATE digests SET mtime = ?, digest_check_time = datetime('now') WHERE id = ?"
-      prepare_method :delete_by_filename, "DELETE FROM digests WHERE filename = ?"
-      prepare_method :query_duplicates, "SELECT digest, filename FROM digests WHERE digest IN (SELECT digest FROM digests GROUP BY digest HAVING count(*) > 1) ORDER BY digest, filename;"
-      prepare_method :update_digest_to_new_digest, "UPDATE digests SET digest = ? WHERE digest = ?"
+      prepare_method :digests_insert, "INSERT INTO digests (filename, mtime, digest, digest_check_time) VALUES (?, ?, ?, datetime('now'))"
+      prepare_method :digests_find_by_filename_query, "SELECT id, mtime, digest FROM digests WHERE filename = ?"
+      prepare_method :digests_touch_check_time, "UPDATE digests SET digest_check_time = datetime('now') WHERE id = ?"
+      prepare_method :digests_update_mtime_and_digest, "UPDATE digests SET mtime = ?, digest = ?, digest_check_time = datetime('now') WHERE id = ?"
+      prepare_method :digests_update_mtime, "UPDATE digests SET mtime = ?, digest_check_time = datetime('now') WHERE id = ?"
+      prepare_method :digests_select_duplicates, "SELECT digest, filename FROM digests WHERE digest IN (SELECT digest FROM digests GROUP BY digest HAVING count(*) > 1) ORDER BY digest, filename;"
 
       unless get_metadata("database_version")
-        set_metadata "database_version", "2"
+        set_metadata "database_version", "3"
       end
 
       # Convert database from 1st to 2nd version
@@ -212,79 +224,110 @@ class FileDigests
         end
       end
 
-      if get_metadata("database_version") != "2"
-        STDERR.puts "This version of file-digests (#{file_digests_gem_version || "unknown"}) is only compartible with the database version 2. Current database version is #{get_metadata("database_version")}. To use this database, please install appropriate version if file-digest."
-        raise "Incompatible database version"
+      if get_metadata("database_version") == "2"
+        execute "CREATE INDEX digests_digest ON digests(digest)"
+        set_metadata "database_version", "3"
       end
+  
+      check_if_database_is_at_certain_version "3"
+
+      create_temporary_tables
     end
   end
 
-  def perform_check
-    perhaps_transaction(@new_digest_algorithm, :exclusive) do
-      @counters = {good: 0, updated: 0, new: 0, renamed: 0, likely_damaged: 0, exceptions: 0}
-      @new_files = {}
-      @new_digests = {}
+  def create_temporary_tables
+    execute "CREATE TEMPORARY TABLE new_files (
+      filename TEXT NOT NULL PRIMARY KEY,
+      digest TEXT NOT NULL)"
+    execute "CREATE INDEX new_files_digest ON new_files(digest)"
+
+    prepare_method :new_files_insert, "INSERT INTO new_files (filename, digest) VALUES (?, ?)"
+    prepare_method :new_files_count_query, "SELECT count(*) FROM new_files"
+
+    execute "CREATE TEMPORARY TABLE missing_files (
+      filename TEXT NOT NULL PRIMARY KEY,
+      digest TEXT NOT NULL)"
+    execute "CREATE INDEX missing_files_digest ON missing_files(digest)"
 
-      @missing_files = Hash[@db.prepare("SELECT filename, digest FROM digests").execute!]
+    execute "INSERT INTO missing_files (filename, digest) SELECT filename, digest FROM digests"
 
-      measure_time do
-        walk_files do |filename|
+    prepare_method :missing_files_delete, "DELETE FROM missing_files WHERE filename = ?"
+    prepare_method :missing_files_delete_renamed_files, "DELETE FROM missing_files WHERE digest IN (SELECT digest FROM new_files)"
+    prepare_method :missing_files_select_all_filenames, "SELECT filename FROM missing_files ORDER BY filename"
+    prepare_method :missing_files_delete_all, "DELETE FROM missing_files"
+    prepare_method :missing_files_count_query, "SELECT count(*) FROM missing_files"
+
+    prepare_method :digests_delete_renamed_files, "DELETE FROM digests WHERE filename IN (SELECT filename FROM missing_files WHERE digest IN (SELECT digest FROM new_files))"
+    prepare_method :digests_delete_all_missing_files, "DELETE FROM digests WHERE filename IN (SELECT filename FROM missing_files)"
+
+    execute "CREATE TEMPORARY TABLE new_digests (
+      filename TEXT NOT NULL PRIMARY KEY,
+      digest TEXT NOT NULL)"
+
+    prepare_method :new_digests_insert, "INSERT INTO new_digests (filename, digest) VALUES (?, ?)"
+    prepare_method :digests_update_digests_to_new_digests, "INSERT INTO digests (filename, digest, digest_check_time) SELECT filename, digest, false FROM new_digests WHERE true ON CONFLICT (filename) DO UPDATE SET digest=excluded.digest"
+  end
+
+  def perform_check
+    measure_time do
+      perhaps_transaction(@new_digest_algorithm, :exclusive) do
+        @counters = {good: 0, updated: 0, renamed: 0, likely_damaged: 0, exceptions: 0}
+
+        walk_files(@files_path.to_s) do |filename|
           process_file filename
         end
-      end
 
-      nested_transaction do
-        puts "Tracking renames..." if @options[:verbose]
-        track_renames
-      end
+        nested_transaction do
+          puts "Tracking renames..." if @options[:verbose]
+          track_renames
+        end
 
-      if any_missing_files?
-        if any_exceptions?
-          STDERR.puts "Due to previously occurred errors, database cleanup from missing files will be skipped this time."
-        else
-          print_missing_files
-          if !@options[:test_only] && (@options[:auto] || confirm("Remove missing files from the database"))
-            nested_transaction do
-              puts "Removing missing files..." if @options[:verbose]
-              remove_missing_files
+        if any_missing_files?
+          if any_exceptions?
+            STDERR.puts "Due to previously occurred errors, missing files will not removed from the database."
+          else
+            print_missing_files
+            if !@options[:test_only] && (@options[:auto] || confirm("Remove missing files from the database"))
+              nested_transaction do
+                puts "Removing missing files..." if @options[:verbose]
+                remove_missing_files
+              end
             end
           end
         end
-      end
 
-      if @new_digest_algorithm && !@options[:test_only]
-        if any_missing_files? || any_likely_damaged? || any_exceptions?
-          STDERR.puts "ERROR: New digest algorithm will not be in effect until there are files that are missing, likely damaged, or processed with an exception."
-        else
-          puts "Updating database to a new digest algorithm..." if @options[:verbose]
-          @new_digests.each do |old_digest, new_digest|
-            update_digest_to_new_digest new_digest, old_digest
+        if @new_digest_algorithm && !@options[:test_only]
+          if any_missing_files? || any_likely_damaged? || any_exceptions?
+            STDERR.puts "ERROR: New digest algorithm will not be in effect until there are files that are missing, likely damaged, or processed with an exception."
+          else
+            puts "Updating database to a new digest algorithm..." if @options[:verbose]
+            digests_update_digests_to_new_digests
+            set_metadata "digest_algorithm", @new_digest_algorithm
+            puts "Transition to a new digest algorithm complete: #{@new_digest_algorithm}"
           end
-          set_metadata "digest_algorithm", @new_digest_algorithm
-          puts "Transition to a new digest algorithm complete: #{@new_digest_algorithm}"
         end
-      end
 
-      if any_likely_damaged? || any_exceptions?
-        STDERR.puts "PLEASE REVIEW ERRORS THAT WERE OCCURRED!"
-      end
+        if any_likely_damaged? || any_exceptions?
+          STDERR.puts "PLEASE REVIEW ERRORS THAT WERE OCCURRED!"
+        end
 
-      set_metadata(@options[:test_only] ? "latest_test_only_check_time" : "latest_complete_check_time", time_to_database(Time.now))
+        set_metadata(@options[:test_only] ? "latest_test_only_check_time" : "latest_complete_check_time", time_to_database(Time.now))
 
-      print_counters
-    end
-    
-    puts "Performing database maintenance..." if @options[:verbose]
-    execute "PRAGMA optimize"
-    execute "VACUUM"
-    execute "PRAGMA wal_checkpoint(TRUNCATE)"
+        print_counters
+      end
+      
+      puts "Performing database maintenance..." if @options[:verbose]
+      execute "PRAGMA optimize"
+      execute "VACUUM"
+      execute "PRAGMA wal_checkpoint(TRUNCATE)"
 
-    hide_database_files
+      hide_database_files
+    end
   end
 
   def show_duplicates
     current_digest = nil
-    query_duplicates.each do |found|
+    digests_select_duplicates.each do |found|
       if current_digest != found["digest"]
         puts "" if current_digest
         current_digest = found["digest"]
@@ -297,9 +340,11 @@ class FileDigests
   private
 
   def process_file filename
-    return if File.symlink? filename
+    perhaps_nt_filename = perhaps_nt_path filename
+
+    return if File.symlink? perhaps_nt_filename
 
-    stat = File.stat filename
+    stat = File.stat perhaps_nt_filename
 
     return if stat.blockdev?
     return if stat.chardev?
@@ -316,9 +361,10 @@ class FileDigests
 
     normalized_filename = filename.delete_prefix("#{@files_path.to_s}/").encode("utf-8", universal_newline: true).unicode_normalize(:nfkc)
     mtime_string = time_to_database stat.mtime
-    digest = get_file_digest(filename)
+    digest, new_digest = get_file_digest(perhaps_nt_filename)
 
     nested_transaction do
+      new_digests_insert(normalized_filename, new_digest) if new_digest
       process_file_indeed normalized_filename, mtime_string, digest
     end
 
@@ -336,15 +382,15 @@ class FileDigests
   end
 
   def process_previously_seen_file found, filename, mtime, digest
-    @missing_files.delete(filename)
+    missing_files_delete filename
     if found["digest"] == digest
       @counters[:good] += 1
       puts "GOOD: #{filename}" if @options[:verbose]
       unless @options[:test_only]
         if found["mtime"] == mtime
-          touch_digest_check_time found["id"]
+          digests_touch_check_time found["id"]
         else
-          update_mtime mtime, found["id"]
+          digests_update_mtime mtime, found["id"]
         end
       end
     else
@@ -355,18 +401,17 @@ class FileDigests
         @counters[:updated] += 1
         puts "UPDATED#{" (FATE ACCEPTED)" if found["mtime"] == mtime && @options[:accept_fate]}: #{filename}" unless @options[:quiet]
         unless @options[:test_only]
-          update_mtime_and_digest mtime, digest, found["id"]
+          digests_update_mtime_and_digest mtime, digest, found["id"]
         end
       end
     end
   end
 
   def process_new_file filename, mtime, digest
-    @counters[:new] += 1
     puts "NEW: #{filename}" unless @options[:quiet]
+    new_files_insert filename, digest
     unless @options[:test_only]
-      @new_files[filename] = digest
-      insert filename, mtime, digest
+      digests_insert filename, mtime, digest
     end
   end
 
@@ -374,29 +419,31 @@ class FileDigests
   # Renames and missing files
 
   def track_renames
-    @missing_files.delete_if do |filename, digest|
-      if @new_files.value?(digest)
-        @counters[:renamed] += 1
-        unless @options[:test_only]
-          delete_by_filename filename
-        end
-        true
-      end
+    unless @options[:test_only]
+      digests_delete_renamed_files
     end
+    missing_files_delete_renamed_files
+    @counters[:renamed] = @db.changes
   end
 
   def print_missing_files
     puts "\nMISSING FILES:"
-    @missing_files.sort.to_h.each do |filename, digest|
-      puts filename
+    missing_files_select_all_filenames.each do |record|
+      puts record["filename"]
     end
   end
 
   def remove_missing_files
-    @missing_files.each do |filename, digest|
-      delete_by_filename filename
-    end
-    @missing_files = {}
+    digests_delete_all_missing_files
+    missing_files_delete_all
+  end
+
+  def missing_files_count
+    missing_files_count_query!&.first&.first
+  end
+
+  def any_missing_files?
+    missing_files_count > 0
   end
 
 
@@ -407,6 +454,7 @@ class FileDigests
   end
 
   def integrity_check
+    puts "Checking database integrity..." if @options[:verbose]
     if execute("PRAGMA integrity_check")&.first&.fetch("integrity_check") != "ok"
       raise "Database integrity check failed"
     end
@@ -461,7 +509,7 @@ class FileDigests
   end
 
   def find_by_filename filename
-    result = find_by_filename_query filename
+    result = digests_find_by_filename_query filename
     found = result.next
     raise "Multiple records found" if result.next
     found
@@ -481,9 +529,37 @@ class FileDigests
     end
   end
 
+  def check_if_database_is_at_certain_version target_version
+    current_version = get_metadata("database_version")
+    if current_version != target_version
+      STDERR.puts "This version of file-digests (#{FileDigests::VERSION || "unknown"}) is only compartible with the database version #{target_version}. Current database version is #{current_version}. To use this database, please install appropriate version if file-digest."
+      raise "Incompatible database version"
+    end
+  end
+
+  def new_files_count
+    new_files_count_query!&.first&.first
+  end
+
 
   # Filesystem-related helpers
 
+  def realpath_with_disk path
+    path = path.realpath
+    if Gem.win_platform? && path.to_s[0] == "/"
+      return Pathname(Dir.pwd[0, 2] + path.to_s)
+    end
+    path
+  end
+
+  def realdirpath_with_disk path
+    path = path.realdirpath
+    if Gem.win_platform? && path.to_s[0] == "/"
+      return Pathname(Dir.pwd[0, 2] + path.to_s)
+    end
+    path
+  end
+
   def patch_path_string path
     Gem.win_platform? ? path.gsub(/\\/, "/") : path
   end
@@ -502,10 +578,28 @@ class FileDigests
     end
   end
 
-  def walk_files
-    puts "Gathering the list of files..." if @options[:verbose]
-    Dir.glob(@files_path + "**" + "*", File::FNM_DOTMATCH) do |filename|
-      yield filename
+  def walk_files(path, &block)
+    Dir.each_child(path, encoding: "UTF-8") do |item|
+      item = "#{path}#{File::SEPARATOR}#{item}"
+      test_item = perhaps_nt_path item
+      if File.readable?(test_item)
+        if File.directory?(test_item)
+          walk_files(item, &block)
+        else
+          yield item
+        end
+      else
+        STDERR.puts "ERROR: Directory entry is not readable: #{item}"
+        @counters[:exceptions] += 1
+      end
+    end
+  end
+
+  def perhaps_nt_path path
+    if Gem.win_platform?
+      "\\??\\#{path.gsub(/\//,"\\")}"
+    else
+      path
     end
   end
 
@@ -519,18 +613,13 @@ class FileDigests
         digest.update(buffer)
         new_digest.update(buffer) if @new_digest_algorithm
       end
-      @new_digests[digest.hexdigest] = new_digest.hexdigest if @new_digest_algorithm
-      return digest.hexdigest
+      return [digest.hexdigest, (new_digest.hexdigest if @new_digest_algorithm)]
     end
   end
 
 
   # Runtime state helpers
 
-  def any_missing_files?
-    @missing_files.length > 0
-  end
-
   def any_exceptions?
     @counters[:exceptions] > 0
   end
@@ -544,14 +633,17 @@ class FileDigests
   def confirm text
     if STDIN.tty? && STDOUT.tty?
       puts "#{text} (y/n)?"
-      STDIN.gets.strip.downcase == "y"
+      start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      result = (STDIN.gets.strip.downcase == "y")
+      @user_input_wait_time += (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start)
+      result
     end
   end
 
   def measure_time
     start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
     yield
-    elapsed = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start)
+    elapsed = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start) - @user_input_wait_time
     puts "Elapsed time: #{elapsed.to_i / 3600}h #{(elapsed.to_i % 3600) / 60}m #{"%.3f" % (elapsed % 60)}s" unless @options[:quiet]
   end
 
@@ -569,11 +661,14 @@ class FileDigests
   end
 
   def print_counters
+    missing_files_count_result = missing_files_count
+    new_files_count_result = new_files_count - @counters[:renamed]
+
     puts "#{@counters[:good]} file(s) passes digest check" if @counters[:good] > 0
     puts "#{@counters[:updated]} file(s) are updated" if @counters[:updated] > 0
-    puts "#{@counters[:new]} file(s) are new" if @counters[:new] > 0
+    puts "#{new_files_count_result} file(s) are new" if new_files_count_result > 0
     puts "#{@counters[:renamed]} file(s) are renamed" if @counters[:renamed] > 0
-    puts "#{@missing_files.length} file(s) are missing" if @missing_files.length > 0
+    puts "#{missing_files_count_result} file(s) are missing" if missing_files_count_result > 0
     puts "#{@counters[:likely_damaged]} file(s) are likely damaged (!)" if @counters[:likely_damaged] > 0
     puts "#{@counters[:exceptions]} file(s) had exceptions occured during processing (!)" if @counters[:exceptions] > 0
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: file-digests
 version: !ruby/object:Gem::Version
-  version: 0.0.33
+  version: 0.0.38
 platform: ruby
 authors:
 - Stanislav Senotrusov
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-10-15 00:00:00.000000000 Z
+date: 2020-10-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: openssl