file-digests 0.0.40 → 0.0.43

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a9d040599aee9aeb62234557b2a92edec36265e4439965e4c9d00a2c9afa117a
-  data.tar.gz: 992fe8a843afe761537a3c48b1ecde27ccdf8101be58dd4b456572c59232141b
+  metadata.gz: e74595cb920bd6b52f8f0a0efab118afae509dc5406389d5203518ce801f2811
+  data.tar.gz: ad4429b2d998dc82c441cb5ebd9b9f3640158f20abe0839ead0f0de57037854b
 SHA512:
-  metadata.gz: 1e15aa584690f8062a51cbb4785c9716e610258fba77859cb2b50642154701ffa1651312be1f6455921ee435ed7df5ed00bcc36f12def4009ac7edcc4aebb93e
-  data.tar.gz: 5c94ae4165677af0be7b790a7a92ffc9af7751f0dd2661bf238d06b02569ef4ec746908721c7d16781d11a431048759e1490f9e180ea9c2e647f5e585a3ebbe0
+  metadata.gz: 8c78ceabcbba358c5be5222681ef45d25acef175f4840022997d652c1740ad1c66a461b8ff2e618bb48a80f97783f7afa57d4281c41d7251c3aa94b4641b5079
+  data.tar.gz: b83b60f4d66e5a57ed7f5f44a6f997c6c47e5df38686a63459fcf7943dd95cda8f9e7cb41a717e071ec5cfef14459b201afc9f5e18f990364f2a7654566af351

data/lib/file-digests.rb

@@ -1,3 +1,5 @@
+# encoding: UTF-8
+
 #  Copyright 2020 Stanislav Senotrusov <stan@senotrusov.com>
 #
 #  Licensed under the Apache License, Version 2.0 (the "License");
@@ -17,7 +19,6 @@ require "digest"
 require "fileutils"
 require "openssl"
 require "optparse"
-require "pathname"
 require "set"
 require "sqlite3"
 
@@ -112,6 +113,7 @@ class FileDigests
 
     file_digests = self.new ARGV[0], ARGV[1], options
     file_digests.send(options[:action] || :perform_check)
+    file_digests.close_database
   end
 
   def initialize files_path, digest_database_path, options = {}
@@ -138,29 +140,109 @@ class FileDigests
     puts "Using #{@digest_algorithm} digest algorithm" if @options[:verbose]
   end
 
+  def perform_check
+    measure_time do
+      perhaps_transaction(@new_digest_algorithm, :exclusive) do
+        @counters = {good: 0, updated: 0, renamed: 0, likely_damaged: 0, exceptions: 0}
+
+        walk_files(@files_path) do |filename|
+          process_file filename
+        end
+
+        nested_transaction do
+          puts "Tracking renames..." if @options[:verbose]
+          track_renames
+        end
+
+        if any_missing_files?
+          if any_exceptions?
+            STDERR.puts "Due to previously occurred errors, missing files will not removed from the database."
+          else
+            report_missing_files
+            if !@options[:test_only] && (@options[:auto] || confirm("Remove missing files from the database"))
+              nested_transaction do
+                puts "Removing missing files..." if @options[:verbose]
+                remove_missing_files
+              end
+            end
+          end
+        end
+
+        if @new_digest_algorithm && !@options[:test_only]
+          if any_missing_files? || any_likely_damaged? || any_exceptions?
+            STDERR.puts "ERROR: New digest algorithm will not be in effect until there are files that are missing, likely damaged, or processed with an exception."
+          else
+            puts "Updating database to a new digest algorithm..." if @options[:verbose]
+            digests_update_digests_to_new_digests
+            set_metadata "digest_algorithm", @new_digest_algorithm
+            puts "Transition to a new digest algorithm complete: #{@new_digest_algorithm}"
+          end
+        end
+
+        if any_likely_damaged? || any_exceptions?
+          STDERR.puts "PLEASE REVIEW ERRORS THAT WERE OCCURRED!"
+          STDERR.puts "A list of errors is also saved in a file: #{@error_log_path}"
+        end
+
+        set_metadata(@options[:test_only] ? "latest_test_only_check_time" : "latest_complete_check_time", time_to_database(Time.now))
+
+        print_counters
+      end
+
+      puts "Performing database maintenance..." if @options[:verbose]
+      execute "PRAGMA optimize"
+      execute "VACUUM"
+      execute "PRAGMA wal_checkpoint(TRUNCATE)"
+    end
+  end
+
+  def show_duplicates
+    current_digest = nil
+    digests_select_duplicates.each do |found|
+      if current_digest != found["digest"]
+        puts "" if current_digest
+        current_digest = found["digest"]
+        puts "#{found["digest"]}:"
+      end
+      puts "  #{found["filename"]}"
+    end
+  end
+
+  def close_database
+    @statements.each(&:close)
+    @db.close
+    hide_database_files
+  end
+
+  private
+
   def initialize_paths files_path, digest_database_path
+    @files_path = realpath(files_path || ".")
+
+    unless File.directory?(@files_path) && File.readable?(@files_path)
+      raise "ERROR: Files path must be a readable directory"
+    end
+
     @start_time_filename_string = Time.now.strftime("%Y-%m-%d %H-%M-%S")
-    @files_path = cleanup_path(files_path || ".")
-    raise "ERROR: Files path must be a readable directory" unless (File.directory?(@files_path) && File.readable?(@files_path))
-    @files_path = realpath_with_disk @files_path
-
-    @error_log_path = @files_path + "file-digests errors #{@start_time_filename_string}.txt"
-    @missing_files_path = @files_path + "file-digests missing files #{@start_time_filename_string}.txt"
-
-    @digest_database_path = digest_database_path ? cleanup_path(digest_database_path) : @files_path
-    @digest_database_path += ".file-digests.sqlite" if File.directory?(@digest_database_path)
-    ensure_dir_exist @digest_database_path.dirname
-    @digest_database_path = realdirpath_with_disk @digest_database_path
-    
+
+    @error_log_path = "#{@files_path}#{File::SEPARATOR}file-digests errors #{@start_time_filename_string}.txt"
+    @missing_files_path = "#{@files_path}#{File::SEPARATOR}file-digests missing files #{@start_time_filename_string}.txt"
+
+    @digest_database_path = digest_database_path ? realdirpath(digest_database_path) : @files_path
+
+    if File.directory?(@digest_database_path)
+      @digest_database_path += "#{File::SEPARATOR}.file-digests.sqlite"
+    end
+
     @digest_database_files = [
-      "#{@digest_database_path}",
+      @digest_database_path,
       "#{@digest_database_path}-wal",
       "#{@digest_database_path}-shm"
     ]
 
     @skip_files = @digest_database_files + [
-      @error_log_path.to_s,
-      @missing_files_path.to_s
+      @error_log_path,
+      @missing_files_path
     ]
 
     if @options[:verbose]
@@ -170,9 +252,10 @@ class FileDigests
   end
 
   def initialize_database
-    @db = SQLite3::Database.new @digest_database_path.to_s
+    @db = SQLite3::Database.new @digest_database_path
     @db.results_as_hash = true
     @db.busy_timeout = 5000
+    @statements = []
 
     execute "PRAGMA encoding = 'UTF-8'"
     execute "PRAGMA locking_mode = 'EXCLUSIVE'"
@@ -229,7 +312,7 @@ class FileDigests
       # Convert database from 1st to 2nd version
       unless get_metadata("digest_algorithm")
         if get_metadata("database_version") == "1"
-          if File.exist?(@digest_database_path.dirname + ".file-digests.sha512")
+          if File.exist?("#{File.dirname(@digest_database_path)}#{File::SEPARATOR}.file-digests.sha512")
             set_metadata("digest_algorithm", "SHA512")
           else
             set_metadata("digest_algorithm", "SHA256")
@@ -242,7 +325,7 @@ class FileDigests
         execute "CREATE INDEX digests_digest ON digests(digest)"
         set_metadata "database_version", "3"
       end
-  
+
       check_if_database_is_at_certain_version "3"
 
       create_temporary_tables
@@ -282,81 +365,81 @@ class FileDigests
     prepare_method :digests_update_digests_to_new_digests, "INSERT INTO digests (filename, digest, digest_check_time) SELECT filename, digest, false FROM new_digests WHERE true ON CONFLICT (filename) DO UPDATE SET digest=excluded.digest"
   end
 
-  def perform_check
-    measure_time do
-      perhaps_transaction(@new_digest_algorithm, :exclusive) do
-        @counters = {good: 0, updated: 0, renamed: 0, likely_damaged: 0, exceptions: 0}
+  # Files
 
-        walk_files(@files_path.to_s) do |filename|
-          process_file filename
-        end
+  def realpath path
+    realxpath path, :realpath
+  end
 
-        nested_transaction do
-          puts "Tracking renames..." if @options[:verbose]
-          track_renames
-        end
+  def realdirpath path
+    realxpath path, :realdirpath
+  end
 
-        if any_missing_files?
-          if any_exceptions?
-            STDERR.puts "Due to previously occurred errors, missing files will not removed from the database."
-          else
-            report_missing_files
-            if !@options[:test_only] && (@options[:auto] || confirm("Remove missing files from the database"))
-              nested_transaction do
-                puts "Removing missing files..." if @options[:verbose]
-                remove_missing_files
-              end
-            end
-          end
-        end
+  def realxpath path, method_name
+    path = path.encode("utf-8")
 
-        if @new_digest_algorithm && !@options[:test_only]
-          if any_missing_files? || any_likely_damaged? || any_exceptions?
-            STDERR.puts "ERROR: New digest algorithm will not be in effect until there are files that are missing, likely damaged, or processed with an exception."
-          else
-            puts "Updating database to a new digest algorithm..." if @options[:verbose]
-            digests_update_digests_to_new_digests
-            set_metadata "digest_algorithm", @new_digest_algorithm
-            puts "Transition to a new digest algorithm complete: #{@new_digest_algorithm}"
-          end
-        end
+    if Gem.win_platform?
+      path = path.gsub(/\\/, "/")
+    end
 
-        if any_likely_damaged? || any_exceptions?
-          STDERR.puts "PLEASE REVIEW ERRORS THAT WERE OCCURRED!"
-        end
+    path = File.send(method_name, path).encode("utf-8")
 
-        set_metadata(@options[:test_only] ? "latest_test_only_check_time" : "latest_complete_check_time", time_to_database(Time.now))
+    if Gem.win_platform? && path[0] == "/"
+      path = Dir.pwd[0, 2].encode("utf-8") + path
+    end
 
-        print_counters
-      end
-      
-      puts "Performing database maintenance..." if @options[:verbose]
-      execute "PRAGMA optimize"
-      execute "VACUUM"
-      execute "PRAGMA wal_checkpoint(TRUNCATE)"
+    path
+  end
 
-      hide_database_files
+  def perhaps_nt_path path
+    if Gem.win_platform?
+      "\\??\\#{path.gsub(/\//,"\\")}"
+    else
+      path
     end
   end
 
-  def show_duplicates
-    current_digest = nil
-    digests_select_duplicates.each do |found|
-      if current_digest != found["digest"]
-        puts "" if current_digest
-        current_digest = found["digest"]
-        puts "#{found["digest"]}:"
+  def get_file_digest filename
+    File.open(filename, "rb") do |io|
+      digest = OpenSSL::Digest.new(@digest_algorithm)
+      new_digest = OpenSSL::Digest.new(@new_digest_algorithm) if @new_digest_algorithm
+
+      buffer = ""
+      while io.read(409600, buffer) # 409600 seems like a sweet spot
+        digest.update(buffer)
+        new_digest.update(buffer) if @new_digest_algorithm
       end
-      puts "  #{found["filename"]}"
+      return [digest.hexdigest, (new_digest.hexdigest if @new_digest_algorithm)]
     end
   end
 
-  private
+  def walk_files(path, &block)
+    Dir.each_child(path, encoding: "UTF-8") do |item|
+      item = "#{path}#{File::SEPARATOR}#{item.encode("utf-8")}"
+      begin
+        item_perhaps_nt_path = perhaps_nt_path item
+
+        unless File.symlink? item_perhaps_nt_path
+          if File.directory?(item_perhaps_nt_path)
+            raise "Directory is not readable" unless File.readable?(item_perhaps_nt_path)
+            walk_files(item, &block)
+          else
+            yield item
+          end
+        end
+
+      rescue => exception
+        @counters[:exceptions] += 1
+        report_file_exception exception, item
+      end
+    end
+  end
 
   def process_file filename
     perhaps_nt_filename = perhaps_nt_path filename
 
-    return if File.symlink? perhaps_nt_filename
+    # this is checked in the walk_files
+    # return if File.symlink? perhaps_nt_filename
 
     stat = File.stat perhaps_nt_filename
 
@@ -373,7 +456,7 @@ class FileDigests
       return
     end
 
-    normalized_filename = filename.delete_prefix("#{@files_path.to_s}/").encode("utf-8", universal_newline: true).unicode_normalize(:nfkc)
+    normalized_filename = filename.delete_prefix("#{@files_path}#{File::SEPARATOR}").encode("utf-8", universal_newline: true).unicode_normalize(:nfkc)
     mtime_string = time_to_database stat.mtime
     digest, new_digest = get_file_digest(perhaps_nt_filename)
 
@@ -443,6 +526,7 @@ class FileDigests
       File.open(@missing_files_path, "a") do |f|
         write_missing_files f
       end
+      puts "\n(A list of missing files is also saved in a file: #{@missing_files_path})"
     end
   end
 
@@ -466,6 +550,17 @@ class FileDigests
   end
 
 
+  # Runtime state helpers
+
+  def any_exceptions?
+    @counters[:exceptions] > 0
+  end
+
+  def any_likely_damaged?
+    @counters[:likely_damaged] > 0
+  end
+
+
   # Database helpers
 
   def execute *args, &block
@@ -506,7 +601,10 @@ class FileDigests
   def prepare_method name, query
     variable = "@#{name}"
 
-    instance_variable_set(variable, @db.prepare(query))
+    statement = @db.prepare(query)
+    @statements.push(statement)
+
+    instance_variable_set(variable, statement)
 
     define_singleton_method name do |*args, &block|
       instance_variable_get(variable).execute(*args, &block)
@@ -561,96 +659,6 @@ class FileDigests
   end
 
 
-  # Filesystem-related helpers
-
-  def realpath_with_disk path
-    path = path.realpath
-    if Gem.win_platform? && path.to_s[0] == "/"
-      return Pathname(Dir.pwd[0, 2] + path.to_s)
-    end
-    path
-  end
-
-  def realdirpath_with_disk path
-    path = path.realdirpath
-    if Gem.win_platform? && path.to_s[0] == "/"
-      return Pathname(Dir.pwd[0, 2] + path.to_s)
-    end
-    path
-  end
-
-  def patch_path_string path
-    Gem.win_platform? ? path.gsub(/\\/, "/") : path
-  end
-
-  def cleanup_path path
-    Pathname.new(patch_path_string(path)).cleanpath
-  end
-
-  def ensure_dir_exist path
-    if File.exist?(path)
-      unless File.directory?(path)
-        raise "#{path} is not a directory"
-      end
-    else
-      FileUtils.mkdir_p path
-    end
-  end
-
-  def walk_files(path, &block)
-    Dir.each_child(path, encoding: "UTF-8") do |item|
-      item = "#{path}#{File::SEPARATOR}#{item}"
-      begin
-        item_perhaps_nt_path = perhaps_nt_path item
-        
-        unless File.symlink? item_perhaps_nt_path
-          if File.directory?(item_perhaps_nt_path)
-            raise "Directory is not readable" unless File.readable?(item_perhaps_nt_path)
-            walk_files(item, &block)
-          else
-            yield item
-          end
-        end
-      rescue => exception
-        @counters[:exceptions] += 1
-        report_file_exception exception, item
-      end
-    end
-  end
-
-  def perhaps_nt_path path
-    if Gem.win_platform?
-      "\\??\\#{path.gsub(/\//,"\\")}"
-    else
-      path
-    end
-  end
-
-  def get_file_digest filename
-    File.open(filename, "rb") do |io|
-      digest = OpenSSL::Digest.new(@digest_algorithm)
-      new_digest = OpenSSL::Digest.new(@new_digest_algorithm) if @new_digest_algorithm
-
-      buffer = ""
-      while io.read(409600, buffer) # 409600 seems like a sweet spot
-        digest.update(buffer)
-        new_digest.update(buffer) if @new_digest_algorithm
-      end
-      return [digest.hexdigest, (new_digest.hexdigest if @new_digest_algorithm)]
-    end
-  end
-
-
-  # Runtime state helpers
-
-  def any_exceptions?
-    @counters[:exceptions] > 0
-  end
-
-  def any_likely_damaged?
-    @counters[:likely_damaged] > 0
-  end
-
   # UI helpers
 
   def confirm text

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: file-digests
 version: !ruby/object:Gem::Version
-  version: 0.0.40
+  version: 0.0.43
 platform: ruby
 authors:
 - Stanislav Senotrusov
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-10-20 00:00:00.000000000 Z
+date: 2022-07-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: openssl
@@ -16,28 +16,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: '3.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: sqlite3
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '1.4'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '1.4'
 description: Calculate file digests and check for the possible file corruption
 email: stan@senotrusov.com
 executables:
@@ -51,7 +51,7 @@ homepage: https://github.com/senotrusov/file-digests
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -66,8 +66,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.2
-signing_key: 
+rubygems_version: 3.3.7
+signing_key:
 specification_version: 4
 summary: file-digests
 test_files: []