file-digests 0.0.15 → 0.0.20

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 573cd697002f30d083625d586b4d03caf8c0f2cb763bed75e2bc9ac0a8b03612
-  data.tar.gz: 7cfbef9cdf7a9110c6b84b27591a498960515842675fabf6bf5512a78d7e54bc
+  metadata.gz: 053daa0145db845876fb37b63656102744c524034b87cb91e85f0bce4658aec9
+  data.tar.gz: eac730f10cbe5812078e3461b74b19a740611e9375dd7c34684b2a896e881fc4
 SHA512:
-  metadata.gz: 4916e667dccfb630a31fdaa74271a9c85a5f28a81f28143a4f5b406784e0e78652a29ff8e3fd9ff6507a5d90a4d9baacd2385f7de169f2c9dda6dc0f3b58866b
-  data.tar.gz: be5ae679122bbcffc703940b113e8d1e1057e46f7ce61ead1f52664f6b147bbd9c201d56b7faeafef29da30297cfde2e40f38c8e9400d615b9bbc6e630557259
+  metadata.gz: ce4894b25eb9ecbcbee2ae6c32a9e7bc09fce12286a31896c5d6093ed1ca0f0618e2eaf01facd464c3ebb52f1f9e327198d6a4abffae4f011f9c9d589e6c2d3c
+  data.tar.gz: bdd07cc7c958095b11b57174d8d98cd065d9a83289aeac5ce62dc604c9526f46ab512a7b229e1620c3ba0a49139cae341ac51b11af372e8395eb5a9bfbece1c6

data/bin/file-digests

@@ -1,8 +1,5 @@
 #!/usr/bin/env ruby
 
-QUIET = (ENV["QUIET"] == "true")
-TEST_ONLY = (ENV["TEST_ONLY"] == "true")
-
 require 'file-digests'
 
 FileDigests.perform_check

data/bin/file-digests-auto

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+ENV["AUTO"] = "true"
+
+require 'file-digests'
+
+FileDigests.perform_check

data/bin/file-digests-test

@@ -1,7 +1,6 @@
 #!/usr/bin/env ruby
 
-QUIET = (ENV["QUIET"] == "true")
-TEST_ONLY = true
+ENV["TEST_ONLY"] = "true"
 
 require 'file-digests'
 

data/lib/file-digests.rb

@@ -1,4 +1,3 @@
-
 require 'date'
 require 'set'
 require 'digest'
@@ -6,252 +5,289 @@ require 'fileutils'
 require 'pathname'
 require 'sqlite3'
 
-module FileDigests
+class FileDigests
 
   def self.perform_check
-    checker = Checker.new ARGV[0], ARGV[1]
-    checker.perform_check
+    options = {
+      auto: (ENV["AUTO"] == "true"),
+      quiet: (ENV["QUIET"] == "true"),
+      test_only: (ENV["TEST_ONLY"] == "true")
+    }
+    file_digests = self.new ARGV[0], ARGV[1], options
+    file_digests.perform_check
   end
 
-  class DigestDatabase
-    def initialize path
-      @db = SQLite3::Database.new path.to_s
-      @db.results_as_hash = true
-
-      execute 'PRAGMA journal_mode = "WAL"'
-      execute 'PRAGMA synchronous = "NORMAL"'
-      execute 'PRAGMA locking_mode = "EXCLUSIVE"'
-      execute 'PRAGMA cache_size = "5000"'
-
-      unless execute("SELECT name FROM sqlite_master WHERE type='table' AND name = 'digests'").length == 1
-        execute 'PRAGMA encoding = "UTF-8"'
-        execute "CREATE TABLE digests (
-          id INTEGER PRIMARY KEY,
-          filename TEXT,
-          mtime TEXT,
-          digest TEXT,
-          digest_check_time TEXT)"
-        execute "CREATE UNIQUE INDEX digests_filename ON digests(filename)"
-      end
+  def self.show_duplicates
+    file_digests = self.new ARGV[0], ARGV[1]
+    file_digests.show_duplicates
+  end
 
-      @missing_files = Hash[@db.prepare("SELECT filename, digest FROM digests").execute!]
-      @new_files = {}
+  def initialize files_path, digest_database_path, options = {}
+    @options = options
 
-      prepare_method :insert, "INSERT INTO digests (filename, mtime, digest, digest_check_time) VALUES (?, ?, ?, datetime('now'))"
-      prepare_method :find_by_filename, "SELECT id, mtime, digest FROM digests WHERE filename = ?"
-      prepare_method :touch_digest_check_time, "UPDATE digests SET digest_check_time = datetime('now') WHERE id = ?"
-      prepare_method :update_mtime_and_digest, "UPDATE digests SET mtime = ?, digest = ?, digest_check_time = datetime('now') WHERE id = ?"
-      prepare_method :update_mtime, "UPDATE digests SET mtime = ?, digest_check_time = datetime('now') WHERE id = ?"
-      prepare_method :delete_by_filename, "DELETE FROM digests WHERE filename = ?"
-    end
+    @files_path = cleanup_path(files_path || ".")
+    @prefix_to_remove = @files_path.to_s + '/'
 
-    def insert_or_update file_path, mtime, digest, counters
-      result = find_by_filename file_path
+    raise "Files path must be a readable directory" unless (File.directory?(@files_path) && File.readable?(@files_path))
 
-      if found = result.next_hash
-        raise "Multiple records found" if result.next
+    @digest_database_path = if digest_database_path
+      cleanup_path(digest_database_path)
+    else
+      @files_path + '.file-digests.sqlite'
+    end
 
-        @missing_files.delete(file_path)
+    if File.directory?(@digest_database_path)
+      @digest_database_path = @digest_database_path + '.file-digests.sqlite'
+    end
 
-        if found['digest'] == digest
-          counters[:good] += 1
-          # puts "GOOD: #{file_path}" unless QUIET
-          unless TEST_ONLY
-            if found['mtime'] == mtime
-              touch_digest_check_time found['id']
-            else
-              update_mtime mtime, found['id']
-            end
-          end
-        else
-          if found['mtime'] == mtime # Digest is different and mtime is the same
-            counters[:likely_damaged] += 1
-            STDERR.puts "LIKELY DAMAGED: #{file_path}"
-          else
-            counters[:updated] += 1
-            puts "UPDATED: #{file_path}" unless QUIET
-            unless TEST_ONLY
-              update_mtime_and_digest mtime, digest, found['id']
-            end
-          end
-        end
-      else
-        counters[:new] += 1
-        puts "NEW: #{file_path}" unless QUIET
-        unless TEST_ONLY
-          @new_files[file_path] = digest
-          insert file_path, mtime, digest
-        end
-      end
+    if @files_path == @digest_database_path.dirname
+      @skip_file_digests_sqlite = true
     end
 
-    def process_missing_files counters
-      @missing_files.delete_if do |filename, digest|
-        if @new_files.value?(digest)
-          counters[:renamed] += 1
-          unless TEST_ONLY
-            delete_by_filename filename
-          end
-          true
-        end
-      end
+    ensure_dir_exists @digest_database_path.dirname
 
-      if (counters[:missing] = @missing_files.length) > 0
-        puts "\nMISSING FILES:"
-        @missing_files.sort.to_h.each do |filename, digest|
-          puts filename
-        end
-        unless TEST_ONLY
-          puts "Remove missing files from the database (y/n)?"
-          if STDIN.gets.strip.downcase == "y"
-            @db.transaction do
-              @missing_files.each do |filename, digest|
-                delete_by_filename filename
-              end
-            end
-          end
-        end
-      end
+    # Please do not use this flag, support for sha512 is here for backward compatibility, and one day it will be removed.
+    if File.exist?(@digest_database_path.dirname + '.file-digests.sha512')
+      @use_sha512 = true
     end
 
-    private
+    initialize_database @digest_database_path
+  end
 
-    def execute *args, &block
-      @db.execute *args, &block
+  def initialize_database path
+    @db = SQLite3::Database.new path.to_s
+    @db.results_as_hash = true
+
+    execute 'PRAGMA journal_mode = "WAL"'
+    execute 'PRAGMA synchronous = "NORMAL"'
+    execute 'PRAGMA locking_mode = "EXCLUSIVE"'
+    execute 'PRAGMA cache_size = "5000"'
+
+    unless execute("SELECT name FROM sqlite_master WHERE type='table' AND name = 'digests'").length == 1
+      execute 'PRAGMA encoding = "UTF-8"'
+      execute "CREATE TABLE digests (
+        id INTEGER PRIMARY KEY,
+        filename TEXT,
+        mtime TEXT,
+        digest TEXT,
+        digest_check_time TEXT)"
+      execute "CREATE UNIQUE INDEX digests_filename ON digests(filename)"
     end
 
-    def prepare_method name, query
-      variable = "@#{name}"
-      instance_variable_set(variable, @db.prepare(query))
-      define_singleton_method name do |*args, &block|
-        instance_variable_get(variable).execute(*args, &block)
-      end
-    end
+    prepare_method :insert, "INSERT INTO digests (filename, mtime, digest, digest_check_time) VALUES (?, ?, ?, datetime('now'))"
+    prepare_method :find_by_filename, "SELECT id, mtime, digest FROM digests WHERE filename = ?"
+    prepare_method :touch_digest_check_time, "UPDATE digests SET digest_check_time = datetime('now') WHERE id = ?"
+    prepare_method :update_mtime_and_digest, "UPDATE digests SET mtime = ?, digest = ?, digest_check_time = datetime('now') WHERE id = ?"
+    prepare_method :update_mtime, "UPDATE digests SET mtime = ?, digest_check_time = datetime('now') WHERE id = ?"
+    prepare_method :delete_by_filename, "DELETE FROM digests WHERE filename = ?"
+    prepare_method :query_duplicates, "SELECT digest, filename FROM digests WHERE digest IN (SELECT digest FROM digests GROUP BY digest HAVING count(*) > 1) ORDER BY digest, filename;"
   end
 
-  class Checker
-    def initialize files_path, digest_database_path
-      @files_path = cleanup_path(files_path || ".")
-      @prefix_to_remove = @files_path.to_s + '/'
+  def perform_check
+    @counters = {good: 0, updated: 0, new: 0, missing: 0, renamed: 0, likely_damaged: 0, exceptions: 0}
+    @missing_files = Hash[@db.prepare("SELECT filename, digest FROM digests").execute!]
+    @new_files = {}
 
-      raise "Files path must be a readable directory" unless (File.directory?(@files_path) && File.readable?(@files_path))
-
-      @digest_database_path = if digest_database_path
-        cleanup_path(digest_database_path)
-      else
-        @files_path + '.file-digests.sqlite'
+    measure_time do
+      walk_files do |filename|
+        process_file filename
       end
+    end
 
-      if File.directory?(@digest_database_path)
-        @digest_database_path = @digest_database_path + '.file-digests.sqlite'
-      end
+    track_renames
 
-      if @files_path == @digest_database_path.dirname
-        @skip_file_digests_sqlite = true
+    if any_missing_files?
+      print_missing_files
+      if !@options[:test_only] && (@options[:auto] || confirm("Remove missing files from the database"))
+        remove_missing_files
       end
+    end
 
-      ensure_dir_exists @digest_database_path.dirname
+    if @counters[:likely_damaged] > 0 || @counters[:exceptions] > 0
+      STDERR.puts "ERRORS WERE OCCURRED"
+    end
 
-      if File.exist?(@digest_database_path.dirname + '.file-digests.sha512')
-        @use_sha512 = true
-      end
+    puts @counters.inspect
+  end
 
-      @digest_database = DigestDatabase.new @digest_database_path
-      @counters = {good: 0, updated: 0, new: 0, missing: 0, renamed: 0, likely_damaged: 0, exceptions: 0}
-    end
+  def show_duplicates
+    current_digest = nil
+    result = query_duplicates
 
-    def perform_check
-      measure_time do
-        walk_files do |filename|
-          process_file filename
-        end
+    while found = result.next_hash do
+      if current_digest != found['digest']
+        puts "" if current_digest
+        current_digest = found['digest']
+        puts "#{found['digest']}:"
       end
+      puts "  #{found['filename']}"
+    end
+  end
 
-      @digest_database.process_missing_files @counters
+  private
 
-      if @counters[:likely_damaged] > 0 || @counters[:exceptions] > 0
-        STDERR.puts "ERRORS WERE OCCURRED"
-      end
+  def process_file filename
+    return if File.symlink? filename
 
-      puts @counters.inspect
+    stat = File.stat filename
+
+    return if stat.blockdev?
+    return if stat.chardev?
+    return if stat.directory?
+    return if stat.pipe?
+    unless stat.readable?
+      raise "File is not readable"
+    end
+    return if stat.socket?
+
+    if @skip_file_digests_sqlite
+      basename = File.basename(filename)
+      return if basename == '.file-digests.sha512'
+      return if basename == '.file-digests.sqlite'
+      return if basename == '.file-digests.sqlite-wal'
+      return if basename == '.file-digests.sqlite-shm'
     end
 
-    private
+    insert_or_update(
+      filename.delete_prefix(@prefix_to_remove).encode('utf-8', universal_newline: true).unicode_normalize(:nfkc),
+      stat.mtime.utc.strftime('%Y-%m-%d %H:%M:%S'),
+      get_file_digest(filename)
+      )
+  rescue => exception
+    @counters[:exceptions] += 1
+    STDERR.puts "EXCEPTION: #{filename.encode('utf-8', universal_newline: true)}: #{exception.message}"
+  end
 
-    def process_file filename
-      return if File.symlink? filename
+  def patch_path_string path
+    Gem.win_platform? ? path.gsub(/\\/, '/') : path
+  end
 
-      stat = File.stat filename
+  def cleanup_path path
+    Pathname.new(patch_path_string(path)).cleanpath
+  end
 
-      return if stat.blockdev?
-      return if stat.chardev?
-      return if stat.directory?
-      return if stat.pipe?
-      unless stat.readable?
-        raise "File is not readable"
-      end
-      return if stat.socket?
-
-      if @skip_file_digests_sqlite
-        basename = File.basename(filename)
-        return if basename == '.file-digests.sha512'
-        return if basename == '.file-digests.sqlite'
-        return if basename == '.file-digests.sqlite-wal'
-        return if basename == '.file-digests.sqlite-shm'
+  def ensure_dir_exists path
+    if File.exist?(path)
+      unless File.directory?(path)
+        raise "#{path} is not a directory"
       end
+    else
+      FileUtils.mkdir_p path
+    end
+  end
 
-      @digest_database.insert_or_update(
-        filename.delete_prefix(@prefix_to_remove).encode('utf-8', universal_newline: true).unicode_normalize(:nfkc),
-        stat.mtime.utc.strftime('%Y-%m-%d %H:%M:%S'),
-        get_file_digest(filename),
-        @counters
-        )
-    rescue => exception
-      @counters[:exceptions] += 1
-      STDERR.puts "EXCEPTION: #{filename.encode('utf-8', universal_newline: true)}: #{exception.message}"
+  def walk_files
+    Dir.glob(@files_path + '**' + '*', File::FNM_DOTMATCH) do |filename|
+      yield filename
     end
+  end
 
-    def patch_path_string path
-      Gem.win_platform? ? path.gsub(/\\/, '/') : path
+  def get_file_digest filename
+    File.open(filename, 'rb') do |io|
+      digest = (@use_sha512 ? Digest::SHA512 : Digest::SHA256).new
+      buffer = ""
+      while io.read(40960, buffer)
+        digest.update(buffer)
+      end
+      return digest.hexdigest
     end
+  end
 
-    def cleanup_path path
-      Pathname.new(patch_path_string(digest_database_path)).cleanpath
+  def confirm text
+    if STDIN.tty? && STDOUT.tty?
+      puts "#{text} (y/n)?"
+      STDIN.gets.strip.downcase == "y"
     end
+  end
+
+  def measure_time
+    start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    yield
+    elapsed = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start).to_i
+    puts "Elapsed time: #{elapsed / 3600}h #{(elapsed % 3600) / 60}m #{elapsed % 60}s" unless @options[:quiet]
+  end
 
-    def ensure_dir_exists path
-      if File.exist?(path)
-        unless File.directory?(path)
-          raise "#{path} is not a directory"
+  def insert_or_update file_path, mtime, digest
+    result = find_by_filename file_path
+
+    if found = result.next_hash
+      raise "Multiple records found" if result.next
+
+      @missing_files.delete(file_path)
+
+      if found['digest'] == digest
+        @counters[:good] += 1
+        # puts "GOOD: #{file_path}" unless @options[:quiet]
+        unless @options[:test_only]
+          if found['mtime'] == mtime
+            touch_digest_check_time found['id']
+          else
+            update_mtime mtime, found['id']
+          end
         end
       else
-        FileUtils.mkdir_p path
+        if found['mtime'] == mtime # Digest is different and mtime is the same
+          @counters[:likely_damaged] += 1
+          STDERR.puts "LIKELY DAMAGED: #{file_path}"
+        else
+          @counters[:updated] += 1
+          puts "UPDATED: #{file_path}" unless @options[:quiet]
+          unless @options[:test_only]
+            update_mtime_and_digest mtime, digest, found['id']
+          end
+        end
       end
-    end
-
-    def walk_files
-      Dir.glob(@files_path + '**' + '*', File::FNM_DOTMATCH) do |filename|
-        yield filename
+    else
+      @counters[:new] += 1
+      puts "NEW: #{file_path}" unless @options[:quiet]
+      unless @options[:test_only]
+        @new_files[file_path] = digest
+        insert file_path, mtime, digest
       end
     end
+  end
 
-    def get_file_digest filename
-      File.open(filename, 'rb') do |io|
-        digest = (@use_sha512 ? Digest::SHA512 : Digest::SHA256).new
-        buffer = ""
-        while io.read(40960, buffer)
-          digest.update(buffer)
+  def track_renames
+    @missing_files.delete_if do |filename, digest|
+      if @new_files.value?(digest)
+        @counters[:renamed] += 1
+        unless @options[:test_only]
+          delete_by_filename filename
         end
-        return digest.hexdigest
+        true
       end
     end
+    @counters[:missing] = @missing_files.length
+  end
+
+  def any_missing_files?
+    @missing_files.length > 0
+  end
+
+  def print_missing_files
+    puts "\nMISSING FILES:"
+    @missing_files.sort.to_h.each do |filename, digest|
+      puts filename
+    end
+  end
 
-    def measure_time
-      start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
-      yield
-      elapsed = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start).to_i
-      puts "Elapsed time: #{elapsed / 3600}h #{(elapsed % 3600) / 60}m #{elapsed % 60}s" unless QUIET
+  def remove_missing_files
+    @db.transaction do
+      @missing_files.each do |filename, digest|
+        delete_by_filename filename
+      end
     end
+  end
+
+  def execute *args, &block
+    @db.execute *args, &block
+  end
 
+  def prepare_method name, query
+    variable = "@#{name}"
+    instance_variable_set(variable, @db.prepare(query))
+    define_singleton_method name do |*args, &block|
+      instance_variable_get(variable).execute(*args, &block)
+    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: file-digests
 version: !ruby/object:Gem::Version
-  version: 0.0.15
+  version: 0.0.20
 platform: ruby
 authors:
 - Stanislav Senotrusov
@@ -28,11 +28,13 @@ description: Calculate file digests and check for the possible file corruption
 email: stan@senotrusov.com
 executables:
 - file-digests
+- file-digests-auto
 - file-digests-test
 extensions: []
 extra_rdoc_files: []
 files:
 - bin/file-digests
+- bin/file-digests-auto
 - bin/file-digests-test
 - lib/file-digests.rb
 homepage: https://github.com/senotrusov/file-digests