file-digests 0.0.2 → 0.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 527ba8a1621dc40e2ee802f7db825aac55b3341d8bc85def9c9c655c25dbca0a
-  data.tar.gz: 576ac2b79f62f9da9976500e919e7e54a90f718bd1a7c9f95d2f0ae7acca524e
+  metadata.gz: 182ed181e50c1af8f9c9d395a4ba2536b78b69c6996e7d5852277b5bd19f3d88
+  data.tar.gz: 772b324b0821699e451db68b2859506b56d975843e925601afc3889fe94f0631
 SHA512:
-  metadata.gz: 4fbb07312ab1579633d42602ac6443ac694e4d19fb28a66ba9017c33dd36327cfc51fe015f784a8aa6b1df2488271dcf60971ad59280e1754ea0230f12533997
-  data.tar.gz: ea821a75f4090ff7c9b3416bfa3b29e9ba6732a21bf337d0023b1f93569b751af7f63413caf13e8c5d7609ecab723e0dfe5e96f2e62ec2eb443c7f01216f219b
+  metadata.gz: 448f9c73ea520b6a68165c6b94f3509ef561085c069613d8bdc5e34c0f40f61073cb2b105d0bb488c7a4dba00cd8c87a693ec3fe59db74f2f6ed54784cbe4c8e
+  data.tar.gz: cf10a1d008765c3f0296401e6c2d3f0d90acb24aa94664be313657e5f698e40ea457b7e38e65cc74815689c3e1764708ed579953ba28ccdb894163914d614c3e

data/bin/file-digests

@@ -21,7 +21,7 @@ def measure_time
   start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
   yield
   elapsed = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start).to_i
-  puts "Elapsed time: #{elapsed / 3600}h #{(elapsed % 3600) / 60}m #{elapsed % 60}s" if VERBOSE
+  puts "Elapsed time: #{elapsed / 3600}h #{(elapsed % 3600) / 60}m #{elapsed % 60}s" unless QUIET
 end
 
 def patch_path_string path
@@ -43,6 +43,11 @@ class DigestDatabase
       @db.execute "CREATE UNIQUE INDEX digests_filename ON digests(filename)"
     end
 
+    @db.execute 'PRAGMA journal_mode = "WAL"'
+    @db.execute 'PRAGMA synchronous = "NORMAL"'
+    @db.execute 'PRAGMA locking_mode = "EXCLUSIVE"'
+    @db.execute 'PRAGMA cache_size = "5000"'
+
     @db.results_as_hash = true
     @missing_files = Hash[@db.prepare("SELECT filename, digest FROM digests").execute!]
     @new_files = {}
@@ -66,7 +71,7 @@ class DigestDatabase
 
       if found['digest'] == digest
         COUNTS[:good] += 1
-        puts "GOOD: #{file_path}" if VERBOSE
+        # puts "GOOD: #{file_path}" unless QUIET
         unless TEST_ONLY
           if found['mtime'] == mtime
             @touch_digest_check_time.execute found['id']
@@ -77,10 +82,10 @@ class DigestDatabase
       else
         if found['mtime'] == mtime # Digest is different and mtime is the same
           COUNTS[:likely_damaged] += 1
-          puts "LIKELY DAMAGED: #{file_path}"
+          STDERR.puts "LIKELY DAMAGED: #{file_path}"
         else
           COUNTS[:updated] += 1
-          puts "UPDATED: #{file_path}" if VERBOSE || TEST_ONLY
+          puts "UPDATED: #{file_path}" unless QUIET
           unless TEST_ONLY
             @update_mtime_and_digest.execute mtime, digest, found['id']
           end
@@ -88,7 +93,7 @@ class DigestDatabase
       end
     else
       COUNTS[:new] += 1
-      puts "NEW: #{file_path}" if VERBOSE || TEST_ONLY
+      puts "NEW: #{file_path}" unless QUIET
       unless TEST_ONLY
         @new_files[file_path] = digest
         @insert.execute! file_path, mtime, digest
@@ -133,7 +138,7 @@ class Checker
       @digest_database_path = digest_database_path
       ensure_dir_exists @digest_database_path.dirname
     else
-      @digest_database_path = @files_path + 'file-digests.sqlite'
+      @digest_database_path = @files_path + '.file-digests.sqlite'
       @skip_file_digests_sqlite = true
     end
 
@@ -156,7 +161,8 @@ class Checker
   def walk_files
     Dir.glob(@files_path + '**' + '*', File::FNM_DOTMATCH) do |filename|
       next unless File.file? filename
-      next if @skip_file_digests_sqlite && filename == 'file-digests.sqlite'
+      next if @skip_file_digests_sqlite && filename == '.file-digests.sqlite'
+      next if @skip_file_digests_sqlite && filename == '.file-digests.sqlite-wal'
       yield filename
     end
   end
@@ -182,7 +188,7 @@ class Checker
 
 end
 
-VERBOSE = (ENV["VERBOSE"] == "true")
+QUIET = (ENV["QUIET"] == "true")
 TEST_ONLY = (ENV["TEST_ONLY"] == "true")
 
 COUNTS = {good: 0, updated: 0, new: 0, missing: 0, renamed: 0, likely_damaged: 0, exceptions: 0}
@@ -202,7 +208,7 @@ begin
   end
 
   if COUNTS[:likely_damaged] > 0 || COUNTS[:exceptions] > 0
-    puts "ERRORS WERE OCCURRED, PLEASE CHECK FOR THEM"
+    STDERR.puts "ERRORS WERE OCCURRED"
   end
 
   puts COUNTS.inspect

data/bin/file-digests-test

@@ -0,0 +1,219 @@
+#!/usr/bin/env ruby
+
+require 'date'
+require 'set'
+require 'digest'
+require 'fileutils'
+require 'pathname'
+require 'sqlite3'
+
+def ensure_dir_exists path
+  if File.exist?(path)
+    unless File.directory?(path)
+      raise "#{path} is not a directory"
+    end
+  else
+    FileUtils.mkdir_p path
+  end
+end
+
+def measure_time
+  start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+  yield
+  elapsed = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start).to_i
+  puts "Elapsed time: #{elapsed / 3600}h #{(elapsed % 3600) / 60}m #{elapsed % 60}s" unless QUIET
+end
+
+def patch_path_string path
+  Gem.win_platform? ? path.gsub(/\\/, '/') : path
+end
+
+class DigestDatabase
+  def initialize path
+    @db = SQLite3::Database.new(path.to_s)
+
+    unless @db.execute("SELECT name FROM sqlite_master WHERE type='table' AND name = 'digests'").length == 1
+      @db.execute 'PRAGMA encoding = "UTF-8"'
+      @db.execute "CREATE TABLE digests (
+        id INTEGER PRIMARY KEY,
+        filename TEXT,
+        mtime TEXT,
+        digest TEXT,
+        digest_check_time TEXT)"
+      @db.execute "CREATE UNIQUE INDEX digests_filename ON digests(filename)"
+    end
+
+    @db.execute 'PRAGMA journal_mode = "WAL"'
+    @db.execute 'PRAGMA synchronous = "NORMAL"'
+    @db.execute 'PRAGMA locking_mode = "EXCLUSIVE"'
+    @db.execute 'PRAGMA cache_size = "5000"'
+
+    @db.results_as_hash = true
+    @missing_files = Hash[@db.prepare("SELECT filename, digest FROM digests").execute!]
+    @new_files = {}
+
+
+    @insert = @db.prepare("INSERT INTO digests (filename, mtime, digest, digest_check_time) VALUES (?, ?, ?, datetime('now'))")
+    @find_by_filename = @db.prepare("SELECT id, mtime, digest FROM digests WHERE filename = ?")
+    @touch_digest_check_time = @db.prepare("UPDATE digests SET digest_check_time = datetime('now') WHERE id = ?")
+    @update_mtime_and_digest = @db.prepare("UPDATE digests SET mtime = ?, digest = ?, digest_check_time = datetime('now') WHERE id = ?")
+    @update_mtime = @db.prepare("UPDATE digests SET mtime = ?, digest_check_time = datetime('now') WHERE id = ?")
+    @delete_by_filename = @db.prepare("DELETE FROM digests WHERE filename = ?")
+  end
+
+  def insert_or_update file_path, mtime, digest
+    result = @find_by_filename.execute file_path
+
+    if found = result.next_hash
+      raise "Multiple records found" if result.next
+
+      @missing_files.delete(file_path)
+
+      if found['digest'] == digest
+        COUNTS[:good] += 1
+        # puts "GOOD: #{file_path}" unless QUIET
+        unless TEST_ONLY
+          if found['mtime'] == mtime
+            @touch_digest_check_time.execute found['id']
+          else
+            @update_mtime.execute mtime, found['id']
+          end
+        end
+      else
+        if found['mtime'] == mtime # Digest is different and mtime is the same
+          COUNTS[:likely_damaged] += 1
+          STDERR.puts "LIKELY DAMAGED: #{file_path}"
+        else
+          COUNTS[:updated] += 1
+          puts "UPDATED: #{file_path}" unless QUIET
+          unless TEST_ONLY
+            @update_mtime_and_digest.execute mtime, digest, found['id']
+          end
+        end
+      end
+    else
+      COUNTS[:new] += 1
+      puts "NEW: #{file_path}" unless QUIET
+      unless TEST_ONLY
+        @new_files[file_path] = digest
+        @insert.execute! file_path, mtime, digest
+      end
+    end
+  end
+
+  def process_missing_files
+    @missing_files.delete_if do |filename, digest|
+      if @new_files.value?(digest)
+        COUNTS[:renamed] += 1
+        unless TEST_ONLY
+          @delete_by_filename.execute filename
+        end
+        true
+      end
+    end
+
+    if (COUNTS[:missing] = @missing_files.length) > 0
+      puts "MISSING FILES:"
+      @missing_files.sort.to_h.each do |filename, digest|
+        puts filename
+      end
+      unless TEST_ONLY
+        puts "Remove missing files from the database (y/n)?"
+        if STDIN.gets.strip == "y"
+          @missing_files.each do |filename, digest|
+            @delete_by_filename.execute filename
+          end
+        end
+      end
+    end
+  end
+end
+
+class Checker
+  def initialize files_path, digest_database_path
+    @files_path = files_path
+    ensure_dir_exists @files_path
+
+    if digest_database_path
+      @digest_database_path = digest_database_path
+      ensure_dir_exists @digest_database_path.dirname
+    else
+      @digest_database_path = @files_path + '.file-digests.sqlite'
+      @skip_file_digests_sqlite = true
+    end
+
+    @digest_database = DigestDatabase.new @digest_database_path
+  end
+
+  def check
+    walk_files do |filename|
+      begin
+        process_file filename
+      rescue => exception
+        COUNTS[:exceptions] += 1
+        STDERR.puts "EXCEPTION: #{filename}: #{exception.message}"
+      end
+    end
+
+    @digest_database.process_missing_files
+  end
+
+  def walk_files
+    Dir.glob(@files_path + '**' + '*', File::FNM_DOTMATCH) do |filename|
+      next unless File.file? filename
+      next if @skip_file_digests_sqlite && filename == '.file-digests.sqlite'
+      next if @skip_file_digests_sqlite && filename == '.file-digests.sqlite-wal'
+      yield filename
+    end
+  end
+
+  def process_file filename
+    @digest_database.insert_or_update(
+      filename.delete_prefix(@files_path.to_s + '/'),
+      File.mtime(filename).utc.strftime('%Y-%m-%d %H:%M:%S'),
+      get_file_digest(filename)
+      )
+  end
+
+  def get_file_digest filename
+    File.open(filename, 'rb') do |io|
+      digest = Digest::SHA512.new
+      buffer = ""
+      while io.read(40960, buffer)
+        digest.update(buffer)
+      end
+      return digest.hexdigest
+    end
+  end
+
+end
+
+QUIET = (ENV["QUIET"] == "true")
+TEST_ONLY = true
+
+COUNTS = {good: 0, updated: 0, new: 0, missing: 0, renamed: 0, likely_damaged: 0, exceptions: 0}
+
+begin
+  if ARGV[0]
+    files_path = Pathname.new patch_path_string(ARGV[0])
+  else
+    files_path = Pathname.new patch_path_string(".")
+  end
+
+  digest_database_path = Pathname.new patch_path_string(ARGV[1]) if ARGV[1]
+
+  measure_time do
+    checker = Checker.new files_path, digest_database_path
+    checker.check
+  end
+
+  if COUNTS[:likely_damaged] > 0 || COUNTS[:exceptions] > 0
+    STDERR.puts "ERRORS WERE OCCURRED"
+  end
+
+  puts COUNTS.inspect
+
+rescue => exception
+  STDERR.puts "EXCEPTION: #{exception.message}"
+  raise exception
+end

metadata

@@ -1,11 +1,11 @@
 --- !ruby/object:Gem::Specification
 name: file-digests
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.7
 platform: ruby
 authors:
 - Stanislav Senotrusov
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
 date: 2020-10-08 00:00:00.000000000 Z
@@ -28,15 +28,17 @@ description: Calculate file digests and check for the possible file corruption
 email: stan@senotrusov.com
 executables:
 - file-digests
+- file-digests-test
 extensions: []
 extra_rdoc_files: []
 files:
 - bin/file-digests
+- bin/file-digests-test
 homepage: https://github.com/senotrusov/file-digests
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -52,7 +54,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.1.2
-signing_key: 
+signing_key:
 specification_version: 4
 summary: file-digests
 test_files: []