file-digests 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 527ba8a1621dc40e2ee802f7db825aac55b3341d8bc85def9c9c655c25dbca0a
-  data.tar.gz: 576ac2b79f62f9da9976500e919e7e54a90f718bd1a7c9f95d2f0ae7acca524e
+  metadata.gz: 4b996011c4d3ae950a0081174ea86d0fe52136f58fd09c3e4543bfbdf3240098
+  data.tar.gz: 5055ba39b9d1e2cccd1cb89603e088ecf8052cb48bc36443be7b520ffc585e83
 SHA512:
-  metadata.gz: 4fbb07312ab1579633d42602ac6443ac694e4d19fb28a66ba9017c33dd36327cfc51fe015f784a8aa6b1df2488271dcf60971ad59280e1754ea0230f12533997
-  data.tar.gz: ea821a75f4090ff7c9b3416bfa3b29e9ba6732a21bf337d0023b1f93569b751af7f63413caf13e8c5d7609ecab723e0dfe5e96f2e62ec2eb443c7f01216f219b
+  metadata.gz: e4325a69e293cdf3b3007bbc100f163bf26ef0ed06da33e0fb58476fd59b2d8c57a6ee9984acd57eaab778b38e334d9d78cd2fa71bd5a63971804b060b0fa167
+  data.tar.gz: 02263b3b793bcdd3caf8cccd9b07a5232b001bd25d45709ead019be0a002ed5f6eb94ebbed3f3719bc5b7fceeb8b32a918e8ed59f15fdf345c78d0437eab7999

data/bin/file-digests-test

@@ -0,0 +1,213 @@
+#!/usr/bin/env ruby
+
+require 'date'
+require 'set'
+require 'digest'
+require 'fileutils'
+require 'pathname'
+require 'sqlite3'
+
+def ensure_dir_exists path
+  if File.exist?(path)
+    unless File.directory?(path)
+      raise "#{path} is not a directory"
+    end
+  else
+    FileUtils.mkdir_p path
+  end
+end
+
+def measure_time
+  start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+  yield
+  elapsed = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start).to_i
+  puts "Elapsed time: #{elapsed / 3600}h #{(elapsed % 3600) / 60}m #{elapsed % 60}s" if VERBOSE
+end
+
+def patch_path_string path
+  Gem.win_platform? ? path.gsub(/\\/, '/') : path
+end
+
+class DigestDatabase
+  def initialize path
+    @db = SQLite3::Database.new(path.to_s)
+
+    unless @db.execute("SELECT name FROM sqlite_master WHERE type='table' AND name = 'digests'").length == 1
+      @db.execute 'PRAGMA encoding = "UTF-8"'
+      @db.execute "CREATE TABLE digests (
+        id INTEGER PRIMARY KEY,
+        filename TEXT,
+        mtime TEXT,
+        digest TEXT,
+        digest_check_time TEXT)"
+      @db.execute "CREATE UNIQUE INDEX digests_filename ON digests(filename)"
+    end
+
+    @db.results_as_hash = true
+    @missing_files = Hash[@db.prepare("SELECT filename, digest FROM digests").execute!]
+    @new_files = {}
+
+
+    @insert = @db.prepare("INSERT INTO digests (filename, mtime, digest, digest_check_time) VALUES (?, ?, ?, datetime('now'))")
+    @find_by_filename = @db.prepare("SELECT id, mtime, digest FROM digests WHERE filename = ?")
+    @touch_digest_check_time = @db.prepare("UPDATE digests SET digest_check_time = datetime('now') WHERE id = ?")
+    @update_mtime_and_digest = @db.prepare("UPDATE digests SET mtime = ?, digest = ?, digest_check_time = datetime('now') WHERE id = ?")
+    @update_mtime = @db.prepare("UPDATE digests SET mtime = ?, digest_check_time = datetime('now') WHERE id = ?")
+    @delete_by_filename = @db.prepare("DELETE FROM digests WHERE filename = ?")
+  end
+
+  def insert_or_update file_path, mtime, digest
+    result = @find_by_filename.execute file_path
+
+    if found = result.next_hash
+      raise "Multiple records found" if result.next
+
+      @missing_files.delete(file_path)
+
+      if found['digest'] == digest
+        COUNTS[:good] += 1
+        puts "GOOD: #{file_path}" if VERBOSE
+        unless TEST_ONLY
+          if found['mtime'] == mtime
+            @touch_digest_check_time.execute found['id']
+          else
+            @update_mtime.execute mtime, found['id']
+          end
+        end
+      else
+        if found['mtime'] == mtime # Digest is different and mtime is the same
+          COUNTS[:likely_damaged] += 1
+          puts "LIKELY DAMAGED: #{file_path}"
+        else
+          COUNTS[:updated] += 1
+          puts "UPDATED: #{file_path}" if VERBOSE || TEST_ONLY
+          unless TEST_ONLY
+            @update_mtime_and_digest.execute mtime, digest, found['id']
+          end
+        end
+      end
+    else
+      COUNTS[:new] += 1
+      puts "NEW: #{file_path}" if VERBOSE || TEST_ONLY
+      unless TEST_ONLY
+        @new_files[file_path] = digest
+        @insert.execute! file_path, mtime, digest
+      end
+    end
+  end
+
+  def process_missing_files
+    @missing_files.delete_if do |filename, digest|
+      if @new_files.value?(digest)
+        COUNTS[:renamed] += 1
+        unless TEST_ONLY
+          @delete_by_filename.execute filename
+        end
+        true
+      end
+    end
+
+    if (COUNTS[:missing] = @missing_files.length) > 0
+      puts "MISSING FILES:"
+      @missing_files.sort.to_h.each do |filename, digest|
+        puts filename
+      end
+      unless TEST_ONLY
+        puts "Remove missing files from the database (y/n)?"
+        if STDIN.gets.strip == "y"
+          @missing_files.each do |filename, digest|
+            @delete_by_filename.execute filename
+          end
+        end
+      end
+    end
+  end
+end
+
+class Checker
+  def initialize files_path, digest_database_path
+    @files_path = files_path
+    ensure_dir_exists @files_path
+
+    if digest_database_path
+      @digest_database_path = digest_database_path
+      ensure_dir_exists @digest_database_path.dirname
+    else
+      @digest_database_path = @files_path + 'file-digests.sqlite'
+      @skip_file_digests_sqlite = true
+    end
+
+    @digest_database = DigestDatabase.new @digest_database_path
+  end
+
+  def check
+    walk_files do |filename|
+      begin
+        process_file filename
+      rescue => exception
+        COUNTS[:exceptions] += 1
+        STDERR.puts "EXCEPTION: #{filename}: #{exception.message}"
+      end
+    end
+
+    @digest_database.process_missing_files
+  end
+
+  def walk_files
+    Dir.glob(@files_path + '**' + '*', File::FNM_DOTMATCH) do |filename|
+      next unless File.file? filename
+      next if @skip_file_digests_sqlite && filename == 'file-digests.sqlite'
+      yield filename
+    end
+  end
+
+  def process_file filename
+    @digest_database.insert_or_update(
+      filename.delete_prefix(@files_path.to_s + '/'),
+      File.mtime(filename).utc.strftime('%Y-%m-%d %H:%M:%S'),
+      get_file_digest(filename)
+      )
+  end
+
+  def get_file_digest filename
+    File.open(filename, 'rb') do |io|
+      digest = Digest::SHA512.new
+      buffer = ""
+      while io.read(40960, buffer)
+        digest.update(buffer)
+      end
+      return digest.hexdigest
+    end
+  end
+
+end
+
+VERBOSE = (ENV["VERBOSE"] == "true")
+TEST_ONLY = true
+
+COUNTS = {good: 0, updated: 0, new: 0, missing: 0, renamed: 0, likely_damaged: 0, exceptions: 0}
+
+begin
+  if ARGV[0]
+    files_path = Pathname.new patch_path_string(ARGV[0])
+  else
+    files_path = Pathname.new patch_path_string(".")
+  end
+
+  digest_database_path = Pathname.new patch_path_string(ARGV[1]) if ARGV[1]
+
+  measure_time do
+    checker = Checker.new files_path, digest_database_path
+    checker.check
+  end
+
+  if COUNTS[:likely_damaged] > 0 || COUNTS[:exceptions] > 0
+    puts "ERRORS WERE OCCURRED, PLEASE CHECK FOR THEM"
+  end
+
+  puts COUNTS.inspect
+
+rescue => exception
+  STDERR.puts "EXCEPTION: #{exception.message}"
+  raise exception
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: file-digests
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - Stanislav Senotrusov
@@ -28,10 +28,12 @@ description: Calculate file digests and check for the possible file corruption
 email: stan@senotrusov.com
 executables:
 - file-digests
+- file-digests-test
 extensions: []
 extra_rdoc_files: []
 files:
 - bin/file-digests
+- bin/file-digests-test
 homepage: https://github.com/senotrusov/file-digests
 licenses:
 - Apache-2.0