file-digests 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 37a0004a1a028f687a5377117734ff57b25c5be9a6788a26adba72a188da5342
+  data.tar.gz: 357a8e73ad41eb05e3e05655e067c45bdd09b2c907a31847d4f087145d5b6f6f
+SHA512:
+  metadata.gz: bcd30f2bc61982f0c5e84a80efa559ccb4d045dae8490e4ecc845af2cc524988abb1bbd9492803b0cbbcfe5a8c29e47db99ef222cc615f392cd9f795d7fdcdbe
+  data.tar.gz: 2a71ac2f2a6c42414c05885bfa902d3c7037021bc030e7b06559105e4e4665512a214e9298ae4d8ae5fd32fe530e466fb3f03d03648d1c5b7072c5733b106cb7

data/bin/file-digests

@@ -0,0 +1,213 @@
+#!/usr/bin/env ruby
+
+require 'date'
+require 'set'
+require 'digest'
+require 'fileutils'
+require 'pathname'
+require 'sqlite3'
+
+def ensure_dir_exists path
+  if File.exist?(path)
+    unless File.directory?(path)
+      raise "#{path} is not a directory"
+    end
+  else
+    FileUtils.mkdir_p path
+  end
+end
+
+def measure_time
+  start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+  yield
+  elapsed = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start).to_i
+  puts "Elapsed time: #{elapsed / 3600}h #{(elapsed % 3600) / 60}m #{elapsed % 60}s" if VERBOSE
+end
+
+def patch_path_string path
+  Gem.win_platform? ? path.gsub(/\\/, '/') : path
+end
+
+class DigestDatabase
+  def initialize path
+    @db = SQLite3::Database.new(path.to_s)
+
+    unless @db.execute("SELECT name FROM sqlite_master WHERE type='table' AND name = 'digests'").length == 1
+      @db.execute 'PRAGMA encoding = "UTF-8"'
+      @db.execute "CREATE TABLE digests (
+        id INTEGER PRIMARY KEY,
+        filename TEXT,
+        mtime TEXT,
+        digest TEXT,
+        digest_check_time TEXT)"
+      @db.execute "CREATE UNIQUE INDEX digests_filename ON digests(filename)"
+    end
+
+    @db.results_as_hash = true
+    @missing_files = Hash[@db.prepare("SELECT filename, digest FROM digests").execute!]
+    @new_files = {}
+
+
+    @insert = @db.prepare("INSERT INTO digests (filename, mtime, digest, digest_check_time) VALUES (?, ?, ?, datetime('now'))")
+    @find_by_filename = @db.prepare("SELECT id, mtime, digest FROM digests WHERE filename = ?")
+    @touch_digest_check_time = @db.prepare("UPDATE digests SET digest_check_time = datetime('now') WHERE id = ?")
+    @update_mtime_and_digest = @db.prepare("UPDATE digests SET mtime = ?, digest = ?, digest_check_time = datetime('now') WHERE id = ?")
+    @update_mtime = @db.prepare("UPDATE digests SET mtime = ?, digest_check_time = datetime('now') WHERE id = ?")
+    @delete_by_filename = @db.prepare("DELETE FROM digests WHERE filename = ?")
+  end
+
+  def insert_or_update file_path, mtime, digest
+    result = @find_by_filename.execute file_path
+
+    if found = result.next_hash
+      raise "Multiple records found" if result.next
+
+      @missing_files.delete(file_path)
+
+      if found['digest'] == digest
+        COUNTS[:good] += 1
+        puts "GOOD: #{file_path}" if VERBOSE
+        unless TEST_ONLY
+          if found['mtime'] == mtime
+            @touch_digest_check_time.execute found['id']
+          else
+            @update_mtime.execute mtime, found['id']
+          end
+        end
+      else
+        if found['mtime'] == mtime # Digest is different and mtime is the same
+          COUNTS[:likely_damaged] += 1
+          puts "LIKELY DAMAGED: #{file_path}"
+        else
+          COUNTS[:updated] += 1
+          puts "UPDATED: #{file_path}" if VERBOSE || TEST_ONLY
+          unless TEST_ONLY
+            @update_mtime_and_digest.execute mtime, digest, found['id']
+          end
+        end
+      end
+    else
+      COUNTS[:new] += 1
+      puts "NEW: #{file_path}" if VERBOSE || TEST_ONLY
+      unless TEST_ONLY
+        @new_files[file_path] = digest
+        @insert.execute! file_path, mtime, digest
+      end
+    end
+  end
+
+  def process_missing_files
+    @missing_files.delete_if do |filename, digest|
+      if @new_files.value?(digest)
+        COUNTS[:renamed] += 1
+        unless TEST_ONLY
+          @delete_by_filename.execute filename
+        end
+        true
+      end
+    end
+
+    if (COUNTS[:missing] = @missing_files.length) > 0
+      puts "MISSING FILES:"
+      @missing_files.sort.to_h.each do |filename, digest|
+        puts filename
+      end
+      unless TEST_ONLY
+        puts "Remove missing files from the database (y/n)?"
+        if STDIN.gets.strip == "y"
+          @missing_files.each do |filename, digest|
+            @delete_by_filename.execute filename
+          end
+        end
+      end
+    end
+  end
+end
+
+class Checker
+  def initialize files_path, digest_database_path
+    @files_path = files_path
+    ensure_dir_exists @files_path
+
+    if digest_database_path
+      @digest_database_path = digest_database_path
+      ensure_dir_exists @digest_database_path.dirname
+    else
+      @digest_database_path = @files_path + 'file-digests.sqlite'
+      @skip_file_digests_sqlite = true
+    end
+
+    @digest_database = DigestDatabase.new @digest_database_path
+  end
+
+  def check
+    walk_files do |filename|
+      begin
+        process_file filename
+      rescue => exception
+        COUNTS[:exceptions] += 1
+        STDERR.puts "EXCEPTION: #{filename}: #{exception.message}"
+      end
+    end
+
+    @digest_database.process_missing_files
+  end
+
+  def walk_files
+    Dir.glob(@files_path + '**' + '*') do |filename|
+      next unless File.file? filename
+      next if @skip_file_digests_sqlite && filename == 'file-digests.sqlite'
+      yield filename
+    end
+  end
+
+  def process_file filename
+    @digest_database.insert_or_update(
+      filename.delete_prefix(@files_path.to_s + '/'),
+      File.mtime(filename).utc.strftime('%Y-%m-%d %H:%M:%S'),
+      get_file_digest(filename)
+      )
+  end
+
+  def get_file_digest filename
+    File.open(filename, 'rb') do |io|
+      digest = Digest::SHA512.new
+      buffer = ""
+      while io.read(40960, buffer)
+        digest.update(buffer)
+      end
+      return digest.hexdigest
+    end
+  end
+
+end
+
+VERBOSE = (ENV["VERBOSE"] == "true")
+TEST_ONLY = (ENV["TEST_ONLY"] == "true")
+
+COUNTS = {good: 0, updated: 0, new: 0, missing: 0, renamed: 0, likely_damaged: 0, exceptions: 0}
+
+begin
+  if ARGV[0]
+    files_path = Pathname.new patch_path_string(ARGV[0])
+  else
+    files_path = Pathname.new patch_path_string(".")
+  end
+
+  digest_database_path = Pathname.new patch_path_string(ARGV[1]) if ARGV[1]
+
+  measure_time do
+    checker = Checker.new files_path, digest_database_path
+    checker.check
+  end
+
+  if COUNTS[:likely_damaged] > 0 || COUNTS[:exceptions] > 0
+    puts "ERRORS WERE OCCURRED, PLEASE CHECK FOR THEM"
+  end
+
+  puts COUNTS.inspect
+
+rescue => exception
+  STDERR.puts "EXCEPTION: #{exception.message}"
+  raise exception
+end

metadata

@@ -0,0 +1,58 @@
+--- !ruby/object:Gem::Specification
+name: file-digests
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Stanislav Senotrusov
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2020-10-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.3.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.3.0
+description: Calculate file digests and check for the possible file corruption
+email: stan@senotrusov.com
+executables:
+- file-digests
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/file-digests
+homepage: https://github.com/senotrusov/file-digests
+licenses:
+- Apache-2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.2
+signing_key: 
+specification_version: 4
+summary: file-digests
+test_files: []