file-digests 0.0.35 → 0.0.40
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/file-digests.rb +103 -30
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: a9d040599aee9aeb62234557b2a92edec36265e4439965e4c9d00a2c9afa117a
|
4
|
+
data.tar.gz: 992fe8a843afe761537a3c48b1ecde27ccdf8101be58dd4b456572c59232141b
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 1e15aa584690f8062a51cbb4785c9716e610258fba77859cb2b50642154701ffa1651312be1f6455921ee435ed7df5ed00bcc36f12def4009ac7edcc4aebb93e
|
7
|
+
data.tar.gz: 5c94ae4165677af0be7b790a7a92ffc9af7751f0dd2661bf238d06b02569ef4ec746908721c7d16781d11a431048759e1490f9e180ea9c2e647f5e585a3ebbe0
|
data/lib/file-digests.rb
CHANGED
@@ -139,15 +139,29 @@ class FileDigests
|
|
139
139
|
end
|
140
140
|
|
141
141
|
def initialize_paths files_path, digest_database_path
|
142
|
+
@start_time_filename_string = Time.now.strftime("%Y-%m-%d %H-%M-%S")
|
142
143
|
@files_path = cleanup_path(files_path || ".")
|
144
|
+
raise "ERROR: Files path must be a readable directory" unless (File.directory?(@files_path) && File.readable?(@files_path))
|
145
|
+
@files_path = realpath_with_disk @files_path
|
143
146
|
|
144
|
-
|
147
|
+
@error_log_path = @files_path + "file-digests errors #{@start_time_filename_string}.txt"
|
148
|
+
@missing_files_path = @files_path + "file-digests missing files #{@start_time_filename_string}.txt"
|
145
149
|
|
146
150
|
@digest_database_path = digest_database_path ? cleanup_path(digest_database_path) : @files_path
|
147
151
|
@digest_database_path += ".file-digests.sqlite" if File.directory?(@digest_database_path)
|
148
152
|
ensure_dir_exist @digest_database_path.dirname
|
149
|
-
|
150
|
-
|
153
|
+
@digest_database_path = realdirpath_with_disk @digest_database_path
|
154
|
+
|
155
|
+
@digest_database_files = [
|
156
|
+
"#{@digest_database_path}",
|
157
|
+
"#{@digest_database_path}-wal",
|
158
|
+
"#{@digest_database_path}-shm"
|
159
|
+
]
|
160
|
+
|
161
|
+
@skip_files = @digest_database_files + [
|
162
|
+
@error_log_path.to_s,
|
163
|
+
@missing_files_path.to_s
|
164
|
+
]
|
151
165
|
|
152
166
|
if @options[:verbose]
|
153
167
|
puts "Target directory: #{@files_path}"
|
@@ -273,7 +287,7 @@ class FileDigests
|
|
273
287
|
perhaps_transaction(@new_digest_algorithm, :exclusive) do
|
274
288
|
@counters = {good: 0, updated: 0, renamed: 0, likely_damaged: 0, exceptions: 0}
|
275
289
|
|
276
|
-
walk_files do |filename|
|
290
|
+
walk_files(@files_path.to_s) do |filename|
|
277
291
|
process_file filename
|
278
292
|
end
|
279
293
|
|
@@ -286,7 +300,7 @@ class FileDigests
|
|
286
300
|
if any_exceptions?
|
287
301
|
STDERR.puts "Due to previously occurred errors, missing files will not removed from the database."
|
288
302
|
else
|
289
|
-
|
303
|
+
report_missing_files
|
290
304
|
if !@options[:test_only] && (@options[:auto] || confirm("Remove missing files from the database"))
|
291
305
|
nested_transaction do
|
292
306
|
puts "Removing missing files..." if @options[:verbose]
|
@@ -340,9 +354,11 @@ class FileDigests
|
|
340
354
|
private
|
341
355
|
|
342
356
|
def process_file filename
|
343
|
-
|
357
|
+
perhaps_nt_filename = perhaps_nt_path filename
|
358
|
+
|
359
|
+
return if File.symlink? perhaps_nt_filename
|
344
360
|
|
345
|
-
stat = File.stat
|
361
|
+
stat = File.stat perhaps_nt_filename
|
346
362
|
|
347
363
|
return if stat.blockdev?
|
348
364
|
return if stat.chardev?
|
@@ -352,23 +368,19 @@ class FileDigests
|
|
352
368
|
|
353
369
|
raise "File is not readable" unless stat.readable?
|
354
370
|
|
355
|
-
if @
|
356
|
-
puts "SKIPPING
|
371
|
+
if @skip_files.include?(filename)
|
372
|
+
puts "SKIPPING FILE: #{filename}" if @options[:verbose]
|
357
373
|
return
|
358
374
|
end
|
359
375
|
|
360
376
|
normalized_filename = filename.delete_prefix("#{@files_path.to_s}/").encode("utf-8", universal_newline: true).unicode_normalize(:nfkc)
|
361
377
|
mtime_string = time_to_database stat.mtime
|
362
|
-
digest, new_digest = get_file_digest(
|
378
|
+
digest, new_digest = get_file_digest(perhaps_nt_filename)
|
363
379
|
|
364
380
|
nested_transaction do
|
365
381
|
new_digests_insert(normalized_filename, new_digest) if new_digest
|
366
382
|
process_file_indeed normalized_filename, mtime_string, digest
|
367
383
|
end
|
368
|
-
|
369
|
-
rescue => exception
|
370
|
-
@counters[:exceptions] += 1
|
371
|
-
print_file_exception exception, filename
|
372
384
|
end
|
373
385
|
|
374
386
|
def process_file_indeed filename, mtime, digest
|
@@ -394,7 +406,7 @@ class FileDigests
|
|
394
406
|
else
|
395
407
|
if found["mtime"] == mtime && !@options[:accept_fate] # Digest is different and mtime is the same
|
396
408
|
@counters[:likely_damaged] += 1
|
397
|
-
|
409
|
+
error_text "LIKELY DAMAGED: #{filename}"
|
398
410
|
else
|
399
411
|
@counters[:updated] += 1
|
400
412
|
puts "UPDATED#{" (FATE ACCEPTED)" if found["mtime"] == mtime && @options[:accept_fate]}: #{filename}" unless @options[:quiet]
|
@@ -424,10 +436,19 @@ class FileDigests
|
|
424
436
|
@counters[:renamed] = @db.changes
|
425
437
|
end
|
426
438
|
|
427
|
-
def
|
439
|
+
def report_missing_files
|
428
440
|
puts "\nMISSING FILES:"
|
441
|
+
write_missing_files STDOUT
|
442
|
+
if missing_files_count > 256
|
443
|
+
File.open(@missing_files_path, "a") do |f|
|
444
|
+
write_missing_files f
|
445
|
+
end
|
446
|
+
end
|
447
|
+
end
|
448
|
+
|
449
|
+
def write_missing_files dest
|
429
450
|
missing_files_select_all_filenames.each do |record|
|
430
|
-
puts record["filename"]
|
451
|
+
dest.puts record["filename"]
|
431
452
|
end
|
432
453
|
end
|
433
454
|
|
@@ -530,7 +551,7 @@ class FileDigests
|
|
530
551
|
def check_if_database_is_at_certain_version target_version
|
531
552
|
current_version = get_metadata("database_version")
|
532
553
|
if current_version != target_version
|
533
|
-
STDERR.puts "This version of file-digests (#{FileDigests::VERSION || "unknown"}) is only compartible with the database version #{target_version}. Current database version is #{current_version}. To use this database, please install appropriate version if file-digest."
|
554
|
+
STDERR.puts "ERROR: This version of file-digests (#{FileDigests::VERSION || "unknown"}) is only compartible with the database version #{target_version}. Current database version is #{current_version}. To use this database, please install appropriate version if file-digest."
|
534
555
|
raise "Incompatible database version"
|
535
556
|
end
|
536
557
|
end
|
@@ -542,6 +563,22 @@ class FileDigests
|
|
542
563
|
|
543
564
|
# Filesystem-related helpers
|
544
565
|
|
566
|
+
def realpath_with_disk path
|
567
|
+
path = path.realpath
|
568
|
+
if Gem.win_platform? && path.to_s[0] == "/"
|
569
|
+
return Pathname(Dir.pwd[0, 2] + path.to_s)
|
570
|
+
end
|
571
|
+
path
|
572
|
+
end
|
573
|
+
|
574
|
+
def realdirpath_with_disk path
|
575
|
+
path = path.realdirpath
|
576
|
+
if Gem.win_platform? && path.to_s[0] == "/"
|
577
|
+
return Pathname(Dir.pwd[0, 2] + path.to_s)
|
578
|
+
end
|
579
|
+
path
|
580
|
+
end
|
581
|
+
|
545
582
|
def patch_path_string path
|
546
583
|
Gem.win_platform? ? path.gsub(/\\/, "/") : path
|
547
584
|
end
|
@@ -560,10 +597,32 @@ class FileDigests
|
|
560
597
|
end
|
561
598
|
end
|
562
599
|
|
563
|
-
def walk_files
|
564
|
-
|
565
|
-
|
566
|
-
|
600
|
+
def walk_files(path, &block)
|
601
|
+
Dir.each_child(path, encoding: "UTF-8") do |item|
|
602
|
+
item = "#{path}#{File::SEPARATOR}#{item}"
|
603
|
+
begin
|
604
|
+
item_perhaps_nt_path = perhaps_nt_path item
|
605
|
+
|
606
|
+
unless File.symlink? item_perhaps_nt_path
|
607
|
+
if File.directory?(item_perhaps_nt_path)
|
608
|
+
raise "Directory is not readable" unless File.readable?(item_perhaps_nt_path)
|
609
|
+
walk_files(item, &block)
|
610
|
+
else
|
611
|
+
yield item
|
612
|
+
end
|
613
|
+
end
|
614
|
+
rescue => exception
|
615
|
+
@counters[:exceptions] += 1
|
616
|
+
report_file_exception exception, item
|
617
|
+
end
|
618
|
+
end
|
619
|
+
end
|
620
|
+
|
621
|
+
def perhaps_nt_path path
|
622
|
+
if Gem.win_platform?
|
623
|
+
"\\??\\#{path.gsub(/\//,"\\")}"
|
624
|
+
else
|
625
|
+
path
|
567
626
|
end
|
568
627
|
end
|
569
628
|
|
@@ -611,17 +670,31 @@ class FileDigests
|
|
611
670
|
puts "Elapsed time: #{elapsed.to_i / 3600}h #{(elapsed.to_i % 3600) / 60}m #{"%.3f" % (elapsed % 60)}s" unless @options[:quiet]
|
612
671
|
end
|
613
672
|
|
614
|
-
def
|
615
|
-
STDERR
|
673
|
+
def report_file_exception exception, filename
|
674
|
+
write_file_exception STDERR, exception, filename
|
675
|
+
File.open(@error_log_path, "a") do |f|
|
676
|
+
write_file_exception f, exception, filename
|
677
|
+
end
|
678
|
+
end
|
679
|
+
|
680
|
+
def write_file_exception dest, exception, filename
|
681
|
+
dest.print "ERROR: #{exception.message}, processing file: "
|
616
682
|
begin
|
617
|
-
|
683
|
+
dest.print filename.encode("utf-8", universal_newline: true)
|
618
684
|
rescue
|
619
|
-
|
620
|
-
|
685
|
+
dest.print "(Unable to encode file name to utf-8) "
|
686
|
+
dest.print filename
|
687
|
+
end
|
688
|
+
dest.print "\n"
|
689
|
+
dest.flush
|
690
|
+
exception.backtrace.each { |line| dest.puts " " + line }
|
691
|
+
end
|
692
|
+
|
693
|
+
def error_text text
|
694
|
+
STDERR.puts text
|
695
|
+
File.open(@error_log_path, "a") do |f|
|
696
|
+
f.puts text
|
621
697
|
end
|
622
|
-
STDERR.print "\n"
|
623
|
-
STDERR.flush
|
624
|
-
exception.backtrace.each { |line| STDERR.puts " " + line }
|
625
698
|
end
|
626
699
|
|
627
700
|
def print_counters
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: file-digests
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.40
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Stanislav Senotrusov
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-10-
|
11
|
+
date: 2020-10-20 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: openssl
|