file-digests 0.0.21 → 0.0.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/bin/file-digests +1 -1
- data/lib/file-digests.rb +393 -166
- metadata +19 -11
- data/bin/file-digests-auto +0 -7
- data/bin/file-digests-show-duplicates +0 -5
- data/bin/file-digests-test +0 -7
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f08714e86e275eb74108da01c667d403134366e6d7e9c5e8b08278d8c8842ba8
|
4
|
+
data.tar.gz: 57802538edae099807ede5460a07d6fd7ea6a5d0810f40d306db8f330435631d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4c642c5aaf06d903114aabc65ec13ae3c5eb04c807b167c7eb1f3c63e8f56144ff3d93964fdc55cbcb33fad743cad635cfa61284171c94ed3b7a9a42c3efbca6
|
7
|
+
data.tar.gz: eebb6b444c6d0921f638200a27f97af5e9026ab0cf21b7b43b3e58f6f13ccc6323f482f93594ae993637a18b7611514469848307defe3796f412dd453c08b932
|
data/bin/file-digests
CHANGED
data/lib/file-digests.rb
CHANGED
@@ -1,120 +1,246 @@
|
|
1
1
|
require 'date'
|
2
|
-
require 'set'
|
3
2
|
require 'digest'
|
4
3
|
require 'fileutils'
|
4
|
+
require 'openssl'
|
5
|
+
require 'optparse'
|
5
6
|
require 'pathname'
|
7
|
+
require 'set'
|
6
8
|
require 'sqlite3'
|
7
9
|
|
8
10
|
class FileDigests
|
11
|
+
DIGEST_ALGORITHMS=["BLAKE2b512", "SHA3-256", "SHA512-256"]
|
9
12
|
|
10
|
-
def self.
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
}
|
16
|
-
file_digests = self.new ARGV[0], ARGV[1], options
|
17
|
-
file_digests.perform_check
|
13
|
+
def self.canonical_digest_algorithm_name(string)
|
14
|
+
if string
|
15
|
+
index = DIGEST_ALGORITHMS.map(&:downcase).index(string.downcase)
|
16
|
+
index && DIGEST_ALGORITHMS[index]
|
17
|
+
end
|
18
18
|
end
|
19
19
|
|
20
|
-
def
|
21
|
-
|
22
|
-
|
20
|
+
def canonical_digest_algorithm_name string
|
21
|
+
self.class.canonical_digest_algorithm_name string
|
22
|
+
end
|
23
|
+
|
24
|
+
def self.digest_algorithms_list_text
|
25
|
+
"Digest algorithm should be one of the following: #{DIGEST_ALGORITHMS.join ", "}"
|
26
|
+
end
|
27
|
+
|
28
|
+
def self.parse_cli_options
|
29
|
+
options = {}
|
30
|
+
|
31
|
+
OptionParser.new do |opts|
|
32
|
+
opts.banner = [
|
33
|
+
"Usage: file-digests [options] [path/to/directory] [path/to/database_file]",
|
34
|
+
" By default the current directory will be operated upon, and the database file will be placed to the current directory as well.",
|
35
|
+
" Should you wish to check current directory but place the database elsewhere, you could provide \".\" as a first argument, and the path to a database_file as a second."
|
36
|
+
].join "\n"
|
37
|
+
|
38
|
+
opts.on("-a", "--auto", "Do not ask for any confirmation") do
|
39
|
+
options[:auto] = true
|
40
|
+
end
|
41
|
+
|
42
|
+
opts.on(
|
43
|
+
'--digest=DIGEST',
|
44
|
+
'Select a digest algorithm to use. Default is "BLAKE2b512".',
|
45
|
+
'You might also consider to use slower "SHA512-256" or even more slower "SHA3-256".',
|
46
|
+
"#{digest_algorithms_list_text}.",
|
47
|
+
'You only need to specify an algorithm on the first run, your choice will be saved to a database.',
|
48
|
+
'Any time later you could specify a new algorithm to change the current one.',
|
49
|
+
'Transition to a new algorithm will only occur if all files pass the check by digests which were stored using the old one.'
|
50
|
+
) do |value|
|
51
|
+
digest_algorithm = canonical_digest_algorithm_name(value)
|
52
|
+
unless digest_algorithm
|
53
|
+
STDERR.puts "ERROR: #{digest_algorithms_list_text}"
|
54
|
+
exit 1
|
55
|
+
end
|
56
|
+
options[:digest_algorithm] = digest_algorithm
|
57
|
+
end
|
58
|
+
|
59
|
+
opts.on("-d", "--duplicates", "Show the list of duplicate files, based on the information out of the database") do
|
60
|
+
options[:action] = :show_duplicates
|
61
|
+
end
|
62
|
+
|
63
|
+
opts.on("-t", "--test", "Perform only the test, do not modify the digest database") do
|
64
|
+
options[:test_only] = true
|
65
|
+
end
|
66
|
+
|
67
|
+
opts.on("-q", "--quiet", "Less verbose output, stil report any found issues") do
|
68
|
+
options[:quiet] = true
|
69
|
+
end
|
70
|
+
|
71
|
+
opts.on("-v", "--verbose", "More verbose output") do
|
72
|
+
options[:verbose] = true
|
73
|
+
end
|
74
|
+
|
75
|
+
opts.on("-h", "--help", "Prints this help") do
|
76
|
+
puts opts
|
77
|
+
exit
|
78
|
+
end
|
79
|
+
end.parse!
|
80
|
+
options
|
81
|
+
end
|
82
|
+
|
83
|
+
def self.run_cli_utility
|
84
|
+
options = parse_cli_options
|
85
|
+
|
86
|
+
file_digests = self.new ARGV[0], ARGV[1], options
|
87
|
+
file_digests.send(options[:action] || :perform_check)
|
23
88
|
end
|
24
89
|
|
25
90
|
def initialize files_path, digest_database_path, options = {}
|
26
91
|
@options = options
|
27
92
|
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
raise "Files path must be a readable directory" unless (File.directory?(@files_path) && File.readable?(@files_path))
|
93
|
+
initialize_paths files_path, digest_database_path
|
94
|
+
initialize_database
|
32
95
|
|
33
|
-
@
|
34
|
-
|
96
|
+
if @digest_algorithm = canonical_digest_algorithm_name(get_metadata("digest_algorithm"))
|
97
|
+
if @options[:digest_algorithm] && @options[:digest_algorithm] != @digest_algorithm
|
98
|
+
@new_digest_algorithm = @options[:digest_algorithm]
|
99
|
+
end
|
35
100
|
else
|
36
|
-
@
|
101
|
+
@digest_algorithm = (@options[:digest_algorithm] || "BLAKE2b512")
|
102
|
+
set_metadata "digest_algorithm", @digest_algorithm
|
37
103
|
end
|
38
104
|
|
39
|
-
if
|
40
|
-
|
41
|
-
end
|
105
|
+
puts "Using #{@digest_algorithm} digest algorithm" if @options[:verbose]
|
106
|
+
end
|
42
107
|
|
43
|
-
|
44
|
-
|
45
|
-
|
108
|
+
def initialize_paths files_path, digest_database_path
|
109
|
+
@files_path = cleanup_path(files_path || ".")
|
110
|
+
|
111
|
+
raise "Files path must be a readable directory" unless (File.directory?(@files_path) && File.readable?(@files_path))
|
46
112
|
|
113
|
+
@digest_database_path = digest_database_path ? cleanup_path(digest_database_path) : @files_path
|
114
|
+
@digest_database_path += '.file-digests.sqlite' if File.directory?(@digest_database_path)
|
47
115
|
ensure_dir_exists @digest_database_path.dirname
|
48
116
|
|
49
|
-
|
50
|
-
|
51
|
-
|
117
|
+
if @options[:verbose]
|
118
|
+
puts "Target directory: #{@files_path}"
|
119
|
+
puts "Database location: #{@digest_database_path}"
|
52
120
|
end
|
53
|
-
|
54
|
-
initialize_database @digest_database_path
|
55
121
|
end
|
56
122
|
|
57
|
-
def initialize_database
|
58
|
-
@db = SQLite3::Database.new
|
123
|
+
def initialize_database
|
124
|
+
@db = SQLite3::Database.new @digest_database_path.to_s
|
59
125
|
@db.results_as_hash = true
|
60
126
|
|
127
|
+
file_digests_gem_version = Gem.loaded_specs["file-digests"]&.version&.to_s
|
128
|
+
|
129
|
+
execute 'PRAGMA encoding = "UTF-8"'
|
61
130
|
execute 'PRAGMA journal_mode = "WAL"'
|
62
131
|
execute 'PRAGMA synchronous = "NORMAL"'
|
63
132
|
execute 'PRAGMA locking_mode = "EXCLUSIVE"'
|
64
133
|
execute 'PRAGMA cache_size = "5000"'
|
65
134
|
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
135
|
+
@db.transaction(:exclusive) do
|
136
|
+
unless table_exist?("metadata")
|
137
|
+
execute "CREATE TABLE metadata (
|
138
|
+
key TEXT NOT NULL PRIMARY KEY,
|
139
|
+
value TEXT)"
|
140
|
+
execute "CREATE UNIQUE INDEX metadata_key ON metadata(key)"
|
141
|
+
set_metadata("metadata_table_created_by_gem_version", file_digests_gem_version) if file_digests_gem_version
|
142
|
+
end
|
143
|
+
|
144
|
+
prepare_method :set_metadata_query, "INSERT INTO metadata (key, value) VALUES (?, ?) ON CONFLICT (key) DO UPDATE SET value=excluded.value"
|
145
|
+
prepare_method :get_metadata_query, "SELECT value FROM metadata WHERE key = ?"
|
146
|
+
|
147
|
+
# Heuristic to detect database version 1 (metadata was not stored back then)
|
148
|
+
unless get_metadata("database_version")
|
149
|
+
if table_exist?("digests")
|
150
|
+
set_metadata "database_version", "1"
|
151
|
+
end
|
152
|
+
end
|
153
|
+
|
154
|
+
unless table_exist?("digests")
|
155
|
+
execute "CREATE TABLE digests (
|
156
|
+
id INTEGER NOT NULL PRIMARY KEY,
|
157
|
+
filename TEXT NOT NULL,
|
158
|
+
mtime TEXT,
|
159
|
+
digest TEXT NOT NULL,
|
160
|
+
digest_check_time TEXT NOT NULL)"
|
161
|
+
execute "CREATE UNIQUE INDEX digests_filename ON digests(filename)"
|
162
|
+
set_metadata("digests_table_created_by_gem_version", file_digests_gem_version) if file_digests_gem_version
|
163
|
+
end
|
164
|
+
|
165
|
+
prepare_method :insert, "INSERT INTO digests (filename, mtime, digest, digest_check_time) VALUES (?, ?, ?, datetime('now'))"
|
166
|
+
prepare_method :find_by_filename_query, "SELECT id, mtime, digest FROM digests WHERE filename = ?"
|
167
|
+
prepare_method :touch_digest_check_time, "UPDATE digests SET digest_check_time = datetime('now') WHERE id = ?"
|
168
|
+
prepare_method :update_mtime_and_digest, "UPDATE digests SET mtime = ?, digest = ?, digest_check_time = datetime('now') WHERE id = ?"
|
169
|
+
prepare_method :update_mtime, "UPDATE digests SET mtime = ?, digest_check_time = datetime('now') WHERE id = ?"
|
170
|
+
prepare_method :delete_by_filename, "DELETE FROM digests WHERE filename = ?"
|
171
|
+
prepare_method :query_duplicates, "SELECT digest, filename FROM digests WHERE digest IN (SELECT digest FROM digests GROUP BY digest HAVING count(*) > 1) ORDER BY digest, filename;"
|
172
|
+
prepare_method :update_digest_to_new_digest, "UPDATE digests SET digest = ? WHERE digest = ?"
|
173
|
+
|
174
|
+
unless get_metadata("database_version")
|
175
|
+
set_metadata "database_version", "2"
|
176
|
+
end
|
177
|
+
|
178
|
+
# Convert database from 1st to 2nd version
|
179
|
+
unless get_metadata("digest_algorithm")
|
180
|
+
if get_metadata("database_version") == "1"
|
181
|
+
if File.exist?(@digest_database_path.dirname + '.file-digests.sha512')
|
182
|
+
set_metadata("digest_algorithm", "SHA512")
|
183
|
+
else
|
184
|
+
set_metadata("digest_algorithm", "SHA256")
|
185
|
+
end
|
186
|
+
set_metadata "database_version", "2"
|
187
|
+
end
|
188
|
+
end
|
76
189
|
|
77
|
-
|
78
|
-
prepare_method :find_by_filename, "SELECT id, mtime, digest FROM digests WHERE filename = ?"
|
79
|
-
prepare_method :touch_digest_check_time, "UPDATE digests SET digest_check_time = datetime('now') WHERE id = ?"
|
80
|
-
prepare_method :update_mtime_and_digest, "UPDATE digests SET mtime = ?, digest = ?, digest_check_time = datetime('now') WHERE id = ?"
|
81
|
-
prepare_method :update_mtime, "UPDATE digests SET mtime = ?, digest_check_time = datetime('now') WHERE id = ?"
|
82
|
-
prepare_method :delete_by_filename, "DELETE FROM digests WHERE filename = ?"
|
83
|
-
prepare_method :query_duplicates, "SELECT digest, filename FROM digests WHERE digest IN (SELECT digest FROM digests GROUP BY digest HAVING count(*) > 1) ORDER BY digest, filename;"
|
190
|
+
end
|
84
191
|
end
|
85
192
|
|
86
193
|
def perform_check
|
87
|
-
@
|
88
|
-
|
89
|
-
|
194
|
+
perhaps_transaction(@new_digest_algorithm, :exclusive) do
|
195
|
+
@counters = {good: 0, updated: 0, new: 0, renamed: 0, likely_damaged: 0, exceptions: 0}
|
196
|
+
@new_files = {}
|
197
|
+
@new_digests = {}
|
90
198
|
|
91
|
-
|
92
|
-
|
93
|
-
|
199
|
+
@missing_files = Hash[@db.prepare("SELECT filename, digest FROM digests").execute!]
|
200
|
+
|
201
|
+
measure_time do
|
202
|
+
walk_files do |filename|
|
203
|
+
process_file filename
|
204
|
+
end
|
94
205
|
end
|
95
|
-
end
|
96
206
|
|
97
|
-
|
207
|
+
track_renames
|
208
|
+
|
209
|
+
if any_missing_files?
|
210
|
+
if any_exceptions?
|
211
|
+
STDERR.puts "Due to previously occurred errors, database cleanup from missing files will be skipped this time."
|
212
|
+
else
|
213
|
+
print_missing_files
|
214
|
+
if !@options[:test_only] && (@options[:auto] || confirm("Remove missing files from the database"))
|
215
|
+
remove_missing_files
|
216
|
+
end
|
217
|
+
end
|
218
|
+
end
|
219
|
+
|
220
|
+
if @new_digest_algorithm && !@options[:test_only]
|
221
|
+
if any_missing_files? || any_likely_damaged? || any_exceptions?
|
222
|
+
STDERR.puts "ERROR: New digest algorithm will not be in effect until there are files that are missing, likely damaged, or processed with an exception."
|
223
|
+
else
|
224
|
+
@new_digests.each do |old_digest, new_digest|
|
225
|
+
update_digest_to_new_digest new_digest, old_digest
|
226
|
+
end
|
227
|
+
set_metadata "digest_algorithm", @new_digest_algorithm
|
228
|
+
end
|
229
|
+
end
|
98
230
|
|
99
|
-
|
100
|
-
|
101
|
-
if !@options[:test_only] && (@options[:auto] || confirm("Remove missing files from the database"))
|
102
|
-
remove_missing_files
|
231
|
+
if any_likely_damaged? || any_exceptions?
|
232
|
+
STDERR.puts "PLEASE REVIEW ERRORS THAT WERE OCCURRED!"
|
103
233
|
end
|
104
|
-
end
|
105
234
|
|
106
|
-
|
107
|
-
STDERR.puts "ERRORS WERE OCCURRED"
|
108
|
-
end
|
235
|
+
set_metadata(@options[:test_only] ? "latest_test_only_check_time" : "latest_complete_check_time", time_to_database(Time.now))
|
109
236
|
|
110
|
-
|
237
|
+
print_counters
|
238
|
+
end
|
111
239
|
end
|
112
240
|
|
113
241
|
def show_duplicates
|
114
242
|
current_digest = nil
|
115
|
-
|
116
|
-
|
117
|
-
while found = result.next_hash do
|
243
|
+
query_duplicates.each do |found|
|
118
244
|
if current_digest != found['digest']
|
119
245
|
puts "" if current_digest
|
120
246
|
current_digest = found['digest']
|
@@ -135,117 +261,72 @@ class FileDigests
|
|
135
261
|
return if stat.chardev?
|
136
262
|
return if stat.directory?
|
137
263
|
return if stat.pipe?
|
138
|
-
unless stat.readable?
|
139
|
-
raise "File is not readable"
|
140
|
-
end
|
141
264
|
return if stat.socket?
|
142
265
|
|
143
|
-
|
144
|
-
|
145
|
-
|
146
|
-
|
147
|
-
|
148
|
-
|
266
|
+
raise "File is not readable" unless stat.readable?
|
267
|
+
|
268
|
+
if filename == "#{@digest_database_path}" ||
|
269
|
+
filename == "#{@digest_database_path}-wal" ||
|
270
|
+
filename == "#{@digest_database_path}-shm"
|
271
|
+
puts "SKIPPING DATABASE FILE: #{filename}" if @options[:verbose]
|
272
|
+
return
|
149
273
|
end
|
150
274
|
|
151
|
-
|
152
|
-
|
153
|
-
stat.mtime.utc.strftime('%Y-%m-%d %H:%M:%S'),
|
154
|
-
get_file_digest(filename)
|
155
|
-
)
|
156
|
-
rescue => exception
|
157
|
-
@counters[:exceptions] += 1
|
158
|
-
STDERR.puts "EXCEPTION: #{filename.encode('utf-8', universal_newline: true)}: #{exception.message}"
|
159
|
-
end
|
275
|
+
normalized_filename = filename.delete_prefix("#{@files_path.to_s}/").encode('utf-8', universal_newline: true).unicode_normalize(:nfkc)
|
276
|
+
mtime_string = time_to_database stat.mtime
|
160
277
|
|
161
|
-
|
162
|
-
Gem.win_platform? ? path.gsub(/\\/, '/') : path
|
163
|
-
end
|
278
|
+
process_file_indeed normalized_filename, mtime_string, get_file_digest(filename)
|
164
279
|
|
165
|
-
|
166
|
-
|
280
|
+
rescue => exception
|
281
|
+
@counters[:exceptions] += 1
|
282
|
+
print_file_exception exception, filename
|
167
283
|
end
|
168
284
|
|
169
|
-
def
|
170
|
-
if
|
171
|
-
|
172
|
-
raise "#{path} is not a directory"
|
173
|
-
end
|
285
|
+
def process_file_indeed filename, mtime, digest
|
286
|
+
if found = find_by_filename(filename)
|
287
|
+
process_previously_seen_file found, filename, mtime, digest
|
174
288
|
else
|
175
|
-
|
176
|
-
end
|
177
|
-
end
|
178
|
-
|
179
|
-
def walk_files
|
180
|
-
Dir.glob(@files_path + '**' + '*', File::FNM_DOTMATCH) do |filename|
|
181
|
-
yield filename
|
289
|
+
process_new_file filename, mtime, digest
|
182
290
|
end
|
183
291
|
end
|
184
292
|
|
185
|
-
def
|
186
|
-
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
|
293
|
+
def process_previously_seen_file found, filename, mtime, digest
|
294
|
+
@missing_files.delete(filename)
|
295
|
+
if found['digest'] == digest
|
296
|
+
@counters[:good] += 1
|
297
|
+
puts "GOOD: #{filename}" if @options[:verbose]
|
298
|
+
unless @options[:test_only]
|
299
|
+
if found['mtime'] == mtime
|
300
|
+
touch_digest_check_time found['id']
|
301
|
+
else
|
302
|
+
update_mtime mtime, found['id']
|
303
|
+
end
|
304
|
+
end
|
305
|
+
else
|
306
|
+
if found['mtime'] == mtime # Digest is different and mtime is the same
|
307
|
+
@counters[:likely_damaged] += 1
|
308
|
+
STDERR.puts "LIKELY DAMAGED: #{filename}"
|
309
|
+
else
|
310
|
+
@counters[:updated] += 1
|
311
|
+
puts "UPDATED: #{filename}" unless @options[:quiet]
|
312
|
+
unless @options[:test_only]
|
313
|
+
update_mtime_and_digest mtime, digest, found['id']
|
314
|
+
end
|
191
315
|
end
|
192
|
-
return digest.hexdigest
|
193
316
|
end
|
194
317
|
end
|
195
318
|
|
196
|
-
def
|
197
|
-
|
198
|
-
|
199
|
-
|
319
|
+
def process_new_file filename, mtime, digest
|
320
|
+
@counters[:new] += 1
|
321
|
+
puts "NEW: #{filename}" unless @options[:quiet]
|
322
|
+
unless @options[:test_only]
|
323
|
+
@new_files[filename] = digest
|
324
|
+
insert filename, mtime, digest
|
200
325
|
end
|
201
326
|
end
|
202
327
|
|
203
|
-
def measure_time
|
204
|
-
start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
205
|
-
yield
|
206
|
-
elapsed = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start).to_i
|
207
|
-
puts "Elapsed time: #{elapsed / 3600}h #{(elapsed % 3600) / 60}m #{elapsed % 60}s" unless @options[:quiet]
|
208
|
-
end
|
209
|
-
|
210
|
-
def insert_or_update file_path, mtime, digest
|
211
|
-
result = find_by_filename file_path
|
212
|
-
|
213
|
-
if found = result.next_hash
|
214
|
-
raise "Multiple records found" if result.next
|
215
328
|
|
216
|
-
|
217
|
-
|
218
|
-
if found['digest'] == digest
|
219
|
-
@counters[:good] += 1
|
220
|
-
# puts "GOOD: #{file_path}" unless @options[:quiet]
|
221
|
-
unless @options[:test_only]
|
222
|
-
if found['mtime'] == mtime
|
223
|
-
touch_digest_check_time found['id']
|
224
|
-
else
|
225
|
-
update_mtime mtime, found['id']
|
226
|
-
end
|
227
|
-
end
|
228
|
-
else
|
229
|
-
if found['mtime'] == mtime # Digest is different and mtime is the same
|
230
|
-
@counters[:likely_damaged] += 1
|
231
|
-
STDERR.puts "LIKELY DAMAGED: #{file_path}"
|
232
|
-
else
|
233
|
-
@counters[:updated] += 1
|
234
|
-
puts "UPDATED: #{file_path}" unless @options[:quiet]
|
235
|
-
unless @options[:test_only]
|
236
|
-
update_mtime_and_digest mtime, digest, found['id']
|
237
|
-
end
|
238
|
-
end
|
239
|
-
end
|
240
|
-
else
|
241
|
-
@counters[:new] += 1
|
242
|
-
puts "NEW: #{file_path}" unless @options[:quiet]
|
243
|
-
unless @options[:test_only]
|
244
|
-
@new_files[file_path] = digest
|
245
|
-
insert file_path, mtime, digest
|
246
|
-
end
|
247
|
-
end
|
248
|
-
end
|
329
|
+
# Renames and missing files
|
249
330
|
|
250
331
|
def track_renames
|
251
332
|
@missing_files.delete_if do |filename, digest|
|
@@ -257,11 +338,6 @@ class FileDigests
|
|
257
338
|
true
|
258
339
|
end
|
259
340
|
end
|
260
|
-
@counters[:missing] = @missing_files.length
|
261
|
-
end
|
262
|
-
|
263
|
-
def any_missing_files?
|
264
|
-
@missing_files.length > 0
|
265
341
|
end
|
266
342
|
|
267
343
|
def print_missing_files
|
@@ -272,22 +348,173 @@ class FileDigests
|
|
272
348
|
end
|
273
349
|
|
274
350
|
def remove_missing_files
|
275
|
-
|
351
|
+
nested_transaction do
|
276
352
|
@missing_files.each do |filename, digest|
|
277
353
|
delete_by_filename filename
|
278
354
|
end
|
355
|
+
@missing_files = {}
|
279
356
|
end
|
280
357
|
end
|
281
358
|
|
359
|
+
|
360
|
+
# Database helpers
|
361
|
+
|
282
362
|
def execute *args, &block
|
283
363
|
@db.execute *args, &block
|
284
364
|
end
|
285
365
|
|
366
|
+
def nested_transaction(mode)
|
367
|
+
if @db.transaction_active?
|
368
|
+
yield
|
369
|
+
else
|
370
|
+
@db.transaction(mode) do
|
371
|
+
yield
|
372
|
+
end
|
373
|
+
end
|
374
|
+
end
|
375
|
+
|
376
|
+
def perhaps_transaction(condition, mode)
|
377
|
+
if condition
|
378
|
+
@db.transaction(mode) do
|
379
|
+
yield
|
380
|
+
end
|
381
|
+
else
|
382
|
+
yield
|
383
|
+
end
|
384
|
+
end
|
385
|
+
|
386
|
+
def table_exist? table_name
|
387
|
+
execute("SELECT name FROM sqlite_master WHERE type='table' AND name = '#{table_name}'").length == 1
|
388
|
+
end
|
389
|
+
|
286
390
|
def prepare_method name, query
|
287
391
|
variable = "@#{name}"
|
392
|
+
|
288
393
|
instance_variable_set(variable, @db.prepare(query))
|
394
|
+
|
289
395
|
define_singleton_method name do |*args, &block|
|
290
396
|
instance_variable_get(variable).execute(*args, &block)
|
291
397
|
end
|
398
|
+
|
399
|
+
define_singleton_method "#{name}!" do |*args, &block|
|
400
|
+
instance_variable_get(variable).execute!(*args, &block)
|
401
|
+
end
|
402
|
+
end
|
403
|
+
|
404
|
+
def set_metadata key, value
|
405
|
+
set_metadata_query key, value
|
406
|
+
puts "#{key} set to: #{value}" if @options[:verbose]
|
407
|
+
value
|
408
|
+
end
|
409
|
+
|
410
|
+
def get_metadata key
|
411
|
+
get_metadata_query!(key)&.first&.first
|
412
|
+
end
|
413
|
+
|
414
|
+
def find_by_filename filename
|
415
|
+
result = find_by_filename_query filename
|
416
|
+
found = result.next
|
417
|
+
raise "Multiple records found" if result.next
|
418
|
+
found
|
419
|
+
end
|
420
|
+
|
421
|
+
def time_to_database time
|
422
|
+
time.utc.strftime('%Y-%m-%d %H:%M:%S')
|
423
|
+
end
|
424
|
+
|
425
|
+
|
426
|
+
# Filesystem-related helpers
|
427
|
+
|
428
|
+
def patch_path_string path
|
429
|
+
Gem.win_platform? ? path.gsub(/\\/, '/') : path
|
430
|
+
end
|
431
|
+
|
432
|
+
def cleanup_path path
|
433
|
+
Pathname.new(patch_path_string(path)).cleanpath
|
434
|
+
end
|
435
|
+
|
436
|
+
def ensure_dir_exists path
|
437
|
+
if File.exist?(path)
|
438
|
+
unless File.directory?(path)
|
439
|
+
raise "#{path} is not a directory"
|
440
|
+
end
|
441
|
+
else
|
442
|
+
FileUtils.mkdir_p path
|
443
|
+
end
|
444
|
+
end
|
445
|
+
|
446
|
+
def walk_files
|
447
|
+
Dir.glob(@files_path + '**' + '*', File::FNM_DOTMATCH) do |filename|
|
448
|
+
yield filename
|
449
|
+
end
|
450
|
+
end
|
451
|
+
|
452
|
+
def get_file_digest filename
|
453
|
+
File.open(filename, 'rb') do |io|
|
454
|
+
digest = OpenSSL::Digest.new(@digest_algorithm)
|
455
|
+
new_digest = OpenSSL::Digest.new(@new_digest_algorithm) if @new_digest_algorithm
|
456
|
+
|
457
|
+
buffer = ""
|
458
|
+
while io.read(409600, buffer) # 409600 seems like a sweet spot
|
459
|
+
digest.update(buffer)
|
460
|
+
new_digest.update(buffer) if @new_digest_algorithm
|
461
|
+
end
|
462
|
+
@new_digests[digest.hexdigest] = new_digest.hexdigest if @new_digest_algorithm
|
463
|
+
return digest.hexdigest
|
464
|
+
end
|
465
|
+
end
|
466
|
+
|
467
|
+
|
468
|
+
# Runtime state helpers
|
469
|
+
|
470
|
+
def any_missing_files?
|
471
|
+
@missing_files.length > 0
|
472
|
+
end
|
473
|
+
|
474
|
+
def any_exceptions?
|
475
|
+
@counters[:exceptions] > 0
|
476
|
+
end
|
477
|
+
|
478
|
+
def any_likely_damaged?
|
479
|
+
@counters[:likely_damaged] > 0
|
480
|
+
end
|
481
|
+
|
482
|
+
# UI helpers
|
483
|
+
|
484
|
+
def confirm text
|
485
|
+
if STDIN.tty? && STDOUT.tty?
|
486
|
+
puts "#{text} (y/n)?"
|
487
|
+
STDIN.gets.strip.downcase == "y"
|
488
|
+
end
|
489
|
+
end
|
490
|
+
|
491
|
+
def measure_time
|
492
|
+
start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
493
|
+
yield
|
494
|
+
elapsed = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start)
|
495
|
+
puts "Elapsed time: #{elapsed.to_i / 3600}h #{(elapsed.to_i % 3600) / 60}m #{'%.3f' % (elapsed % 60)}s" unless @options[:quiet]
|
496
|
+
end
|
497
|
+
|
498
|
+
def print_file_exception exception, filename
|
499
|
+
STDERR.print "EXCEPTION: #{exception.message}, processing file: "
|
500
|
+
begin
|
501
|
+
STDERR.print filename.encode('utf-8', universal_newline: true)
|
502
|
+
rescue
|
503
|
+
STDERR.print "(Unable to encode file name to utf-8) "
|
504
|
+
STDERR.print filename
|
505
|
+
end
|
506
|
+
STDERR.print "\n"
|
507
|
+
STDERR.flush
|
508
|
+
exception.backtrace.each { |line| STDERR.puts " " + line }
|
509
|
+
end
|
510
|
+
|
511
|
+
def print_counters
|
512
|
+
puts "#{@counters[:good]} file(s) passes digest check" if @counters[:good] > 0
|
513
|
+
puts "#{@counters[:updated]} file(s) are updated" if @counters[:updated] > 0
|
514
|
+
puts "#{@counters[:new]} file(s) are new" if @counters[:new] > 0
|
515
|
+
puts "#{@counters[:renamed]} file(s) are renamed" if @counters[:renamed] > 0
|
516
|
+
puts "#{@missing_files.length} file(s) are missing" if @missing_files.length > 0
|
517
|
+
puts "#{@counters[:likely_damaged]} file(s) are likely damaged (!)" if @counters[:likely_damaged] > 0
|
518
|
+
puts "#{@counters[:exceptions]} file(s) had exceptions occured during processing (!)" if @counters[:exceptions] > 0
|
292
519
|
end
|
293
520
|
end
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: file-digests
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.22
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Stanislav Senotrusov
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-10-
|
11
|
+
date: 2020-10-12 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: sqlite3
|
@@ -24,26 +24,34 @@ dependencies:
|
|
24
24
|
- - ">="
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: 1.3.0
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: openssl
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: 2.1.0
|
34
|
+
type: :runtime
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: 2.1.0
|
27
41
|
description: Calculate file digests and check for the possible file corruption
|
28
42
|
email: stan@senotrusov.com
|
29
43
|
executables:
|
30
44
|
- file-digests
|
31
|
-
- file-digests-auto
|
32
|
-
- file-digests-show-duplicates
|
33
|
-
- file-digests-test
|
34
45
|
extensions: []
|
35
46
|
extra_rdoc_files: []
|
36
47
|
files:
|
37
48
|
- bin/file-digests
|
38
|
-
- bin/file-digests-auto
|
39
|
-
- bin/file-digests-show-duplicates
|
40
|
-
- bin/file-digests-test
|
41
49
|
- lib/file-digests.rb
|
42
50
|
homepage: https://github.com/senotrusov/file-digests
|
43
51
|
licenses:
|
44
52
|
- Apache-2.0
|
45
53
|
metadata: {}
|
46
|
-
post_install_message:
|
54
|
+
post_install_message:
|
47
55
|
rdoc_options: []
|
48
56
|
require_paths:
|
49
57
|
- lib
|
@@ -59,7 +67,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
59
67
|
version: '0'
|
60
68
|
requirements: []
|
61
69
|
rubygems_version: 3.1.2
|
62
|
-
signing_key:
|
70
|
+
signing_key:
|
63
71
|
specification_version: 4
|
64
72
|
summary: file-digests
|
65
73
|
test_files: []
|
data/bin/file-digests-auto
DELETED