file-digests 0.0.21 → 0.0.22

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: e753f7530a113f1520ac771e1d79c7ee46e2258e32df0030d2789a65a46c4915
4
- data.tar.gz: 23211d18931a6edc0c79701aafc7383dee4b71d5695d46d7a6225c41f047580d
3
+ metadata.gz: f08714e86e275eb74108da01c667d403134366e6d7e9c5e8b08278d8c8842ba8
4
+ data.tar.gz: 57802538edae099807ede5460a07d6fd7ea6a5d0810f40d306db8f330435631d
5
5
  SHA512:
6
- metadata.gz: bd91dd28783b2ba8a088cd3d9e8b05c7e6d7b651323abc6958a2602859889d7aac1351c217a52804ce89ea840aa1e6b40f5abdce4fe6c33bda374200b0c73e50
7
- data.tar.gz: 6e79e4ffc8a3edb5cedbeed64abf85091abe6dc9cbd899aa81932a689dfa25b0b957c9245ed6ff0f4333cf2a1d13a1bb286e13ef673a675b65d7269fa77b5904
6
+ metadata.gz: 4c642c5aaf06d903114aabc65ec13ae3c5eb04c807b167c7eb1f3c63e8f56144ff3d93964fdc55cbcb33fad743cad635cfa61284171c94ed3b7a9a42c3efbca6
7
+ data.tar.gz: eebb6b444c6d0921f638200a27f97af5e9026ab0cf21b7b43b3e58f6f13ccc6323f482f93594ae993637a18b7611514469848307defe3796f412dd453c08b932
@@ -2,4 +2,4 @@
2
2
 
3
3
  require 'file-digests'
4
4
 
5
- FileDigests.perform_check
5
+ FileDigests.run_cli_utility
@@ -1,120 +1,246 @@
1
1
  require 'date'
2
- require 'set'
3
2
  require 'digest'
4
3
  require 'fileutils'
4
+ require 'openssl'
5
+ require 'optparse'
5
6
  require 'pathname'
7
+ require 'set'
6
8
  require 'sqlite3'
7
9
 
8
10
  class FileDigests
11
+ DIGEST_ALGORITHMS=["BLAKE2b512", "SHA3-256", "SHA512-256"]
9
12
 
10
- def self.perform_check
11
- options = {
12
- auto: (ENV["AUTO"] == "true"),
13
- quiet: (ENV["QUIET"] == "true"),
14
- test_only: (ENV["TEST_ONLY"] == "true")
15
- }
16
- file_digests = self.new ARGV[0], ARGV[1], options
17
- file_digests.perform_check
13
+ def self.canonical_digest_algorithm_name(string)
14
+ if string
15
+ index = DIGEST_ALGORITHMS.map(&:downcase).index(string.downcase)
16
+ index && DIGEST_ALGORITHMS[index]
17
+ end
18
18
  end
19
19
 
20
- def self.show_duplicates
21
- file_digests = self.new ARGV[0], ARGV[1]
22
- file_digests.show_duplicates
20
+ def canonical_digest_algorithm_name string
21
+ self.class.canonical_digest_algorithm_name string
22
+ end
23
+
24
+ def self.digest_algorithms_list_text
25
+ "Digest algorithm should be one of the following: #{DIGEST_ALGORITHMS.join ", "}"
26
+ end
27
+
28
+ def self.parse_cli_options
29
+ options = {}
30
+
31
+ OptionParser.new do |opts|
32
+ opts.banner = [
33
+ "Usage: file-digests [options] [path/to/directory] [path/to/database_file]",
34
+ " By default the current directory will be operated upon, and the database file will be placed to the current directory as well.",
35
+ " Should you wish to check current directory but place the database elsewhere, you could provide \".\" as a first argument, and the path to a database_file as a second."
36
+ ].join "\n"
37
+
38
+ opts.on("-a", "--auto", "Do not ask for any confirmation") do
39
+ options[:auto] = true
40
+ end
41
+
42
+ opts.on(
43
+ '--digest=DIGEST',
44
+ 'Select a digest algorithm to use. Default is "BLAKE2b512".',
45
+ 'You might also consider to use slower "SHA512-256" or even more slower "SHA3-256".',
46
+ "#{digest_algorithms_list_text}.",
47
+ 'You only need to specify an algorithm on the first run, your choice will be saved to a database.',
48
+ 'Any time later you could specify a new algorithm to change the current one.',
49
+ 'Transition to a new algorithm will only occur if all files pass the check by digests which were stored using the old one.'
50
+ ) do |value|
51
+ digest_algorithm = canonical_digest_algorithm_name(value)
52
+ unless digest_algorithm
53
+ STDERR.puts "ERROR: #{digest_algorithms_list_text}"
54
+ exit 1
55
+ end
56
+ options[:digest_algorithm] = digest_algorithm
57
+ end
58
+
59
+ opts.on("-d", "--duplicates", "Show the list of duplicate files, based on the information out of the database") do
60
+ options[:action] = :show_duplicates
61
+ end
62
+
63
+ opts.on("-t", "--test", "Perform only the test, do not modify the digest database") do
64
+ options[:test_only] = true
65
+ end
66
+
67
+ opts.on("-q", "--quiet", "Less verbose output, stil report any found issues") do
68
+ options[:quiet] = true
69
+ end
70
+
71
+ opts.on("-v", "--verbose", "More verbose output") do
72
+ options[:verbose] = true
73
+ end
74
+
75
+ opts.on("-h", "--help", "Prints this help") do
76
+ puts opts
77
+ exit
78
+ end
79
+ end.parse!
80
+ options
81
+ end
82
+
83
+ def self.run_cli_utility
84
+ options = parse_cli_options
85
+
86
+ file_digests = self.new ARGV[0], ARGV[1], options
87
+ file_digests.send(options[:action] || :perform_check)
23
88
  end
24
89
 
25
90
  def initialize files_path, digest_database_path, options = {}
26
91
  @options = options
27
92
 
28
- @files_path = cleanup_path(files_path || ".")
29
- @prefix_to_remove = @files_path.to_s + '/'
30
-
31
- raise "Files path must be a readable directory" unless (File.directory?(@files_path) && File.readable?(@files_path))
93
+ initialize_paths files_path, digest_database_path
94
+ initialize_database
32
95
 
33
- @digest_database_path = if digest_database_path
34
- cleanup_path(digest_database_path)
96
+ if @digest_algorithm = canonical_digest_algorithm_name(get_metadata("digest_algorithm"))
97
+ if @options[:digest_algorithm] && @options[:digest_algorithm] != @digest_algorithm
98
+ @new_digest_algorithm = @options[:digest_algorithm]
99
+ end
35
100
  else
36
- @files_path + '.file-digests.sqlite'
101
+ @digest_algorithm = (@options[:digest_algorithm] || "BLAKE2b512")
102
+ set_metadata "digest_algorithm", @digest_algorithm
37
103
  end
38
104
 
39
- if File.directory?(@digest_database_path)
40
- @digest_database_path = @digest_database_path + '.file-digests.sqlite'
41
- end
105
+ puts "Using #{@digest_algorithm} digest algorithm" if @options[:verbose]
106
+ end
42
107
 
43
- if @files_path == @digest_database_path.dirname
44
- @skip_file_digests_sqlite = true
45
- end
108
+ def initialize_paths files_path, digest_database_path
109
+ @files_path = cleanup_path(files_path || ".")
110
+
111
+ raise "Files path must be a readable directory" unless (File.directory?(@files_path) && File.readable?(@files_path))
46
112
 
113
+ @digest_database_path = digest_database_path ? cleanup_path(digest_database_path) : @files_path
114
+ @digest_database_path += '.file-digests.sqlite' if File.directory?(@digest_database_path)
47
115
  ensure_dir_exists @digest_database_path.dirname
48
116
 
49
- # Please do not use this flag, support for sha512 is here for backward compatibility, and one day it will be removed.
50
- if File.exist?(@digest_database_path.dirname + '.file-digests.sha512')
51
- @use_sha512 = true
117
+ if @options[:verbose]
118
+ puts "Target directory: #{@files_path}"
119
+ puts "Database location: #{@digest_database_path}"
52
120
  end
53
-
54
- initialize_database @digest_database_path
55
121
  end
56
122
 
57
- def initialize_database path
58
- @db = SQLite3::Database.new path.to_s
123
+ def initialize_database
124
+ @db = SQLite3::Database.new @digest_database_path.to_s
59
125
  @db.results_as_hash = true
60
126
 
127
+ file_digests_gem_version = Gem.loaded_specs["file-digests"]&.version&.to_s
128
+
129
+ execute 'PRAGMA encoding = "UTF-8"'
61
130
  execute 'PRAGMA journal_mode = "WAL"'
62
131
  execute 'PRAGMA synchronous = "NORMAL"'
63
132
  execute 'PRAGMA locking_mode = "EXCLUSIVE"'
64
133
  execute 'PRAGMA cache_size = "5000"'
65
134
 
66
- unless execute("SELECT name FROM sqlite_master WHERE type='table' AND name = 'digests'").length == 1
67
- execute 'PRAGMA encoding = "UTF-8"'
68
- execute "CREATE TABLE digests (
69
- id INTEGER PRIMARY KEY,
70
- filename TEXT,
71
- mtime TEXT,
72
- digest TEXT,
73
- digest_check_time TEXT)"
74
- execute "CREATE UNIQUE INDEX digests_filename ON digests(filename)"
75
- end
135
+ @db.transaction(:exclusive) do
136
+ unless table_exist?("metadata")
137
+ execute "CREATE TABLE metadata (
138
+ key TEXT NOT NULL PRIMARY KEY,
139
+ value TEXT)"
140
+ execute "CREATE UNIQUE INDEX metadata_key ON metadata(key)"
141
+ set_metadata("metadata_table_created_by_gem_version", file_digests_gem_version) if file_digests_gem_version
142
+ end
143
+
144
+ prepare_method :set_metadata_query, "INSERT INTO metadata (key, value) VALUES (?, ?) ON CONFLICT (key) DO UPDATE SET value=excluded.value"
145
+ prepare_method :get_metadata_query, "SELECT value FROM metadata WHERE key = ?"
146
+
147
+ # Heuristic to detect database version 1 (metadata was not stored back then)
148
+ unless get_metadata("database_version")
149
+ if table_exist?("digests")
150
+ set_metadata "database_version", "1"
151
+ end
152
+ end
153
+
154
+ unless table_exist?("digests")
155
+ execute "CREATE TABLE digests (
156
+ id INTEGER NOT NULL PRIMARY KEY,
157
+ filename TEXT NOT NULL,
158
+ mtime TEXT,
159
+ digest TEXT NOT NULL,
160
+ digest_check_time TEXT NOT NULL)"
161
+ execute "CREATE UNIQUE INDEX digests_filename ON digests(filename)"
162
+ set_metadata("digests_table_created_by_gem_version", file_digests_gem_version) if file_digests_gem_version
163
+ end
164
+
165
+ prepare_method :insert, "INSERT INTO digests (filename, mtime, digest, digest_check_time) VALUES (?, ?, ?, datetime('now'))"
166
+ prepare_method :find_by_filename_query, "SELECT id, mtime, digest FROM digests WHERE filename = ?"
167
+ prepare_method :touch_digest_check_time, "UPDATE digests SET digest_check_time = datetime('now') WHERE id = ?"
168
+ prepare_method :update_mtime_and_digest, "UPDATE digests SET mtime = ?, digest = ?, digest_check_time = datetime('now') WHERE id = ?"
169
+ prepare_method :update_mtime, "UPDATE digests SET mtime = ?, digest_check_time = datetime('now') WHERE id = ?"
170
+ prepare_method :delete_by_filename, "DELETE FROM digests WHERE filename = ?"
171
+ prepare_method :query_duplicates, "SELECT digest, filename FROM digests WHERE digest IN (SELECT digest FROM digests GROUP BY digest HAVING count(*) > 1) ORDER BY digest, filename;"
172
+ prepare_method :update_digest_to_new_digest, "UPDATE digests SET digest = ? WHERE digest = ?"
173
+
174
+ unless get_metadata("database_version")
175
+ set_metadata "database_version", "2"
176
+ end
177
+
178
+ # Convert database from 1st to 2nd version
179
+ unless get_metadata("digest_algorithm")
180
+ if get_metadata("database_version") == "1"
181
+ if File.exist?(@digest_database_path.dirname + '.file-digests.sha512')
182
+ set_metadata("digest_algorithm", "SHA512")
183
+ else
184
+ set_metadata("digest_algorithm", "SHA256")
185
+ end
186
+ set_metadata "database_version", "2"
187
+ end
188
+ end
76
189
 
77
- prepare_method :insert, "INSERT INTO digests (filename, mtime, digest, digest_check_time) VALUES (?, ?, ?, datetime('now'))"
78
- prepare_method :find_by_filename, "SELECT id, mtime, digest FROM digests WHERE filename = ?"
79
- prepare_method :touch_digest_check_time, "UPDATE digests SET digest_check_time = datetime('now') WHERE id = ?"
80
- prepare_method :update_mtime_and_digest, "UPDATE digests SET mtime = ?, digest = ?, digest_check_time = datetime('now') WHERE id = ?"
81
- prepare_method :update_mtime, "UPDATE digests SET mtime = ?, digest_check_time = datetime('now') WHERE id = ?"
82
- prepare_method :delete_by_filename, "DELETE FROM digests WHERE filename = ?"
83
- prepare_method :query_duplicates, "SELECT digest, filename FROM digests WHERE digest IN (SELECT digest FROM digests GROUP BY digest HAVING count(*) > 1) ORDER BY digest, filename;"
190
+ end
84
191
  end
85
192
 
86
193
  def perform_check
87
- @counters = {good: 0, updated: 0, new: 0, missing: 0, renamed: 0, likely_damaged: 0, exceptions: 0}
88
- @missing_files = Hash[@db.prepare("SELECT filename, digest FROM digests").execute!]
89
- @new_files = {}
194
+ perhaps_transaction(@new_digest_algorithm, :exclusive) do
195
+ @counters = {good: 0, updated: 0, new: 0, renamed: 0, likely_damaged: 0, exceptions: 0}
196
+ @new_files = {}
197
+ @new_digests = {}
90
198
 
91
- measure_time do
92
- walk_files do |filename|
93
- process_file filename
199
+ @missing_files = Hash[@db.prepare("SELECT filename, digest FROM digests").execute!]
200
+
201
+ measure_time do
202
+ walk_files do |filename|
203
+ process_file filename
204
+ end
94
205
  end
95
- end
96
206
 
97
- track_renames
207
+ track_renames
208
+
209
+ if any_missing_files?
210
+ if any_exceptions?
211
+ STDERR.puts "Due to previously occurred errors, database cleanup from missing files will be skipped this time."
212
+ else
213
+ print_missing_files
214
+ if !@options[:test_only] && (@options[:auto] || confirm("Remove missing files from the database"))
215
+ remove_missing_files
216
+ end
217
+ end
218
+ end
219
+
220
+ if @new_digest_algorithm && !@options[:test_only]
221
+ if any_missing_files? || any_likely_damaged? || any_exceptions?
222
+ STDERR.puts "ERROR: New digest algorithm will not be in effect until there are files that are missing, likely damaged, or processed with an exception."
223
+ else
224
+ @new_digests.each do |old_digest, new_digest|
225
+ update_digest_to_new_digest new_digest, old_digest
226
+ end
227
+ set_metadata "digest_algorithm", @new_digest_algorithm
228
+ end
229
+ end
98
230
 
99
- if any_missing_files?
100
- print_missing_files
101
- if !@options[:test_only] && (@options[:auto] || confirm("Remove missing files from the database"))
102
- remove_missing_files
231
+ if any_likely_damaged? || any_exceptions?
232
+ STDERR.puts "PLEASE REVIEW ERRORS THAT WERE OCCURRED!"
103
233
  end
104
- end
105
234
 
106
- if @counters[:likely_damaged] > 0 || @counters[:exceptions] > 0
107
- STDERR.puts "ERRORS WERE OCCURRED"
108
- end
235
+ set_metadata(@options[:test_only] ? "latest_test_only_check_time" : "latest_complete_check_time", time_to_database(Time.now))
109
236
 
110
- puts @counters.inspect
237
+ print_counters
238
+ end
111
239
  end
112
240
 
113
241
  def show_duplicates
114
242
  current_digest = nil
115
- result = query_duplicates
116
-
117
- while found = result.next_hash do
243
+ query_duplicates.each do |found|
118
244
  if current_digest != found['digest']
119
245
  puts "" if current_digest
120
246
  current_digest = found['digest']
@@ -135,117 +261,72 @@ class FileDigests
135
261
  return if stat.chardev?
136
262
  return if stat.directory?
137
263
  return if stat.pipe?
138
- unless stat.readable?
139
- raise "File is not readable"
140
- end
141
264
  return if stat.socket?
142
265
 
143
- if @skip_file_digests_sqlite
144
- basename = File.basename(filename)
145
- return if basename == '.file-digests.sha512'
146
- return if basename == '.file-digests.sqlite'
147
- return if basename == '.file-digests.sqlite-wal'
148
- return if basename == '.file-digests.sqlite-shm'
266
+ raise "File is not readable" unless stat.readable?
267
+
268
+ if filename == "#{@digest_database_path}" ||
269
+ filename == "#{@digest_database_path}-wal" ||
270
+ filename == "#{@digest_database_path}-shm"
271
+ puts "SKIPPING DATABASE FILE: #{filename}" if @options[:verbose]
272
+ return
149
273
  end
150
274
 
151
- insert_or_update(
152
- filename.delete_prefix(@prefix_to_remove).encode('utf-8', universal_newline: true).unicode_normalize(:nfkc),
153
- stat.mtime.utc.strftime('%Y-%m-%d %H:%M:%S'),
154
- get_file_digest(filename)
155
- )
156
- rescue => exception
157
- @counters[:exceptions] += 1
158
- STDERR.puts "EXCEPTION: #{filename.encode('utf-8', universal_newline: true)}: #{exception.message}"
159
- end
275
+ normalized_filename = filename.delete_prefix("#{@files_path.to_s}/").encode('utf-8', universal_newline: true).unicode_normalize(:nfkc)
276
+ mtime_string = time_to_database stat.mtime
160
277
 
161
- def patch_path_string path
162
- Gem.win_platform? ? path.gsub(/\\/, '/') : path
163
- end
278
+ process_file_indeed normalized_filename, mtime_string, get_file_digest(filename)
164
279
 
165
- def cleanup_path path
166
- Pathname.new(patch_path_string(path)).cleanpath
280
+ rescue => exception
281
+ @counters[:exceptions] += 1
282
+ print_file_exception exception, filename
167
283
  end
168
284
 
169
- def ensure_dir_exists path
170
- if File.exist?(path)
171
- unless File.directory?(path)
172
- raise "#{path} is not a directory"
173
- end
285
+ def process_file_indeed filename, mtime, digest
286
+ if found = find_by_filename(filename)
287
+ process_previously_seen_file found, filename, mtime, digest
174
288
  else
175
- FileUtils.mkdir_p path
176
- end
177
- end
178
-
179
- def walk_files
180
- Dir.glob(@files_path + '**' + '*', File::FNM_DOTMATCH) do |filename|
181
- yield filename
289
+ process_new_file filename, mtime, digest
182
290
  end
183
291
  end
184
292
 
185
- def get_file_digest filename
186
- File.open(filename, 'rb') do |io|
187
- digest = (@use_sha512 ? Digest::SHA512 : Digest::SHA256).new
188
- buffer = ""
189
- while io.read(40960, buffer)
190
- digest.update(buffer)
293
+ def process_previously_seen_file found, filename, mtime, digest
294
+ @missing_files.delete(filename)
295
+ if found['digest'] == digest
296
+ @counters[:good] += 1
297
+ puts "GOOD: #{filename}" if @options[:verbose]
298
+ unless @options[:test_only]
299
+ if found['mtime'] == mtime
300
+ touch_digest_check_time found['id']
301
+ else
302
+ update_mtime mtime, found['id']
303
+ end
304
+ end
305
+ else
306
+ if found['mtime'] == mtime # Digest is different and mtime is the same
307
+ @counters[:likely_damaged] += 1
308
+ STDERR.puts "LIKELY DAMAGED: #{filename}"
309
+ else
310
+ @counters[:updated] += 1
311
+ puts "UPDATED: #{filename}" unless @options[:quiet]
312
+ unless @options[:test_only]
313
+ update_mtime_and_digest mtime, digest, found['id']
314
+ end
191
315
  end
192
- return digest.hexdigest
193
316
  end
194
317
  end
195
318
 
196
- def confirm text
197
- if STDIN.tty? && STDOUT.tty?
198
- puts "#{text} (y/n)?"
199
- STDIN.gets.strip.downcase == "y"
319
+ def process_new_file filename, mtime, digest
320
+ @counters[:new] += 1
321
+ puts "NEW: #{filename}" unless @options[:quiet]
322
+ unless @options[:test_only]
323
+ @new_files[filename] = digest
324
+ insert filename, mtime, digest
200
325
  end
201
326
  end
202
327
 
203
- def measure_time
204
- start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
205
- yield
206
- elapsed = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start).to_i
207
- puts "Elapsed time: #{elapsed / 3600}h #{(elapsed % 3600) / 60}m #{elapsed % 60}s" unless @options[:quiet]
208
- end
209
-
210
- def insert_or_update file_path, mtime, digest
211
- result = find_by_filename file_path
212
-
213
- if found = result.next_hash
214
- raise "Multiple records found" if result.next
215
328
 
216
- @missing_files.delete(file_path)
217
-
218
- if found['digest'] == digest
219
- @counters[:good] += 1
220
- # puts "GOOD: #{file_path}" unless @options[:quiet]
221
- unless @options[:test_only]
222
- if found['mtime'] == mtime
223
- touch_digest_check_time found['id']
224
- else
225
- update_mtime mtime, found['id']
226
- end
227
- end
228
- else
229
- if found['mtime'] == mtime # Digest is different and mtime is the same
230
- @counters[:likely_damaged] += 1
231
- STDERR.puts "LIKELY DAMAGED: #{file_path}"
232
- else
233
- @counters[:updated] += 1
234
- puts "UPDATED: #{file_path}" unless @options[:quiet]
235
- unless @options[:test_only]
236
- update_mtime_and_digest mtime, digest, found['id']
237
- end
238
- end
239
- end
240
- else
241
- @counters[:new] += 1
242
- puts "NEW: #{file_path}" unless @options[:quiet]
243
- unless @options[:test_only]
244
- @new_files[file_path] = digest
245
- insert file_path, mtime, digest
246
- end
247
- end
248
- end
329
+ # Renames and missing files
249
330
 
250
331
  def track_renames
251
332
  @missing_files.delete_if do |filename, digest|
@@ -257,11 +338,6 @@ class FileDigests
257
338
  true
258
339
  end
259
340
  end
260
- @counters[:missing] = @missing_files.length
261
- end
262
-
263
- def any_missing_files?
264
- @missing_files.length > 0
265
341
  end
266
342
 
267
343
  def print_missing_files
@@ -272,22 +348,173 @@ class FileDigests
272
348
  end
273
349
 
274
350
  def remove_missing_files
275
- @db.transaction do
351
+ nested_transaction do
276
352
  @missing_files.each do |filename, digest|
277
353
  delete_by_filename filename
278
354
  end
355
+ @missing_files = {}
279
356
  end
280
357
  end
281
358
 
359
+
360
+ # Database helpers
361
+
282
362
  def execute *args, &block
283
363
  @db.execute *args, &block
284
364
  end
285
365
 
366
+ def nested_transaction(mode)
367
+ if @db.transaction_active?
368
+ yield
369
+ else
370
+ @db.transaction(mode) do
371
+ yield
372
+ end
373
+ end
374
+ end
375
+
376
+ def perhaps_transaction(condition, mode)
377
+ if condition
378
+ @db.transaction(mode) do
379
+ yield
380
+ end
381
+ else
382
+ yield
383
+ end
384
+ end
385
+
386
+ def table_exist? table_name
387
+ execute("SELECT name FROM sqlite_master WHERE type='table' AND name = '#{table_name}'").length == 1
388
+ end
389
+
286
390
  def prepare_method name, query
287
391
  variable = "@#{name}"
392
+
288
393
  instance_variable_set(variable, @db.prepare(query))
394
+
289
395
  define_singleton_method name do |*args, &block|
290
396
  instance_variable_get(variable).execute(*args, &block)
291
397
  end
398
+
399
+ define_singleton_method "#{name}!" do |*args, &block|
400
+ instance_variable_get(variable).execute!(*args, &block)
401
+ end
402
+ end
403
+
404
+ def set_metadata key, value
405
+ set_metadata_query key, value
406
+ puts "#{key} set to: #{value}" if @options[:verbose]
407
+ value
408
+ end
409
+
410
+ def get_metadata key
411
+ get_metadata_query!(key)&.first&.first
412
+ end
413
+
414
+ def find_by_filename filename
415
+ result = find_by_filename_query filename
416
+ found = result.next
417
+ raise "Multiple records found" if result.next
418
+ found
419
+ end
420
+
421
+ def time_to_database time
422
+ time.utc.strftime('%Y-%m-%d %H:%M:%S')
423
+ end
424
+
425
+
426
+ # Filesystem-related helpers
427
+
428
+ def patch_path_string path
429
+ Gem.win_platform? ? path.gsub(/\\/, '/') : path
430
+ end
431
+
432
+ def cleanup_path path
433
+ Pathname.new(patch_path_string(path)).cleanpath
434
+ end
435
+
436
+ def ensure_dir_exists path
437
+ if File.exist?(path)
438
+ unless File.directory?(path)
439
+ raise "#{path} is not a directory"
440
+ end
441
+ else
442
+ FileUtils.mkdir_p path
443
+ end
444
+ end
445
+
446
+ def walk_files
447
+ Dir.glob(@files_path + '**' + '*', File::FNM_DOTMATCH) do |filename|
448
+ yield filename
449
+ end
450
+ end
451
+
452
+ def get_file_digest filename
453
+ File.open(filename, 'rb') do |io|
454
+ digest = OpenSSL::Digest.new(@digest_algorithm)
455
+ new_digest = OpenSSL::Digest.new(@new_digest_algorithm) if @new_digest_algorithm
456
+
457
+ buffer = ""
458
+ while io.read(409600, buffer) # 409600 seems like a sweet spot
459
+ digest.update(buffer)
460
+ new_digest.update(buffer) if @new_digest_algorithm
461
+ end
462
+ @new_digests[digest.hexdigest] = new_digest.hexdigest if @new_digest_algorithm
463
+ return digest.hexdigest
464
+ end
465
+ end
466
+
467
+
468
+ # Runtime state helpers
469
+
470
+ def any_missing_files?
471
+ @missing_files.length > 0
472
+ end
473
+
474
+ def any_exceptions?
475
+ @counters[:exceptions] > 0
476
+ end
477
+
478
+ def any_likely_damaged?
479
+ @counters[:likely_damaged] > 0
480
+ end
481
+
482
+ # UI helpers
483
+
484
+ def confirm text
485
+ if STDIN.tty? && STDOUT.tty?
486
+ puts "#{text} (y/n)?"
487
+ STDIN.gets.strip.downcase == "y"
488
+ end
489
+ end
490
+
491
+ def measure_time
492
+ start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
493
+ yield
494
+ elapsed = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start)
495
+ puts "Elapsed time: #{elapsed.to_i / 3600}h #{(elapsed.to_i % 3600) / 60}m #{'%.3f' % (elapsed % 60)}s" unless @options[:quiet]
496
+ end
497
+
498
+ def print_file_exception exception, filename
499
+ STDERR.print "EXCEPTION: #{exception.message}, processing file: "
500
+ begin
501
+ STDERR.print filename.encode('utf-8', universal_newline: true)
502
+ rescue
503
+ STDERR.print "(Unable to encode file name to utf-8) "
504
+ STDERR.print filename
505
+ end
506
+ STDERR.print "\n"
507
+ STDERR.flush
508
+ exception.backtrace.each { |line| STDERR.puts " " + line }
509
+ end
510
+
511
+ def print_counters
512
+ puts "#{@counters[:good]} file(s) passes digest check" if @counters[:good] > 0
513
+ puts "#{@counters[:updated]} file(s) are updated" if @counters[:updated] > 0
514
+ puts "#{@counters[:new]} file(s) are new" if @counters[:new] > 0
515
+ puts "#{@counters[:renamed]} file(s) are renamed" if @counters[:renamed] > 0
516
+ puts "#{@missing_files.length} file(s) are missing" if @missing_files.length > 0
517
+ puts "#{@counters[:likely_damaged]} file(s) are likely damaged (!)" if @counters[:likely_damaged] > 0
518
+ puts "#{@counters[:exceptions]} file(s) had exceptions occured during processing (!)" if @counters[:exceptions] > 0
292
519
  end
293
520
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: file-digests
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.21
4
+ version: 0.0.22
5
5
  platform: ruby
6
6
  authors:
7
7
  - Stanislav Senotrusov
8
- autorequire:
8
+ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2020-10-08 00:00:00.000000000 Z
11
+ date: 2020-10-12 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: sqlite3
@@ -24,26 +24,34 @@ dependencies:
24
24
  - - ">="
25
25
  - !ruby/object:Gem::Version
26
26
  version: 1.3.0
27
+ - !ruby/object:Gem::Dependency
28
+ name: openssl
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ">="
32
+ - !ruby/object:Gem::Version
33
+ version: 2.1.0
34
+ type: :runtime
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ">="
39
+ - !ruby/object:Gem::Version
40
+ version: 2.1.0
27
41
  description: Calculate file digests and check for the possible file corruption
28
42
  email: stan@senotrusov.com
29
43
  executables:
30
44
  - file-digests
31
- - file-digests-auto
32
- - file-digests-show-duplicates
33
- - file-digests-test
34
45
  extensions: []
35
46
  extra_rdoc_files: []
36
47
  files:
37
48
  - bin/file-digests
38
- - bin/file-digests-auto
39
- - bin/file-digests-show-duplicates
40
- - bin/file-digests-test
41
49
  - lib/file-digests.rb
42
50
  homepage: https://github.com/senotrusov/file-digests
43
51
  licenses:
44
52
  - Apache-2.0
45
53
  metadata: {}
46
- post_install_message:
54
+ post_install_message:
47
55
  rdoc_options: []
48
56
  require_paths:
49
57
  - lib
@@ -59,7 +67,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
59
67
  version: '0'
60
68
  requirements: []
61
69
  rubygems_version: 3.1.2
62
- signing_key:
70
+ signing_key:
63
71
  specification_version: 4
64
72
  summary: file-digests
65
73
  test_files: []
@@ -1,7 +0,0 @@
1
- #!/usr/bin/env ruby
2
-
3
- ENV["AUTO"] = "true"
4
-
5
- require 'file-digests'
6
-
7
- FileDigests.perform_check
@@ -1,5 +0,0 @@
1
- #!/usr/bin/env ruby
2
-
3
- require 'file-digests'
4
-
5
- FileDigests.show_duplicates
@@ -1,7 +0,0 @@
1
- #!/usr/bin/env ruby
2
-
3
- ENV["TEST_ONLY"] = "true"
4
-
5
- require 'file-digests'
6
-
7
- FileDigests.perform_check