file-digests 0.0.7 → 0.0.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/bin/file-digests +2 -213
- data/bin/file-digests-test +2 -213
- data/lib/file-digests.rb +247 -0
- metadata +2 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 5b7e14619d7b7a27805bd7cf3a27240df66b3e6144063d356f04912df21be688
|
4
|
+
data.tar.gz: 0ee51f824ffb7b159ef4f147aaa46e239f367388e3531440918fda3756340002
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 80debef1838828adc8eefcc1c16f8a4467d4b7f871475d26af80a07bace18541673ae331260908a66e4942dd396d997a383dd9a3a4a1e73a5ff9fa253b87fe94
|
7
|
+
data.tar.gz: f1df92ff7323f09673dcb81cdaf33511cf4cdd0ffbc78b1c84955885bad6408c693e6abca1b4d39ea9bbfe4ea70d1cf82d62d91c6c65c130c96422c77b8f8a17
|
data/bin/file-digests
CHANGED
@@ -1,219 +1,8 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
2
|
|
3
|
-
require 'date'
|
4
|
-
require 'set'
|
5
|
-
require 'digest'
|
6
|
-
require 'fileutils'
|
7
|
-
require 'pathname'
|
8
|
-
require 'sqlite3'
|
9
|
-
|
10
|
-
def ensure_dir_exists path
|
11
|
-
if File.exist?(path)
|
12
|
-
unless File.directory?(path)
|
13
|
-
raise "#{path} is not a directory"
|
14
|
-
end
|
15
|
-
else
|
16
|
-
FileUtils.mkdir_p path
|
17
|
-
end
|
18
|
-
end
|
19
|
-
|
20
|
-
def measure_time
|
21
|
-
start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
22
|
-
yield
|
23
|
-
elapsed = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start).to_i
|
24
|
-
puts "Elapsed time: #{elapsed / 3600}h #{(elapsed % 3600) / 60}m #{elapsed % 60}s" unless QUIET
|
25
|
-
end
|
26
|
-
|
27
|
-
def patch_path_string path
|
28
|
-
Gem.win_platform? ? path.gsub(/\\/, '/') : path
|
29
|
-
end
|
30
|
-
|
31
|
-
class DigestDatabase
|
32
|
-
def initialize path
|
33
|
-
@db = SQLite3::Database.new(path.to_s)
|
34
|
-
|
35
|
-
unless @db.execute("SELECT name FROM sqlite_master WHERE type='table' AND name = 'digests'").length == 1
|
36
|
-
@db.execute 'PRAGMA encoding = "UTF-8"'
|
37
|
-
@db.execute "CREATE TABLE digests (
|
38
|
-
id INTEGER PRIMARY KEY,
|
39
|
-
filename TEXT,
|
40
|
-
mtime TEXT,
|
41
|
-
digest TEXT,
|
42
|
-
digest_check_time TEXT)"
|
43
|
-
@db.execute "CREATE UNIQUE INDEX digests_filename ON digests(filename)"
|
44
|
-
end
|
45
|
-
|
46
|
-
@db.execute 'PRAGMA journal_mode = "WAL"'
|
47
|
-
@db.execute 'PRAGMA synchronous = "NORMAL"'
|
48
|
-
@db.execute 'PRAGMA locking_mode = "EXCLUSIVE"'
|
49
|
-
@db.execute 'PRAGMA cache_size = "5000"'
|
50
|
-
|
51
|
-
@db.results_as_hash = true
|
52
|
-
@missing_files = Hash[@db.prepare("SELECT filename, digest FROM digests").execute!]
|
53
|
-
@new_files = {}
|
54
|
-
|
55
|
-
|
56
|
-
@insert = @db.prepare("INSERT INTO digests (filename, mtime, digest, digest_check_time) VALUES (?, ?, ?, datetime('now'))")
|
57
|
-
@find_by_filename = @db.prepare("SELECT id, mtime, digest FROM digests WHERE filename = ?")
|
58
|
-
@touch_digest_check_time = @db.prepare("UPDATE digests SET digest_check_time = datetime('now') WHERE id = ?")
|
59
|
-
@update_mtime_and_digest = @db.prepare("UPDATE digests SET mtime = ?, digest = ?, digest_check_time = datetime('now') WHERE id = ?")
|
60
|
-
@update_mtime = @db.prepare("UPDATE digests SET mtime = ?, digest_check_time = datetime('now') WHERE id = ?")
|
61
|
-
@delete_by_filename = @db.prepare("DELETE FROM digests WHERE filename = ?")
|
62
|
-
end
|
63
|
-
|
64
|
-
def insert_or_update file_path, mtime, digest
|
65
|
-
result = @find_by_filename.execute file_path
|
66
|
-
|
67
|
-
if found = result.next_hash
|
68
|
-
raise "Multiple records found" if result.next
|
69
|
-
|
70
|
-
@missing_files.delete(file_path)
|
71
|
-
|
72
|
-
if found['digest'] == digest
|
73
|
-
COUNTS[:good] += 1
|
74
|
-
# puts "GOOD: #{file_path}" unless QUIET
|
75
|
-
unless TEST_ONLY
|
76
|
-
if found['mtime'] == mtime
|
77
|
-
@touch_digest_check_time.execute found['id']
|
78
|
-
else
|
79
|
-
@update_mtime.execute mtime, found['id']
|
80
|
-
end
|
81
|
-
end
|
82
|
-
else
|
83
|
-
if found['mtime'] == mtime # Digest is different and mtime is the same
|
84
|
-
COUNTS[:likely_damaged] += 1
|
85
|
-
STDERR.puts "LIKELY DAMAGED: #{file_path}"
|
86
|
-
else
|
87
|
-
COUNTS[:updated] += 1
|
88
|
-
puts "UPDATED: #{file_path}" unless QUIET
|
89
|
-
unless TEST_ONLY
|
90
|
-
@update_mtime_and_digest.execute mtime, digest, found['id']
|
91
|
-
end
|
92
|
-
end
|
93
|
-
end
|
94
|
-
else
|
95
|
-
COUNTS[:new] += 1
|
96
|
-
puts "NEW: #{file_path}" unless QUIET
|
97
|
-
unless TEST_ONLY
|
98
|
-
@new_files[file_path] = digest
|
99
|
-
@insert.execute! file_path, mtime, digest
|
100
|
-
end
|
101
|
-
end
|
102
|
-
end
|
103
|
-
|
104
|
-
def process_missing_files
|
105
|
-
@missing_files.delete_if do |filename, digest|
|
106
|
-
if @new_files.value?(digest)
|
107
|
-
COUNTS[:renamed] += 1
|
108
|
-
unless TEST_ONLY
|
109
|
-
@delete_by_filename.execute filename
|
110
|
-
end
|
111
|
-
true
|
112
|
-
end
|
113
|
-
end
|
114
|
-
|
115
|
-
if (COUNTS[:missing] = @missing_files.length) > 0
|
116
|
-
puts "MISSING FILES:"
|
117
|
-
@missing_files.sort.to_h.each do |filename, digest|
|
118
|
-
puts filename
|
119
|
-
end
|
120
|
-
unless TEST_ONLY
|
121
|
-
puts "Remove missing files from the database (y/n)?"
|
122
|
-
if STDIN.gets.strip == "y"
|
123
|
-
@missing_files.each do |filename, digest|
|
124
|
-
@delete_by_filename.execute filename
|
125
|
-
end
|
126
|
-
end
|
127
|
-
end
|
128
|
-
end
|
129
|
-
end
|
130
|
-
end
|
131
|
-
|
132
|
-
class Checker
|
133
|
-
def initialize files_path, digest_database_path
|
134
|
-
@files_path = files_path
|
135
|
-
ensure_dir_exists @files_path
|
136
|
-
|
137
|
-
if digest_database_path
|
138
|
-
@digest_database_path = digest_database_path
|
139
|
-
ensure_dir_exists @digest_database_path.dirname
|
140
|
-
else
|
141
|
-
@digest_database_path = @files_path + '.file-digests.sqlite'
|
142
|
-
@skip_file_digests_sqlite = true
|
143
|
-
end
|
144
|
-
|
145
|
-
@digest_database = DigestDatabase.new @digest_database_path
|
146
|
-
end
|
147
|
-
|
148
|
-
def check
|
149
|
-
walk_files do |filename|
|
150
|
-
begin
|
151
|
-
process_file filename
|
152
|
-
rescue => exception
|
153
|
-
COUNTS[:exceptions] += 1
|
154
|
-
STDERR.puts "EXCEPTION: #{filename}: #{exception.message}"
|
155
|
-
end
|
156
|
-
end
|
157
|
-
|
158
|
-
@digest_database.process_missing_files
|
159
|
-
end
|
160
|
-
|
161
|
-
def walk_files
|
162
|
-
Dir.glob(@files_path + '**' + '*', File::FNM_DOTMATCH) do |filename|
|
163
|
-
next unless File.file? filename
|
164
|
-
next if @skip_file_digests_sqlite && filename == '.file-digests.sqlite'
|
165
|
-
next if @skip_file_digests_sqlite && filename == '.file-digests.sqlite-wal'
|
166
|
-
yield filename
|
167
|
-
end
|
168
|
-
end
|
169
|
-
|
170
|
-
def process_file filename
|
171
|
-
@digest_database.insert_or_update(
|
172
|
-
filename.delete_prefix(@files_path.to_s + '/'),
|
173
|
-
File.mtime(filename).utc.strftime('%Y-%m-%d %H:%M:%S'),
|
174
|
-
get_file_digest(filename)
|
175
|
-
)
|
176
|
-
end
|
177
|
-
|
178
|
-
def get_file_digest filename
|
179
|
-
File.open(filename, 'rb') do |io|
|
180
|
-
digest = Digest::SHA512.new
|
181
|
-
buffer = ""
|
182
|
-
while io.read(40960, buffer)
|
183
|
-
digest.update(buffer)
|
184
|
-
end
|
185
|
-
return digest.hexdigest
|
186
|
-
end
|
187
|
-
end
|
188
|
-
|
189
|
-
end
|
190
|
-
|
191
3
|
QUIET = (ENV["QUIET"] == "true")
|
192
4
|
TEST_ONLY = (ENV["TEST_ONLY"] == "true")
|
193
5
|
|
194
|
-
|
195
|
-
|
196
|
-
begin
|
197
|
-
if ARGV[0]
|
198
|
-
files_path = Pathname.new patch_path_string(ARGV[0])
|
199
|
-
else
|
200
|
-
files_path = Pathname.new patch_path_string(".")
|
201
|
-
end
|
202
|
-
|
203
|
-
digest_database_path = Pathname.new patch_path_string(ARGV[1]) if ARGV[1]
|
204
|
-
|
205
|
-
measure_time do
|
206
|
-
checker = Checker.new files_path, digest_database_path
|
207
|
-
checker.check
|
208
|
-
end
|
209
|
-
|
210
|
-
if COUNTS[:likely_damaged] > 0 || COUNTS[:exceptions] > 0
|
211
|
-
STDERR.puts "ERRORS WERE OCCURRED"
|
212
|
-
end
|
213
|
-
|
214
|
-
puts COUNTS.inspect
|
6
|
+
require 'file-digests'
|
215
7
|
|
216
|
-
|
217
|
-
STDERR.puts "EXCEPTION: #{exception.message}"
|
218
|
-
raise exception
|
219
|
-
end
|
8
|
+
FileDigests.perform_check
|
data/bin/file-digests-test
CHANGED
@@ -1,219 +1,8 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
2
|
|
3
|
-
require 'date'
|
4
|
-
require 'set'
|
5
|
-
require 'digest'
|
6
|
-
require 'fileutils'
|
7
|
-
require 'pathname'
|
8
|
-
require 'sqlite3'
|
9
|
-
|
10
|
-
def ensure_dir_exists path
|
11
|
-
if File.exist?(path)
|
12
|
-
unless File.directory?(path)
|
13
|
-
raise "#{path} is not a directory"
|
14
|
-
end
|
15
|
-
else
|
16
|
-
FileUtils.mkdir_p path
|
17
|
-
end
|
18
|
-
end
|
19
|
-
|
20
|
-
def measure_time
|
21
|
-
start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
22
|
-
yield
|
23
|
-
elapsed = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start).to_i
|
24
|
-
puts "Elapsed time: #{elapsed / 3600}h #{(elapsed % 3600) / 60}m #{elapsed % 60}s" unless QUIET
|
25
|
-
end
|
26
|
-
|
27
|
-
def patch_path_string path
|
28
|
-
Gem.win_platform? ? path.gsub(/\\/, '/') : path
|
29
|
-
end
|
30
|
-
|
31
|
-
class DigestDatabase
|
32
|
-
def initialize path
|
33
|
-
@db = SQLite3::Database.new(path.to_s)
|
34
|
-
|
35
|
-
unless @db.execute("SELECT name FROM sqlite_master WHERE type='table' AND name = 'digests'").length == 1
|
36
|
-
@db.execute 'PRAGMA encoding = "UTF-8"'
|
37
|
-
@db.execute "CREATE TABLE digests (
|
38
|
-
id INTEGER PRIMARY KEY,
|
39
|
-
filename TEXT,
|
40
|
-
mtime TEXT,
|
41
|
-
digest TEXT,
|
42
|
-
digest_check_time TEXT)"
|
43
|
-
@db.execute "CREATE UNIQUE INDEX digests_filename ON digests(filename)"
|
44
|
-
end
|
45
|
-
|
46
|
-
@db.execute 'PRAGMA journal_mode = "WAL"'
|
47
|
-
@db.execute 'PRAGMA synchronous = "NORMAL"'
|
48
|
-
@db.execute 'PRAGMA locking_mode = "EXCLUSIVE"'
|
49
|
-
@db.execute 'PRAGMA cache_size = "5000"'
|
50
|
-
|
51
|
-
@db.results_as_hash = true
|
52
|
-
@missing_files = Hash[@db.prepare("SELECT filename, digest FROM digests").execute!]
|
53
|
-
@new_files = {}
|
54
|
-
|
55
|
-
|
56
|
-
@insert = @db.prepare("INSERT INTO digests (filename, mtime, digest, digest_check_time) VALUES (?, ?, ?, datetime('now'))")
|
57
|
-
@find_by_filename = @db.prepare("SELECT id, mtime, digest FROM digests WHERE filename = ?")
|
58
|
-
@touch_digest_check_time = @db.prepare("UPDATE digests SET digest_check_time = datetime('now') WHERE id = ?")
|
59
|
-
@update_mtime_and_digest = @db.prepare("UPDATE digests SET mtime = ?, digest = ?, digest_check_time = datetime('now') WHERE id = ?")
|
60
|
-
@update_mtime = @db.prepare("UPDATE digests SET mtime = ?, digest_check_time = datetime('now') WHERE id = ?")
|
61
|
-
@delete_by_filename = @db.prepare("DELETE FROM digests WHERE filename = ?")
|
62
|
-
end
|
63
|
-
|
64
|
-
def insert_or_update file_path, mtime, digest
|
65
|
-
result = @find_by_filename.execute file_path
|
66
|
-
|
67
|
-
if found = result.next_hash
|
68
|
-
raise "Multiple records found" if result.next
|
69
|
-
|
70
|
-
@missing_files.delete(file_path)
|
71
|
-
|
72
|
-
if found['digest'] == digest
|
73
|
-
COUNTS[:good] += 1
|
74
|
-
# puts "GOOD: #{file_path}" unless QUIET
|
75
|
-
unless TEST_ONLY
|
76
|
-
if found['mtime'] == mtime
|
77
|
-
@touch_digest_check_time.execute found['id']
|
78
|
-
else
|
79
|
-
@update_mtime.execute mtime, found['id']
|
80
|
-
end
|
81
|
-
end
|
82
|
-
else
|
83
|
-
if found['mtime'] == mtime # Digest is different and mtime is the same
|
84
|
-
COUNTS[:likely_damaged] += 1
|
85
|
-
STDERR.puts "LIKELY DAMAGED: #{file_path}"
|
86
|
-
else
|
87
|
-
COUNTS[:updated] += 1
|
88
|
-
puts "UPDATED: #{file_path}" unless QUIET
|
89
|
-
unless TEST_ONLY
|
90
|
-
@update_mtime_and_digest.execute mtime, digest, found['id']
|
91
|
-
end
|
92
|
-
end
|
93
|
-
end
|
94
|
-
else
|
95
|
-
COUNTS[:new] += 1
|
96
|
-
puts "NEW: #{file_path}" unless QUIET
|
97
|
-
unless TEST_ONLY
|
98
|
-
@new_files[file_path] = digest
|
99
|
-
@insert.execute! file_path, mtime, digest
|
100
|
-
end
|
101
|
-
end
|
102
|
-
end
|
103
|
-
|
104
|
-
def process_missing_files
|
105
|
-
@missing_files.delete_if do |filename, digest|
|
106
|
-
if @new_files.value?(digest)
|
107
|
-
COUNTS[:renamed] += 1
|
108
|
-
unless TEST_ONLY
|
109
|
-
@delete_by_filename.execute filename
|
110
|
-
end
|
111
|
-
true
|
112
|
-
end
|
113
|
-
end
|
114
|
-
|
115
|
-
if (COUNTS[:missing] = @missing_files.length) > 0
|
116
|
-
puts "MISSING FILES:"
|
117
|
-
@missing_files.sort.to_h.each do |filename, digest|
|
118
|
-
puts filename
|
119
|
-
end
|
120
|
-
unless TEST_ONLY
|
121
|
-
puts "Remove missing files from the database (y/n)?"
|
122
|
-
if STDIN.gets.strip == "y"
|
123
|
-
@missing_files.each do |filename, digest|
|
124
|
-
@delete_by_filename.execute filename
|
125
|
-
end
|
126
|
-
end
|
127
|
-
end
|
128
|
-
end
|
129
|
-
end
|
130
|
-
end
|
131
|
-
|
132
|
-
class Checker
|
133
|
-
def initialize files_path, digest_database_path
|
134
|
-
@files_path = files_path
|
135
|
-
ensure_dir_exists @files_path
|
136
|
-
|
137
|
-
if digest_database_path
|
138
|
-
@digest_database_path = digest_database_path
|
139
|
-
ensure_dir_exists @digest_database_path.dirname
|
140
|
-
else
|
141
|
-
@digest_database_path = @files_path + '.file-digests.sqlite'
|
142
|
-
@skip_file_digests_sqlite = true
|
143
|
-
end
|
144
|
-
|
145
|
-
@digest_database = DigestDatabase.new @digest_database_path
|
146
|
-
end
|
147
|
-
|
148
|
-
def check
|
149
|
-
walk_files do |filename|
|
150
|
-
begin
|
151
|
-
process_file filename
|
152
|
-
rescue => exception
|
153
|
-
COUNTS[:exceptions] += 1
|
154
|
-
STDERR.puts "EXCEPTION: #{filename}: #{exception.message}"
|
155
|
-
end
|
156
|
-
end
|
157
|
-
|
158
|
-
@digest_database.process_missing_files
|
159
|
-
end
|
160
|
-
|
161
|
-
def walk_files
|
162
|
-
Dir.glob(@files_path + '**' + '*', File::FNM_DOTMATCH) do |filename|
|
163
|
-
next unless File.file? filename
|
164
|
-
next if @skip_file_digests_sqlite && filename == '.file-digests.sqlite'
|
165
|
-
next if @skip_file_digests_sqlite && filename == '.file-digests.sqlite-wal'
|
166
|
-
yield filename
|
167
|
-
end
|
168
|
-
end
|
169
|
-
|
170
|
-
def process_file filename
|
171
|
-
@digest_database.insert_or_update(
|
172
|
-
filename.delete_prefix(@files_path.to_s + '/'),
|
173
|
-
File.mtime(filename).utc.strftime('%Y-%m-%d %H:%M:%S'),
|
174
|
-
get_file_digest(filename)
|
175
|
-
)
|
176
|
-
end
|
177
|
-
|
178
|
-
def get_file_digest filename
|
179
|
-
File.open(filename, 'rb') do |io|
|
180
|
-
digest = Digest::SHA512.new
|
181
|
-
buffer = ""
|
182
|
-
while io.read(40960, buffer)
|
183
|
-
digest.update(buffer)
|
184
|
-
end
|
185
|
-
return digest.hexdigest
|
186
|
-
end
|
187
|
-
end
|
188
|
-
|
189
|
-
end
|
190
|
-
|
191
3
|
QUIET = (ENV["QUIET"] == "true")
|
192
4
|
TEST_ONLY = true
|
193
5
|
|
194
|
-
|
195
|
-
|
196
|
-
begin
|
197
|
-
if ARGV[0]
|
198
|
-
files_path = Pathname.new patch_path_string(ARGV[0])
|
199
|
-
else
|
200
|
-
files_path = Pathname.new patch_path_string(".")
|
201
|
-
end
|
202
|
-
|
203
|
-
digest_database_path = Pathname.new patch_path_string(ARGV[1]) if ARGV[1]
|
204
|
-
|
205
|
-
measure_time do
|
206
|
-
checker = Checker.new files_path, digest_database_path
|
207
|
-
checker.check
|
208
|
-
end
|
209
|
-
|
210
|
-
if COUNTS[:likely_damaged] > 0 || COUNTS[:exceptions] > 0
|
211
|
-
STDERR.puts "ERRORS WERE OCCURRED"
|
212
|
-
end
|
213
|
-
|
214
|
-
puts COUNTS.inspect
|
6
|
+
require 'file-digests'
|
215
7
|
|
216
|
-
|
217
|
-
STDERR.puts "EXCEPTION: #{exception.message}"
|
218
|
-
raise exception
|
219
|
-
end
|
8
|
+
FileDigests.perform_check
|
data/lib/file-digests.rb
ADDED
@@ -0,0 +1,247 @@
|
|
1
|
+
|
2
|
+
require 'date'
|
3
|
+
require 'set'
|
4
|
+
require 'digest'
|
5
|
+
require 'fileutils'
|
6
|
+
require 'pathname'
|
7
|
+
require 'sqlite3'
|
8
|
+
|
9
|
+
module FileDigests
|
10
|
+
|
11
|
+
def self.ensure_dir_exists path
|
12
|
+
if File.exist?(path)
|
13
|
+
unless File.directory?(path)
|
14
|
+
raise "#{path} is not a directory"
|
15
|
+
end
|
16
|
+
else
|
17
|
+
FileUtils.mkdir_p path
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
def self.measure_time
|
22
|
+
start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
23
|
+
yield
|
24
|
+
elapsed = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start).to_i
|
25
|
+
puts "Elapsed time: #{elapsed / 3600}h #{(elapsed % 3600) / 60}m #{elapsed % 60}s" unless QUIET
|
26
|
+
end
|
27
|
+
|
28
|
+
def self.patch_path_string path
|
29
|
+
Gem.win_platform? ? path.gsub(/\\/, '/') : path
|
30
|
+
end
|
31
|
+
|
32
|
+
def self.perform_check
|
33
|
+
checker = Checker.new ARGV[0], ARGV[1]
|
34
|
+
checker.perform_check
|
35
|
+
end
|
36
|
+
|
37
|
+
class DigestDatabase
|
38
|
+
def initialize path
|
39
|
+
@db = SQLite3::Database.new path.to_s
|
40
|
+
@db.results_as_hash = true
|
41
|
+
|
42
|
+
execute 'PRAGMA journal_mode = "WAL"'
|
43
|
+
execute 'PRAGMA synchronous = "NORMAL"'
|
44
|
+
execute 'PRAGMA locking_mode = "EXCLUSIVE"'
|
45
|
+
execute 'PRAGMA cache_size = "5000"'
|
46
|
+
|
47
|
+
unless execute("SELECT name FROM sqlite_master WHERE type='table' AND name = 'digests'").length == 1
|
48
|
+
execute 'PRAGMA encoding = "UTF-8"'
|
49
|
+
execute "CREATE TABLE digests (
|
50
|
+
id INTEGER PRIMARY KEY,
|
51
|
+
filename TEXT,
|
52
|
+
mtime TEXT,
|
53
|
+
digest TEXT,
|
54
|
+
digest_check_time TEXT)"
|
55
|
+
execute "CREATE UNIQUE INDEX digests_filename ON digests(filename)"
|
56
|
+
end
|
57
|
+
|
58
|
+
@missing_files = Hash[@db.prepare("SELECT filename, digest FROM digests").execute!]
|
59
|
+
@new_files = {}
|
60
|
+
|
61
|
+
prepare_method :insert, "INSERT INTO digests (filename, mtime, digest, digest_check_time) VALUES (?, ?, ?, datetime('now'))"
|
62
|
+
prepare_method :find_by_filename, "SELECT id, mtime, digest FROM digests WHERE filename = ?"
|
63
|
+
prepare_method :touch_digest_check_time, "UPDATE digests SET digest_check_time = datetime('now') WHERE id = ?"
|
64
|
+
prepare_method :update_mtime_and_digest, "UPDATE digests SET mtime = ?, digest = ?, digest_check_time = datetime('now') WHERE id = ?"
|
65
|
+
prepare_method :update_mtime, "UPDATE digests SET mtime = ?, digest_check_time = datetime('now') WHERE id = ?"
|
66
|
+
prepare_method :delete_by_filename, "DELETE FROM digests WHERE filename = ?"
|
67
|
+
end
|
68
|
+
|
69
|
+
def insert_or_update file_path, mtime, digest, counters
|
70
|
+
result = find_by_filename file_path
|
71
|
+
|
72
|
+
if found = result.next_hash
|
73
|
+
raise "Multiple records found" if result.next
|
74
|
+
|
75
|
+
@missing_files.delete(file_path)
|
76
|
+
|
77
|
+
if found['digest'] == digest
|
78
|
+
counters[:good] += 1
|
79
|
+
# puts "GOOD: #{file_path}" unless QUIET
|
80
|
+
unless TEST_ONLY
|
81
|
+
if found['mtime'] == mtime
|
82
|
+
touch_digest_check_time found['id']
|
83
|
+
else
|
84
|
+
update_mtime mtime, found['id']
|
85
|
+
end
|
86
|
+
end
|
87
|
+
else
|
88
|
+
if found['mtime'] == mtime # Digest is different and mtime is the same
|
89
|
+
counters[:likely_damaged] += 1
|
90
|
+
STDERR.puts "LIKELY DAMAGED: #{file_path}"
|
91
|
+
else
|
92
|
+
counters[:updated] += 1
|
93
|
+
puts "UPDATED: #{file_path}" unless QUIET
|
94
|
+
unless TEST_ONLY
|
95
|
+
update_mtime_and_digest mtime, digest, found['id']
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
99
|
+
else
|
100
|
+
counters[:new] += 1
|
101
|
+
puts "NEW: #{file_path}" unless QUIET
|
102
|
+
unless TEST_ONLY
|
103
|
+
@new_files[file_path] = digest
|
104
|
+
insert file_path, mtime, digest
|
105
|
+
end
|
106
|
+
end
|
107
|
+
end
|
108
|
+
|
109
|
+
def process_missing_files counters
|
110
|
+
@missing_files.delete_if do |filename, digest|
|
111
|
+
if @new_files.value?(digest)
|
112
|
+
counters[:renamed] += 1
|
113
|
+
unless TEST_ONLY
|
114
|
+
delete_by_filename filename
|
115
|
+
end
|
116
|
+
true
|
117
|
+
end
|
118
|
+
end
|
119
|
+
|
120
|
+
if (counters[:missing] = @missing_files.length) > 0
|
121
|
+
puts "\nMISSING FILES:"
|
122
|
+
@missing_files.sort.to_h.each do |filename, digest|
|
123
|
+
puts filename
|
124
|
+
end
|
125
|
+
unless TEST_ONLY
|
126
|
+
puts "Remove missing files from the database (y/n)?"
|
127
|
+
if STDIN.gets.strip.downcase == "y"
|
128
|
+
@db.transaction do
|
129
|
+
@missing_files.each do |filename, digest|
|
130
|
+
delete_by_filename filename
|
131
|
+
end
|
132
|
+
end
|
133
|
+
end
|
134
|
+
end
|
135
|
+
end
|
136
|
+
end
|
137
|
+
|
138
|
+
private
|
139
|
+
|
140
|
+
def execute *args, &block
|
141
|
+
@db.execute *args, &block
|
142
|
+
end
|
143
|
+
|
144
|
+
def prepare_method name, query
|
145
|
+
variable = "@#{name}"
|
146
|
+
instance_variable_set(variable, @db.prepare(query))
|
147
|
+
define_singleton_method name do |*args, &block|
|
148
|
+
instance_variable_get(variable).execute(*args, &block)
|
149
|
+
end
|
150
|
+
end
|
151
|
+
end
|
152
|
+
|
153
|
+
class Checker
|
154
|
+
def initialize files_path, digest_database_path
|
155
|
+
@files_path = Pathname.new(FileDigests::patch_path_string(files_path || ".")).cleanpath
|
156
|
+
@prefix_to_remove = @files_path.to_s + '/'
|
157
|
+
|
158
|
+
raise "Files path must be a readable directory" unless (File.directory?(@files_path) && File.readable?(@files_path))
|
159
|
+
|
160
|
+
@digest_database_path = if digest_database_path
|
161
|
+
Pathname.new(FileDigests::patch_path_string(digest_database_path)).cleanpath
|
162
|
+
else
|
163
|
+
@files_path + '.file-digests.sqlite'
|
164
|
+
end
|
165
|
+
|
166
|
+
if @files_path == @digest_database_path.dirname
|
167
|
+
@skip_file_digests_sqlite = true
|
168
|
+
end
|
169
|
+
|
170
|
+
FileDigests::ensure_dir_exists @digest_database_path.dirname
|
171
|
+
|
172
|
+
if File.exist?(@digest_database_path.dirname + '.file-digests.sha512')
|
173
|
+
@use_sha512 = true
|
174
|
+
end
|
175
|
+
|
176
|
+
@digest_database = DigestDatabase.new @digest_database_path
|
177
|
+
@counters = {good: 0, updated: 0, new: 0, missing: 0, renamed: 0, likely_damaged: 0, exceptions: 0}
|
178
|
+
end
|
179
|
+
|
180
|
+
def perform_check
|
181
|
+
FileDigests::measure_time do
|
182
|
+
walk_files do |filename|
|
183
|
+
process_file filename
|
184
|
+
end
|
185
|
+
end
|
186
|
+
|
187
|
+
@digest_database.process_missing_files @counters
|
188
|
+
|
189
|
+
if @counters[:likely_damaged] > 0 || @counters[:exceptions] > 0
|
190
|
+
STDERR.puts "ERRORS WERE OCCURRED"
|
191
|
+
end
|
192
|
+
|
193
|
+
puts @counters.inspect
|
194
|
+
end
|
195
|
+
|
196
|
+
def walk_files
|
197
|
+
Dir.glob(@files_path + '**' + '*', File::FNM_DOTMATCH) do |filename|
|
198
|
+
yield filename
|
199
|
+
end
|
200
|
+
end
|
201
|
+
|
202
|
+
def process_file filename
|
203
|
+
return if File.symlink? filename
|
204
|
+
|
205
|
+
stat = File.stat filename
|
206
|
+
|
207
|
+
return if stat.blockdev?
|
208
|
+
return if stat.chardev?
|
209
|
+
return if stat.directory?
|
210
|
+
return if stat.pipe?
|
211
|
+
unless stat.readable?
|
212
|
+
raise "File is not readable"
|
213
|
+
end
|
214
|
+
return if stat.socket?
|
215
|
+
|
216
|
+
if @skip_file_digests_sqlite
|
217
|
+
basename = File.basename(filename)
|
218
|
+
return if basename == '.file-digests.sha512'
|
219
|
+
return if basename == '.file-digests.sqlite'
|
220
|
+
return if basename == '.file-digests.sqlite-wal'
|
221
|
+
return if basename == '.file-digests.sqlite-shm'
|
222
|
+
end
|
223
|
+
|
224
|
+
@digest_database.insert_or_update(
|
225
|
+
filename.delete_prefix(@prefix_to_remove).encode('utf-8', universal_newline: true).unicode_normalize(:nfkc),
|
226
|
+
stat.mtime.utc.strftime('%Y-%m-%d %H:%M:%S'),
|
227
|
+
get_file_digest(filename),
|
228
|
+
@counters
|
229
|
+
)
|
230
|
+
rescue => exception
|
231
|
+
@counters[:exceptions] += 1
|
232
|
+
STDERR.puts "EXCEPTION: #{filename}: #{exception.message}"
|
233
|
+
end
|
234
|
+
|
235
|
+
def get_file_digest filename
|
236
|
+
File.open(filename, 'rb') do |io|
|
237
|
+
digest = (@use_sha512 ? Digest::SHA512 : Digest::SHA256).new
|
238
|
+
buffer = ""
|
239
|
+
while io.read(40960, buffer)
|
240
|
+
digest.update(buffer)
|
241
|
+
end
|
242
|
+
return digest.hexdigest
|
243
|
+
end
|
244
|
+
end
|
245
|
+
|
246
|
+
end
|
247
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: file-digests
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.12
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Stanislav Senotrusov
|
@@ -34,6 +34,7 @@ extra_rdoc_files: []
|
|
34
34
|
files:
|
35
35
|
- bin/file-digests
|
36
36
|
- bin/file-digests-test
|
37
|
+
- lib/file-digests.rb
|
37
38
|
homepage: https://github.com/senotrusov/file-digests
|
38
39
|
licenses:
|
39
40
|
- Apache-2.0
|