file-digests 0.0.8 → 0.0.9
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/bin/file-digests +2 -214
- data/bin/file-digests-test +2 -214
- data/lib/file-digests.rb +237 -0
- metadata +2 -1
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ee2508a2c4d70b5a4847f6403d74848db85145b181ff103c4b9dbc631ea03720
|
4
|
+
data.tar.gz: 7e0b68a73d34e54ba8b98b884f2184eda906ca9c11dac04d98607d8727ec1911
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: db27ac6edfbe7f6c243d4e2c54cf2c0d6419328e751151e6745bad97836881116ea5710490532cf87c1c4a5f6f50d7495b1ae05863f78b66436a20e5c83f7884
|
7
|
+
data.tar.gz: 3959e854c8b5539ba8e1882857730fd17f91ffb2f31252db070b31dcc69fb6fdf4ac02f99e2ce53ba1767085bada121840f2bb582f708ca6881128f92359cbb9
|
data/bin/file-digests
CHANGED
@@ -1,220 +1,8 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
2
|
|
3
|
-
require '
|
4
|
-
require 'set'
|
5
|
-
require 'digest'
|
6
|
-
require 'fileutils'
|
7
|
-
require 'pathname'
|
8
|
-
require 'sqlite3'
|
9
|
-
|
10
|
-
def ensure_dir_exists path
|
11
|
-
if File.exist?(path)
|
12
|
-
unless File.directory?(path)
|
13
|
-
raise "#{path} is not a directory"
|
14
|
-
end
|
15
|
-
else
|
16
|
-
FileUtils.mkdir_p path
|
17
|
-
end
|
18
|
-
end
|
19
|
-
|
20
|
-
def measure_time
|
21
|
-
start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
22
|
-
yield
|
23
|
-
elapsed = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start).to_i
|
24
|
-
puts "Elapsed time: #{elapsed / 3600}h #{(elapsed % 3600) / 60}m #{elapsed % 60}s" unless QUIET
|
25
|
-
end
|
26
|
-
|
27
|
-
def patch_path_string path
|
28
|
-
Gem.win_platform? ? path.gsub(/\\/, '/') : path
|
29
|
-
end
|
30
|
-
|
31
|
-
class DigestDatabase
|
32
|
-
def initialize path
|
33
|
-
@db = SQLite3::Database.new(path.to_s)
|
34
|
-
|
35
|
-
unless @db.execute("SELECT name FROM sqlite_master WHERE type='table' AND name = 'digests'").length == 1
|
36
|
-
@db.execute 'PRAGMA encoding = "UTF-8"'
|
37
|
-
@db.execute "CREATE TABLE digests (
|
38
|
-
id INTEGER PRIMARY KEY,
|
39
|
-
filename TEXT,
|
40
|
-
mtime TEXT,
|
41
|
-
digest TEXT,
|
42
|
-
digest_check_time TEXT)"
|
43
|
-
@db.execute "CREATE UNIQUE INDEX digests_filename ON digests(filename)"
|
44
|
-
end
|
45
|
-
|
46
|
-
@db.execute 'PRAGMA journal_mode = "WAL"'
|
47
|
-
@db.execute 'PRAGMA synchronous = "NORMAL"'
|
48
|
-
@db.execute 'PRAGMA locking_mode = "EXCLUSIVE"'
|
49
|
-
@db.execute 'PRAGMA cache_size = "5000"'
|
50
|
-
|
51
|
-
@db.results_as_hash = true
|
52
|
-
@missing_files = Hash[@db.prepare("SELECT filename, digest FROM digests").execute!]
|
53
|
-
@new_files = {}
|
54
|
-
|
55
|
-
|
56
|
-
@insert = @db.prepare("INSERT INTO digests (filename, mtime, digest, digest_check_time) VALUES (?, ?, ?, datetime('now'))")
|
57
|
-
@find_by_filename = @db.prepare("SELECT id, mtime, digest FROM digests WHERE filename = ?")
|
58
|
-
@touch_digest_check_time = @db.prepare("UPDATE digests SET digest_check_time = datetime('now') WHERE id = ?")
|
59
|
-
@update_mtime_and_digest = @db.prepare("UPDATE digests SET mtime = ?, digest = ?, digest_check_time = datetime('now') WHERE id = ?")
|
60
|
-
@update_mtime = @db.prepare("UPDATE digests SET mtime = ?, digest_check_time = datetime('now') WHERE id = ?")
|
61
|
-
@delete_by_filename = @db.prepare("DELETE FROM digests WHERE filename = ?")
|
62
|
-
end
|
63
|
-
|
64
|
-
def insert_or_update file_path, mtime, digest
|
65
|
-
result = @find_by_filename.execute file_path
|
66
|
-
|
67
|
-
if found = result.next_hash
|
68
|
-
raise "Multiple records found" if result.next
|
69
|
-
|
70
|
-
@missing_files.delete(file_path)
|
71
|
-
|
72
|
-
if found['digest'] == digest
|
73
|
-
COUNTS[:good] += 1
|
74
|
-
# puts "GOOD: #{file_path}" unless QUIET
|
75
|
-
unless TEST_ONLY
|
76
|
-
if found['mtime'] == mtime
|
77
|
-
@touch_digest_check_time.execute found['id']
|
78
|
-
else
|
79
|
-
@update_mtime.execute mtime, found['id']
|
80
|
-
end
|
81
|
-
end
|
82
|
-
else
|
83
|
-
if found['mtime'] == mtime # Digest is different and mtime is the same
|
84
|
-
COUNTS[:likely_damaged] += 1
|
85
|
-
STDERR.puts "LIKELY DAMAGED: #{file_path}"
|
86
|
-
else
|
87
|
-
COUNTS[:updated] += 1
|
88
|
-
puts "UPDATED: #{file_path}" unless QUIET
|
89
|
-
unless TEST_ONLY
|
90
|
-
@update_mtime_and_digest.execute mtime, digest, found['id']
|
91
|
-
end
|
92
|
-
end
|
93
|
-
end
|
94
|
-
else
|
95
|
-
COUNTS[:new] += 1
|
96
|
-
puts "NEW: #{file_path}" unless QUIET
|
97
|
-
unless TEST_ONLY
|
98
|
-
@new_files[file_path] = digest
|
99
|
-
@insert.execute! file_path, mtime, digest
|
100
|
-
end
|
101
|
-
end
|
102
|
-
end
|
103
|
-
|
104
|
-
def process_missing_files
|
105
|
-
@missing_files.delete_if do |filename, digest|
|
106
|
-
if @new_files.value?(digest)
|
107
|
-
COUNTS[:renamed] += 1
|
108
|
-
unless TEST_ONLY
|
109
|
-
@delete_by_filename.execute filename
|
110
|
-
end
|
111
|
-
true
|
112
|
-
end
|
113
|
-
end
|
114
|
-
|
115
|
-
if (COUNTS[:missing] = @missing_files.length) > 0
|
116
|
-
puts "MISSING FILES:"
|
117
|
-
@missing_files.sort.to_h.each do |filename, digest|
|
118
|
-
puts filename
|
119
|
-
end
|
120
|
-
unless TEST_ONLY
|
121
|
-
puts "Remove missing files from the database (y/n)?"
|
122
|
-
if STDIN.gets.strip == "y"
|
123
|
-
@missing_files.each do |filename, digest|
|
124
|
-
@delete_by_filename.execute filename
|
125
|
-
end
|
126
|
-
end
|
127
|
-
end
|
128
|
-
end
|
129
|
-
end
|
130
|
-
end
|
131
|
-
|
132
|
-
class Checker
|
133
|
-
def initialize files_path, digest_database_path
|
134
|
-
@files_path = files_path
|
135
|
-
ensure_dir_exists @files_path
|
136
|
-
|
137
|
-
if digest_database_path
|
138
|
-
@digest_database_path = digest_database_path
|
139
|
-
ensure_dir_exists @digest_database_path.dirname
|
140
|
-
else
|
141
|
-
@digest_database_path = @files_path + '.file-digests.sqlite'
|
142
|
-
@skip_file_digests_sqlite = true
|
143
|
-
end
|
144
|
-
|
145
|
-
@digest_database = DigestDatabase.new @digest_database_path
|
146
|
-
end
|
147
|
-
|
148
|
-
def check
|
149
|
-
walk_files do |filename|
|
150
|
-
begin
|
151
|
-
process_file filename
|
152
|
-
rescue => exception
|
153
|
-
COUNTS[:exceptions] += 1
|
154
|
-
STDERR.puts "EXCEPTION: #{filename}: #{exception.message}"
|
155
|
-
end
|
156
|
-
end
|
157
|
-
|
158
|
-
@digest_database.process_missing_files
|
159
|
-
end
|
160
|
-
|
161
|
-
def walk_files
|
162
|
-
Dir.glob(@files_path + '**' + '*', File::FNM_DOTMATCH) do |filename|
|
163
|
-
next unless File.file? filename
|
164
|
-
next if @skip_file_digests_sqlite && filename == '.file-digests.sqlite'
|
165
|
-
next if @skip_file_digests_sqlite && filename == '.file-digests.sqlite-wal'
|
166
|
-
next if @skip_file_digests_sqlite && filename == '.file-digests.sqlite-shm'
|
167
|
-
yield filename
|
168
|
-
end
|
169
|
-
end
|
170
|
-
|
171
|
-
def process_file filename
|
172
|
-
@digest_database.insert_or_update(
|
173
|
-
filename.delete_prefix(@files_path.to_s + '/'),
|
174
|
-
File.mtime(filename).utc.strftime('%Y-%m-%d %H:%M:%S'),
|
175
|
-
get_file_digest(filename)
|
176
|
-
)
|
177
|
-
end
|
178
|
-
|
179
|
-
def get_file_digest filename
|
180
|
-
File.open(filename, 'rb') do |io|
|
181
|
-
digest = Digest::SHA512.new
|
182
|
-
buffer = ""
|
183
|
-
while io.read(40960, buffer)
|
184
|
-
digest.update(buffer)
|
185
|
-
end
|
186
|
-
return digest.hexdigest
|
187
|
-
end
|
188
|
-
end
|
189
|
-
|
190
|
-
end
|
3
|
+
require 'file-digests'
|
191
4
|
|
192
5
|
QUIET = (ENV["QUIET"] == "true")
|
193
6
|
TEST_ONLY = (ENV["TEST_ONLY"] == "true")
|
194
7
|
|
195
|
-
|
196
|
-
|
197
|
-
begin
|
198
|
-
if ARGV[0]
|
199
|
-
files_path = Pathname.new patch_path_string(ARGV[0])
|
200
|
-
else
|
201
|
-
files_path = Pathname.new patch_path_string(".")
|
202
|
-
end
|
203
|
-
|
204
|
-
digest_database_path = Pathname.new patch_path_string(ARGV[1]) if ARGV[1]
|
205
|
-
|
206
|
-
measure_time do
|
207
|
-
checker = Checker.new files_path, digest_database_path
|
208
|
-
checker.check
|
209
|
-
end
|
210
|
-
|
211
|
-
if COUNTS[:likely_damaged] > 0 || COUNTS[:exceptions] > 0
|
212
|
-
STDERR.puts "ERRORS WERE OCCURRED"
|
213
|
-
end
|
214
|
-
|
215
|
-
puts COUNTS.inspect
|
216
|
-
|
217
|
-
rescue => exception
|
218
|
-
STDERR.puts "EXCEPTION: #{exception.message}"
|
219
|
-
raise exception
|
220
|
-
end
|
8
|
+
FileDigests.perform_check
|
data/bin/file-digests-test
CHANGED
@@ -1,220 +1,8 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
2
|
|
3
|
-
require '
|
4
|
-
require 'set'
|
5
|
-
require 'digest'
|
6
|
-
require 'fileutils'
|
7
|
-
require 'pathname'
|
8
|
-
require 'sqlite3'
|
9
|
-
|
10
|
-
def ensure_dir_exists path
|
11
|
-
if File.exist?(path)
|
12
|
-
unless File.directory?(path)
|
13
|
-
raise "#{path} is not a directory"
|
14
|
-
end
|
15
|
-
else
|
16
|
-
FileUtils.mkdir_p path
|
17
|
-
end
|
18
|
-
end
|
19
|
-
|
20
|
-
def measure_time
|
21
|
-
start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
22
|
-
yield
|
23
|
-
elapsed = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start).to_i
|
24
|
-
puts "Elapsed time: #{elapsed / 3600}h #{(elapsed % 3600) / 60}m #{elapsed % 60}s" unless QUIET
|
25
|
-
end
|
26
|
-
|
27
|
-
def patch_path_string path
|
28
|
-
Gem.win_platform? ? path.gsub(/\\/, '/') : path
|
29
|
-
end
|
30
|
-
|
31
|
-
class DigestDatabase
|
32
|
-
def initialize path
|
33
|
-
@db = SQLite3::Database.new(path.to_s)
|
34
|
-
|
35
|
-
unless @db.execute("SELECT name FROM sqlite_master WHERE type='table' AND name = 'digests'").length == 1
|
36
|
-
@db.execute 'PRAGMA encoding = "UTF-8"'
|
37
|
-
@db.execute "CREATE TABLE digests (
|
38
|
-
id INTEGER PRIMARY KEY,
|
39
|
-
filename TEXT,
|
40
|
-
mtime TEXT,
|
41
|
-
digest TEXT,
|
42
|
-
digest_check_time TEXT)"
|
43
|
-
@db.execute "CREATE UNIQUE INDEX digests_filename ON digests(filename)"
|
44
|
-
end
|
45
|
-
|
46
|
-
@db.execute 'PRAGMA journal_mode = "WAL"'
|
47
|
-
@db.execute 'PRAGMA synchronous = "NORMAL"'
|
48
|
-
@db.execute 'PRAGMA locking_mode = "EXCLUSIVE"'
|
49
|
-
@db.execute 'PRAGMA cache_size = "5000"'
|
50
|
-
|
51
|
-
@db.results_as_hash = true
|
52
|
-
@missing_files = Hash[@db.prepare("SELECT filename, digest FROM digests").execute!]
|
53
|
-
@new_files = {}
|
54
|
-
|
55
|
-
|
56
|
-
@insert = @db.prepare("INSERT INTO digests (filename, mtime, digest, digest_check_time) VALUES (?, ?, ?, datetime('now'))")
|
57
|
-
@find_by_filename = @db.prepare("SELECT id, mtime, digest FROM digests WHERE filename = ?")
|
58
|
-
@touch_digest_check_time = @db.prepare("UPDATE digests SET digest_check_time = datetime('now') WHERE id = ?")
|
59
|
-
@update_mtime_and_digest = @db.prepare("UPDATE digests SET mtime = ?, digest = ?, digest_check_time = datetime('now') WHERE id = ?")
|
60
|
-
@update_mtime = @db.prepare("UPDATE digests SET mtime = ?, digest_check_time = datetime('now') WHERE id = ?")
|
61
|
-
@delete_by_filename = @db.prepare("DELETE FROM digests WHERE filename = ?")
|
62
|
-
end
|
63
|
-
|
64
|
-
def insert_or_update file_path, mtime, digest
|
65
|
-
result = @find_by_filename.execute file_path
|
66
|
-
|
67
|
-
if found = result.next_hash
|
68
|
-
raise "Multiple records found" if result.next
|
69
|
-
|
70
|
-
@missing_files.delete(file_path)
|
71
|
-
|
72
|
-
if found['digest'] == digest
|
73
|
-
COUNTS[:good] += 1
|
74
|
-
# puts "GOOD: #{file_path}" unless QUIET
|
75
|
-
unless TEST_ONLY
|
76
|
-
if found['mtime'] == mtime
|
77
|
-
@touch_digest_check_time.execute found['id']
|
78
|
-
else
|
79
|
-
@update_mtime.execute mtime, found['id']
|
80
|
-
end
|
81
|
-
end
|
82
|
-
else
|
83
|
-
if found['mtime'] == mtime # Digest is different and mtime is the same
|
84
|
-
COUNTS[:likely_damaged] += 1
|
85
|
-
STDERR.puts "LIKELY DAMAGED: #{file_path}"
|
86
|
-
else
|
87
|
-
COUNTS[:updated] += 1
|
88
|
-
puts "UPDATED: #{file_path}" unless QUIET
|
89
|
-
unless TEST_ONLY
|
90
|
-
@update_mtime_and_digest.execute mtime, digest, found['id']
|
91
|
-
end
|
92
|
-
end
|
93
|
-
end
|
94
|
-
else
|
95
|
-
COUNTS[:new] += 1
|
96
|
-
puts "NEW: #{file_path}" unless QUIET
|
97
|
-
unless TEST_ONLY
|
98
|
-
@new_files[file_path] = digest
|
99
|
-
@insert.execute! file_path, mtime, digest
|
100
|
-
end
|
101
|
-
end
|
102
|
-
end
|
103
|
-
|
104
|
-
def process_missing_files
|
105
|
-
@missing_files.delete_if do |filename, digest|
|
106
|
-
if @new_files.value?(digest)
|
107
|
-
COUNTS[:renamed] += 1
|
108
|
-
unless TEST_ONLY
|
109
|
-
@delete_by_filename.execute filename
|
110
|
-
end
|
111
|
-
true
|
112
|
-
end
|
113
|
-
end
|
114
|
-
|
115
|
-
if (COUNTS[:missing] = @missing_files.length) > 0
|
116
|
-
puts "MISSING FILES:"
|
117
|
-
@missing_files.sort.to_h.each do |filename, digest|
|
118
|
-
puts filename
|
119
|
-
end
|
120
|
-
unless TEST_ONLY
|
121
|
-
puts "Remove missing files from the database (y/n)?"
|
122
|
-
if STDIN.gets.strip == "y"
|
123
|
-
@missing_files.each do |filename, digest|
|
124
|
-
@delete_by_filename.execute filename
|
125
|
-
end
|
126
|
-
end
|
127
|
-
end
|
128
|
-
end
|
129
|
-
end
|
130
|
-
end
|
131
|
-
|
132
|
-
class Checker
|
133
|
-
def initialize files_path, digest_database_path
|
134
|
-
@files_path = files_path
|
135
|
-
ensure_dir_exists @files_path
|
136
|
-
|
137
|
-
if digest_database_path
|
138
|
-
@digest_database_path = digest_database_path
|
139
|
-
ensure_dir_exists @digest_database_path.dirname
|
140
|
-
else
|
141
|
-
@digest_database_path = @files_path + '.file-digests.sqlite'
|
142
|
-
@skip_file_digests_sqlite = true
|
143
|
-
end
|
144
|
-
|
145
|
-
@digest_database = DigestDatabase.new @digest_database_path
|
146
|
-
end
|
147
|
-
|
148
|
-
def check
|
149
|
-
walk_files do |filename|
|
150
|
-
begin
|
151
|
-
process_file filename
|
152
|
-
rescue => exception
|
153
|
-
COUNTS[:exceptions] += 1
|
154
|
-
STDERR.puts "EXCEPTION: #{filename}: #{exception.message}"
|
155
|
-
end
|
156
|
-
end
|
157
|
-
|
158
|
-
@digest_database.process_missing_files
|
159
|
-
end
|
160
|
-
|
161
|
-
def walk_files
|
162
|
-
Dir.glob(@files_path + '**' + '*', File::FNM_DOTMATCH) do |filename|
|
163
|
-
next unless File.file? filename
|
164
|
-
next if @skip_file_digests_sqlite && filename == '.file-digests.sqlite'
|
165
|
-
next if @skip_file_digests_sqlite && filename == '.file-digests.sqlite-wal'
|
166
|
-
next if @skip_file_digests_sqlite && filename == '.file-digests.sqlite-shm'
|
167
|
-
yield filename
|
168
|
-
end
|
169
|
-
end
|
170
|
-
|
171
|
-
def process_file filename
|
172
|
-
@digest_database.insert_or_update(
|
173
|
-
filename.delete_prefix(@files_path.to_s + '/'),
|
174
|
-
File.mtime(filename).utc.strftime('%Y-%m-%d %H:%M:%S'),
|
175
|
-
get_file_digest(filename)
|
176
|
-
)
|
177
|
-
end
|
178
|
-
|
179
|
-
def get_file_digest filename
|
180
|
-
File.open(filename, 'rb') do |io|
|
181
|
-
digest = Digest::SHA512.new
|
182
|
-
buffer = ""
|
183
|
-
while io.read(40960, buffer)
|
184
|
-
digest.update(buffer)
|
185
|
-
end
|
186
|
-
return digest.hexdigest
|
187
|
-
end
|
188
|
-
end
|
189
|
-
|
190
|
-
end
|
3
|
+
require 'file-digests'
|
191
4
|
|
192
5
|
QUIET = (ENV["QUIET"] == "true")
|
193
6
|
TEST_ONLY = true
|
194
7
|
|
195
|
-
|
196
|
-
|
197
|
-
begin
|
198
|
-
if ARGV[0]
|
199
|
-
files_path = Pathname.new patch_path_string(ARGV[0])
|
200
|
-
else
|
201
|
-
files_path = Pathname.new patch_path_string(".")
|
202
|
-
end
|
203
|
-
|
204
|
-
digest_database_path = Pathname.new patch_path_string(ARGV[1]) if ARGV[1]
|
205
|
-
|
206
|
-
measure_time do
|
207
|
-
checker = Checker.new files_path, digest_database_path
|
208
|
-
checker.check
|
209
|
-
end
|
210
|
-
|
211
|
-
if COUNTS[:likely_damaged] > 0 || COUNTS[:exceptions] > 0
|
212
|
-
STDERR.puts "ERRORS WERE OCCURRED"
|
213
|
-
end
|
214
|
-
|
215
|
-
puts COUNTS.inspect
|
216
|
-
|
217
|
-
rescue => exception
|
218
|
-
STDERR.puts "EXCEPTION: #{exception.message}"
|
219
|
-
raise exception
|
220
|
-
end
|
8
|
+
FileDigests.perform_check
|
data/lib/file-digests.rb
ADDED
@@ -0,0 +1,237 @@
|
|
1
|
+
|
2
|
+
require 'date'
|
3
|
+
require 'set'
|
4
|
+
require 'digest'
|
5
|
+
require 'fileutils'
|
6
|
+
require 'pathname'
|
7
|
+
require 'sqlite3'
|
8
|
+
|
9
|
+
module FileDigests
|
10
|
+
|
11
|
+
def self.ensure_dir_exists path
|
12
|
+
if File.exist?(path)
|
13
|
+
unless File.directory?(path)
|
14
|
+
raise "#{path} is not a directory"
|
15
|
+
end
|
16
|
+
else
|
17
|
+
FileUtils.mkdir_p path
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
def self.measure_time
|
22
|
+
start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
23
|
+
yield
|
24
|
+
elapsed = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start).to_i
|
25
|
+
puts "Elapsed time: #{elapsed / 3600}h #{(elapsed % 3600) / 60}m #{elapsed % 60}s" unless QUIET
|
26
|
+
end
|
27
|
+
|
28
|
+
def self.patch_path_string path
|
29
|
+
Gem.win_platform? ? path.gsub(/\\/, '/') : path
|
30
|
+
end
|
31
|
+
|
32
|
+
def self.perform_check
|
33
|
+
files_path = Pathname.new patch_path_string(ARGV[0] || ".")
|
34
|
+
digest_database_path = Pathname.new patch_path_string(ARGV[1]) if ARGV[1]
|
35
|
+
checker = Checker.new files_path, digest_database_path
|
36
|
+
checker.check
|
37
|
+
end
|
38
|
+
|
39
|
+
class DigestDatabase
|
40
|
+
def initialize path
|
41
|
+
@db = SQLite3::Database.new path.to_s
|
42
|
+
@db.results_as_hash = true
|
43
|
+
|
44
|
+
execute 'PRAGMA journal_mode = "WAL"'
|
45
|
+
execute 'PRAGMA synchronous = "NORMAL"'
|
46
|
+
execute 'PRAGMA locking_mode = "EXCLUSIVE"'
|
47
|
+
execute 'PRAGMA cache_size = "5000"'
|
48
|
+
|
49
|
+
unless execute("SELECT name FROM sqlite_master WHERE type='table' AND name = 'digests'").length == 1
|
50
|
+
execute 'PRAGMA encoding = "UTF-8"'
|
51
|
+
execute "CREATE TABLE digests (
|
52
|
+
id INTEGER PRIMARY KEY,
|
53
|
+
filename TEXT,
|
54
|
+
mtime TEXT,
|
55
|
+
digest TEXT,
|
56
|
+
digest_check_time TEXT)"
|
57
|
+
execute "CREATE UNIQUE INDEX digests_filename ON digests(filename)"
|
58
|
+
end
|
59
|
+
|
60
|
+
@missing_files = Hash[@db.prepare("SELECT filename, digest FROM digests").execute!]
|
61
|
+
@new_files = {}
|
62
|
+
|
63
|
+
prepare_method :insert, "INSERT INTO digests (filename, mtime, digest, digest_check_time) VALUES (?, ?, ?, datetime('now'))"
|
64
|
+
prepare_method :find_by_filename, "SELECT id, mtime, digest FROM digests WHERE filename = ?"
|
65
|
+
prepare_method :touch_digest_check_time, "UPDATE digests SET digest_check_time = datetime('now') WHERE id = ?"
|
66
|
+
prepare_method :update_mtime_and_digest, "UPDATE digests SET mtime = ?, digest = ?, digest_check_time = datetime('now') WHERE id = ?"
|
67
|
+
prepare_method :update_mtime, "UPDATE digests SET mtime = ?, digest_check_time = datetime('now') WHERE id = ?"
|
68
|
+
prepare_method :delete_by_filename, "DELETE FROM digests WHERE filename = ?"
|
69
|
+
end
|
70
|
+
|
71
|
+
def insert_or_update file_path, mtime, digest, counters
|
72
|
+
result = find_by_filename file_path
|
73
|
+
|
74
|
+
if found = result.next_hash
|
75
|
+
raise "Multiple records found" if result.next
|
76
|
+
|
77
|
+
@missing_files.delete(file_path)
|
78
|
+
|
79
|
+
if found['digest'] == digest
|
80
|
+
counters[:good] += 1
|
81
|
+
# puts "GOOD: #{file_path}" unless QUIET
|
82
|
+
unless TEST_ONLY
|
83
|
+
if found['mtime'] == mtime
|
84
|
+
touch_digest_check_time found['id']
|
85
|
+
else
|
86
|
+
update_mtime mtime, found['id']
|
87
|
+
end
|
88
|
+
end
|
89
|
+
else
|
90
|
+
if found['mtime'] == mtime # Digest is different and mtime is the same
|
91
|
+
counters[:likely_damaged] += 1
|
92
|
+
STDERR.puts "LIKELY DAMAGED: #{file_path}"
|
93
|
+
else
|
94
|
+
counters[:updated] += 1
|
95
|
+
puts "UPDATED: #{file_path}" unless QUIET
|
96
|
+
unless TEST_ONLY
|
97
|
+
update_mtime_and_digest mtime, digest, found['id']
|
98
|
+
end
|
99
|
+
end
|
100
|
+
end
|
101
|
+
else
|
102
|
+
counters[:new] += 1
|
103
|
+
puts "NEW: #{file_path}" unless QUIET
|
104
|
+
unless TEST_ONLY
|
105
|
+
@new_files[file_path] = digest
|
106
|
+
insert file_path, mtime, digest
|
107
|
+
end
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
def process_missing_files counters
|
112
|
+
@missing_files.delete_if do |filename, digest|
|
113
|
+
if @new_files.value?(digest)
|
114
|
+
counters[:renamed] += 1
|
115
|
+
unless TEST_ONLY
|
116
|
+
delete_by_filename filename
|
117
|
+
end
|
118
|
+
true
|
119
|
+
end
|
120
|
+
end
|
121
|
+
|
122
|
+
if (counters[:missing] = @missing_files.length) > 0
|
123
|
+
puts "\nMISSING FILES:"
|
124
|
+
@missing_files.sort.to_h.each do |filename, digest|
|
125
|
+
puts filename
|
126
|
+
end
|
127
|
+
unless TEST_ONLY
|
128
|
+
puts "Remove missing files from the database (y/n)?"
|
129
|
+
if STDIN.gets.strip.downcase == "y"
|
130
|
+
@db.transaction do
|
131
|
+
@missing_files.each do |filename, digest|
|
132
|
+
delete_by_filename filename
|
133
|
+
end
|
134
|
+
end
|
135
|
+
end
|
136
|
+
end
|
137
|
+
end
|
138
|
+
end
|
139
|
+
|
140
|
+
private
|
141
|
+
|
142
|
+
def execute *args, &block
|
143
|
+
@db.execute *args, &block
|
144
|
+
end
|
145
|
+
|
146
|
+
def prepare_method name, query
|
147
|
+
variable = "@#{name}"
|
148
|
+
instance_variable_set(variable, @db.prepare(query))
|
149
|
+
define_singleton_method name do |*args, &block|
|
150
|
+
instance_variable_get(variable).execute(*args, &block)
|
151
|
+
end
|
152
|
+
end
|
153
|
+
end
|
154
|
+
|
155
|
+
class Checker
|
156
|
+
def initialize files_path, digest_database_path
|
157
|
+
@counters = {good: 0, updated: 0, new: 0, missing: 0, renamed: 0, likely_damaged: 0, exceptions: 0}
|
158
|
+
@files_path = files_path
|
159
|
+
@prefix_to_remove = @files_path.to_s + '/'
|
160
|
+
|
161
|
+
unless digest_database_path
|
162
|
+
digest_database_path = @files_path + '.file-digests.sqlite'
|
163
|
+
@skip_file_digests_sqlite = true
|
164
|
+
end
|
165
|
+
|
166
|
+
FileDigests::ensure_dir_exists @files_path
|
167
|
+
FileDigests::ensure_dir_exists digest_database_path.dirname
|
168
|
+
|
169
|
+
@digest_database = DigestDatabase.new digest_database_path
|
170
|
+
end
|
171
|
+
|
172
|
+
def check
|
173
|
+
FileDigests::measure_time do
|
174
|
+
walk_files do |filename|
|
175
|
+
process_file filename
|
176
|
+
end
|
177
|
+
end
|
178
|
+
|
179
|
+
@digest_database.process_missing_files @counters
|
180
|
+
|
181
|
+
if @counters[:likely_damaged] > 0 || @counters[:exceptions] > 0
|
182
|
+
STDERR.puts "ERRORS WERE OCCURRED"
|
183
|
+
end
|
184
|
+
|
185
|
+
puts @counters.inspect
|
186
|
+
end
|
187
|
+
|
188
|
+
def walk_files
|
189
|
+
Dir.glob(@files_path + '**' + '*', File::FNM_DOTMATCH) do |filename|
|
190
|
+
yield filename
|
191
|
+
end
|
192
|
+
end
|
193
|
+
|
194
|
+
def process_file filename
|
195
|
+
return if File.symlink? filename
|
196
|
+
|
197
|
+
stat = File.stat filename
|
198
|
+
|
199
|
+
return if stat.blockdev?
|
200
|
+
return if stat.chardev?
|
201
|
+
return if stat.directory?
|
202
|
+
return if stat.pipe?
|
203
|
+
unless stat.readable?
|
204
|
+
raise "File is not readable"
|
205
|
+
end
|
206
|
+
return if stat.socket?
|
207
|
+
|
208
|
+
if @skip_file_digests_sqlite
|
209
|
+
return if filename == '.file-digests.sqlite'
|
210
|
+
return if filename == '.file-digests.sqlite-wal'
|
211
|
+
return if filename == '.file-digests.sqlite-shm'
|
212
|
+
end
|
213
|
+
|
214
|
+
@digest_database.insert_or_update(
|
215
|
+
filename.delete_prefix(@prefix_to_remove).unicode_normalize(:nfkc),
|
216
|
+
stat.mtime.utc.strftime('%Y-%m-%d %H:%M:%S'),
|
217
|
+
get_file_digest(filename),
|
218
|
+
@counters
|
219
|
+
)
|
220
|
+
rescue => exception
|
221
|
+
@counters[:exceptions] += 1
|
222
|
+
STDERR.puts "EXCEPTION: #{filename}: #{exception.message}"
|
223
|
+
end
|
224
|
+
|
225
|
+
def get_file_digest filename
|
226
|
+
File.open(filename, 'rb') do |io|
|
227
|
+
digest = Digest::SHA512.new
|
228
|
+
buffer = ""
|
229
|
+
while io.read(40960, buffer)
|
230
|
+
digest.update(buffer)
|
231
|
+
end
|
232
|
+
return digest.hexdigest
|
233
|
+
end
|
234
|
+
end
|
235
|
+
|
236
|
+
end
|
237
|
+
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: file-digests
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.9
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Stanislav Senotrusov
|
@@ -34,6 +34,7 @@ extra_rdoc_files: []
|
|
34
34
|
files:
|
35
35
|
- bin/file-digests
|
36
36
|
- bin/file-digests-test
|
37
|
+
- lib/file-digests.rb
|
37
38
|
homepage: https://github.com/senotrusov/file-digests
|
38
39
|
licenses:
|
39
40
|
- Apache-2.0
|