metaclean 4.0.1 → 4.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +96 -25
- data/SECURITY.md +59 -0
- data/bin/metaclean +0 -1
- data/lib/metaclean/cli.rb +46 -43
- data/lib/metaclean/committer.rb +130 -0
- data/lib/metaclean/discovery.rb +127 -0
- data/lib/metaclean/display.rb +73 -90
- data/lib/metaclean/exiftool.rb +36 -67
- data/lib/metaclean/ffmpeg.rb +96 -50
- data/lib/metaclean/file_ops.rb +214 -0
- data/lib/metaclean/mat2.rb +28 -77
- data/lib/metaclean/qpdf.rb +46 -46
- data/lib/metaclean/runner.rb +138 -418
- data/lib/metaclean/strategy.rb +75 -100
- data/lib/metaclean/version.rb +1 -5
- data/lib/metaclean.rb +87 -31
- metadata +7 -1
data/lib/metaclean/runner.rb
CHANGED
|
@@ -1,47 +1,45 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
# The orchestrator. Given a list of paths and parsed CLI options, this class:
|
|
4
|
-
#
|
|
5
|
-
# 1. Expands paths into a flat list of files (handling directories and
|
|
6
|
-
# recursion; symlinks are skipped).
|
|
7
|
-
# 2. Asks the user for confirmation (unless --force).
|
|
8
|
-
# 3. For each file, runs the strategy pipeline (mat2 / exiftool / qpdf)
|
|
9
|
-
# using the "atomic write" pattern so a crash never leaves a
|
|
10
|
-
# half-cleaned file.
|
|
11
|
-
# 4. Prints a before/after diff and a final summary.
|
|
12
|
-
|
|
13
|
-
require 'fileutils'
|
|
14
|
-
require 'securerandom'
|
|
15
|
-
|
|
16
3
|
module Metaclean
|
|
17
4
|
class Runner
|
|
18
5
|
def initialize(options)
|
|
19
6
|
@options = options
|
|
7
|
+
Display.configure(
|
|
8
|
+
quiet: options.fetch(:quiet, false),
|
|
9
|
+
redact_values: options.fetch(:redact_values, !$stdout.tty?)
|
|
10
|
+
)
|
|
11
|
+
@scan_errors = 0
|
|
12
|
+
@path_guards = {}
|
|
13
|
+
@discovery = Discovery.new(recursive: options.fetch(:recursive, false))
|
|
14
|
+
@committer = Committer.new(
|
|
15
|
+
in_place: options.fetch(:in_place, false),
|
|
16
|
+
dry_run: options.fetch(:dry_run, false)
|
|
17
|
+
)
|
|
20
18
|
end
|
|
21
19
|
|
|
22
|
-
# Public entry points: one for `--inspect`, one for the cleaning flow.
|
|
23
|
-
|
|
24
20
|
def inspect_paths(paths)
|
|
25
21
|
files = expand_files(paths)
|
|
26
22
|
if files.empty?
|
|
27
23
|
Display.warning('No files to inspect.')
|
|
28
24
|
exit 1
|
|
29
25
|
end
|
|
26
|
+
failed = 0
|
|
30
27
|
files.each do |file|
|
|
31
28
|
Display.header "📄 #{file}"
|
|
29
|
+
FileOps.ensure_path_guard!(file, guard_for(file))
|
|
32
30
|
meta = Exiftool.read(file)
|
|
33
31
|
Display.section "Metadata (#{Display.count_embedded(meta)} embedded tags)"
|
|
34
32
|
Display.metadata_table(meta)
|
|
35
33
|
rescue Error, SystemCallError => e
|
|
36
|
-
# One unreadable/odd file shouldn't abort inspecting the rest — mirrors
|
|
37
|
-
# the per-file rescue in the clean batch.
|
|
38
34
|
warn Display.error("#{file}: #{e.message}")
|
|
35
|
+
failed += 1
|
|
39
36
|
end
|
|
37
|
+
|
|
38
|
+
exit 1 if failed.positive? || @scan_errors.positive?
|
|
40
39
|
end
|
|
41
40
|
|
|
42
41
|
def clean_paths(paths)
|
|
43
42
|
files = expand_files(paths)
|
|
44
|
-
# See inspect_paths: nothing to act on is a non-zero condition, not success.
|
|
45
43
|
if files.empty?
|
|
46
44
|
Display.warning('No files to process.')
|
|
47
45
|
exit 1
|
|
@@ -49,45 +47,45 @@ module Metaclean
|
|
|
49
47
|
|
|
50
48
|
announce_tools
|
|
51
49
|
|
|
52
|
-
|
|
53
|
-
|
|
50
|
+
if @options[:in_place] && !@options[:dry_run]
|
|
51
|
+
Display.warning 'Each backup is the ORIGINAL with all metadata intact; move or delete it before sharing.'
|
|
52
|
+
end
|
|
53
|
+
|
|
54
54
|
unless @options[:force] || @options[:dry_run]
|
|
55
55
|
action = @options[:in_place] ? 'OVERWRITE' : 'create cleaned copies of'
|
|
56
56
|
puts Display.c("About to #{action} #{files.size} file(s).", :yellow)
|
|
57
|
-
if @options[:in_place]
|
|
58
|
-
puts Display.c('Backups will be saved alongside as <file>.bak.', :gray)
|
|
59
|
-
end
|
|
60
57
|
print Display.c('Proceed? [y/N] ', :bold)
|
|
61
|
-
ans = $stdin.gets&.strip&.downcase
|
|
62
|
-
|
|
58
|
+
ans = $stdin.gets&.strip&.downcase
|
|
59
|
+
unless %w[y yes].include?(ans)
|
|
60
|
+
Display.warning('Aborted.')
|
|
61
|
+
exit 1
|
|
62
|
+
end
|
|
63
63
|
end
|
|
64
64
|
|
|
65
|
-
summary = { cleaned: 0, unverified: 0, failed: 0, removed_total: 0, residual_files: 0 }
|
|
65
|
+
summary = { cleaned: 0, unverified: 0, unsupported: 0, failed: 0, removed_total: 0, residual_files: 0 }
|
|
66
66
|
|
|
67
|
-
# index/total let clean_one render "[3/47]" in batch mode.
|
|
68
67
|
files.each_with_index do |file, idx|
|
|
69
|
-
|
|
70
|
-
summary[result[:status]] += 1
|
|
71
|
-
summary[:removed_total] += result[:removed].to_i
|
|
72
|
-
summary[:residual_files] += 1 if result[:residual].to_i.positive?
|
|
68
|
+
tally(summary, clean_one(file, index: idx + 1, total: files.size))
|
|
73
69
|
rescue Error, SystemCallError => e
|
|
74
|
-
# One bad file shouldn't abort the whole batch. SystemCallError
|
|
75
|
-
# (Errno::*: disk full, permission denied, read-only fs) is a SIBLING
|
|
76
|
-
# of our Error, not a subclass, so it must be named explicitly or it
|
|
77
|
-
# would escape this rescue and crash the run with a raw backtrace.
|
|
78
70
|
warn Display.error("#{file}: #{e.message}")
|
|
79
71
|
summary[:failed] += 1
|
|
80
72
|
end
|
|
81
73
|
|
|
82
74
|
print_summary(summary)
|
|
83
75
|
|
|
84
|
-
|
|
85
|
-
|
|
76
|
+
if summary[:failed].positive? || summary[:unverified].positive? ||
|
|
77
|
+
summary[:unsupported].positive? || @scan_errors.positive?
|
|
78
|
+
exit 1
|
|
79
|
+
end
|
|
86
80
|
end
|
|
87
81
|
|
|
88
82
|
private
|
|
89
83
|
|
|
90
|
-
|
|
84
|
+
def tally(summary, result)
|
|
85
|
+
summary[result[:status]] += 1
|
|
86
|
+
summary[:removed_total] += result[:removed].to_i if result[:status] == :cleaned
|
|
87
|
+
summary[:residual_files] += 1 if result[:residual].to_i.positive?
|
|
88
|
+
end
|
|
91
89
|
|
|
92
90
|
def announce_tools
|
|
93
91
|
have = []
|
|
@@ -99,118 +97,105 @@ module Metaclean
|
|
|
99
97
|
Display.info '(dry-run — no files will be modified)' if @options[:dry_run]
|
|
100
98
|
end
|
|
101
99
|
|
|
102
|
-
# Cleaning a single file.
|
|
103
100
|
def clean_one(file, index:, total:)
|
|
104
101
|
prefix = total > 1 ? "[#{index}/#{total}] " : ''
|
|
105
102
|
Display.header "#{prefix}📄 #{file}"
|
|
103
|
+
path_guard = guard_for(file)
|
|
104
|
+
FileOps.ensure_path_guard!(file, path_guard)
|
|
106
105
|
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
before = read_metadata(file)
|
|
110
|
-
Display.section "Before (#{Display.count_embedded(before)} embedded tags)"
|
|
111
|
-
Display.metadata_table(before, only_embedded: true)
|
|
112
|
-
|
|
113
|
-
# Ask the strategy module which tools to run for this file type.
|
|
114
|
-
tools = Strategy.tools_for(file)
|
|
115
|
-
# Warn when the stricter tool for a document format won't run: ExifTool
|
|
116
|
-
# alone leaves (and can't fully verify) document-internal metadata.
|
|
117
|
-
if Strategy.mat2_essential?(file) && !tools.include?(:mat2)
|
|
118
|
-
Display.warning 'mat2 will not run for this format — document-internal metadata may remain and cannot be verified.'
|
|
119
|
-
end
|
|
120
|
-
Display.info "Pipeline: #{tools.join(' → ')}"
|
|
121
|
-
|
|
122
|
-
# Atomic write setup:
|
|
123
|
-
# `final_path` = where the cleaned file will end up.
|
|
124
|
-
# `staging` = a temp file we mutate. After all tools succeed, we
|
|
125
|
-
# rename staging → final_path. If anything goes wrong
|
|
126
|
-
# in the middle, we delete staging in the `ensure`
|
|
127
|
-
# block and the original is untouched.
|
|
106
|
+
before = inspect_before(file)
|
|
107
|
+
tools = selected_tools(file)
|
|
128
108
|
final_path = resolve_final_path(file)
|
|
129
109
|
staging = staging_path_for(final_path)
|
|
130
110
|
|
|
131
|
-
tool_results = []
|
|
132
111
|
begin
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
# partial temp if cp is interrupted (Ctrl-C) or fails mid-copy (disk full,
|
|
141
|
-
# read-only fs). cp only ever reads the original, so the source is intact
|
|
142
|
-
# regardless.
|
|
143
|
-
copy_file_exclusive(file, staging)
|
|
144
|
-
tools.each do |tool|
|
|
145
|
-
tool_results << run_tool(tool, staging)
|
|
146
|
-
end
|
|
112
|
+
FileOps.ensure_path_guard!(file, path_guard)
|
|
113
|
+
source_stat = copy_file_exclusive(file, staging)
|
|
114
|
+
tool_results = tools.map { |tool| run_tool(tool, staging) }
|
|
115
|
+
after = inspect_after(staging, before)
|
|
116
|
+
residual = report_residual(after)
|
|
117
|
+
result = finalize_result(tool_results, before, after, residual, file: file)
|
|
118
|
+
return result if discard_result?(result)
|
|
147
119
|
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
120
|
+
commit_cleaned(file, staging, final_path, source_stat, path_guard)
|
|
121
|
+
result
|
|
122
|
+
ensure
|
|
123
|
+
cleanup_staging(staging)
|
|
124
|
+
end
|
|
125
|
+
end
|
|
152
126
|
|
|
153
|
-
|
|
154
|
-
|
|
127
|
+
def inspect_before(file)
|
|
128
|
+
metadata = read_metadata(file)
|
|
129
|
+
Display.section "Before (#{Display.count_embedded(metadata)} embedded tags)"
|
|
130
|
+
Display.metadata_table(metadata, only_embedded: true)
|
|
131
|
+
metadata
|
|
132
|
+
end
|
|
155
133
|
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
134
|
+
def selected_tools(file)
|
|
135
|
+
tools = Strategy.tools_for(file)
|
|
136
|
+
if Strategy.mat2_essential?(file) && !tools.include?(:mat2)
|
|
137
|
+
Display.warning 'mat2 will not run for this format — document-internal metadata may remain and cannot be verified.'
|
|
138
|
+
end
|
|
139
|
+
Display.info "Pipeline: #{tools.join(' → ')}"
|
|
140
|
+
tools
|
|
141
|
+
end
|
|
162
142
|
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
143
|
+
def inspect_after(staging, before)
|
|
144
|
+
metadata = read_metadata(staging)
|
|
145
|
+
Display.section "After (#{Display.count_embedded(metadata)} embedded tags)"
|
|
146
|
+
Display.metadata_table(metadata, only_embedded: true)
|
|
147
|
+
Display.section 'Diff'
|
|
148
|
+
Display.diff(before, metadata)
|
|
149
|
+
metadata
|
|
150
|
+
end
|
|
169
151
|
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
152
|
+
def report_residual(metadata)
|
|
153
|
+
residual = Strategy.privacy_residual(
|
|
154
|
+
metadata,
|
|
155
|
+
allow_icc_metadata: @options.fetch(:allow_icc_metadata, false)
|
|
156
|
+
)
|
|
157
|
+
return residual if residual.empty?
|
|
158
|
+
|
|
159
|
+
Display.warning "Privacy-relevant tags still present (#{residual.size}):"
|
|
160
|
+
unless Display.quiet?
|
|
161
|
+
residual.each do |key, value|
|
|
162
|
+
puts " #{Display.c(key, :yellow)} = #{Display.truncate(Display.visible_value(value), 60)}"
|
|
180
163
|
end
|
|
164
|
+
end
|
|
165
|
+
residual
|
|
166
|
+
end
|
|
181
167
|
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
168
|
+
def discard_result?(result)
|
|
169
|
+
if @options[:dry_run]
|
|
170
|
+
Display.info '(dry-run: temporary copy removed; no output was kept)'
|
|
171
|
+
return true
|
|
172
|
+
end
|
|
173
|
+
return false if result[:status] == :cleaned
|
|
186
174
|
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
175
|
+
reason = case result[:status]
|
|
176
|
+
when :unsupported then 'No cleaning tool supports this format'
|
|
177
|
+
when :unverified then 'The complete cleaning pipeline could not be verified'
|
|
178
|
+
else 'Privacy-relevant metadata survived or all tools failed'
|
|
179
|
+
end
|
|
180
|
+
Display.warning "#{reason} — not writing output."
|
|
181
|
+
true
|
|
182
|
+
end
|
|
192
183
|
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
end
|
|
202
|
-
result
|
|
203
|
-
ensure
|
|
204
|
-
# Last-resort cleanup. If `commit!` already moved the staging file,
|
|
205
|
-
# `File.exist?(staging)` is false and this is a no-op. The path-
|
|
206
|
-
# comparison protects against deleting the final file by accident
|
|
207
|
-
# in the (impossible) case where staging == final.
|
|
208
|
-
File.delete(staging) if File.exist?(staging) && File.expand_path(staging) != File.expand_path(final_path)
|
|
184
|
+
def commit_cleaned(file, staging, final_path, source_stat, path_guard)
|
|
185
|
+
FileOps.ensure_same_file!(file, source_stat)
|
|
186
|
+
FileOps.ensure_path_guard!(file, path_guard)
|
|
187
|
+
if @options[:in_place]
|
|
188
|
+
FileOps.prepare_in_place_commit!(staging, file, source_stat)
|
|
189
|
+
warn_if_hardlinked(file)
|
|
190
|
+
else
|
|
191
|
+
FileOps.restore_metadata(staging, source_stat)
|
|
209
192
|
end
|
|
193
|
+
|
|
194
|
+
committed = commit!(staging, final_path, source_stat: source_stat)
|
|
195
|
+
Display.success "→ #{committed[:path]}"
|
|
196
|
+
Display.warning "Backup with original metadata: #{committed[:backup]}" if committed[:backup]
|
|
210
197
|
end
|
|
211
198
|
|
|
212
|
-
# Warn when an in-place target has more than one hard link: a rename only
|
|
213
|
-
# cleans the named link, leaving the others pointing at the original metadata.
|
|
214
199
|
def warn_if_hardlinked(file)
|
|
215
200
|
nlink = File.stat(file).nlink
|
|
216
201
|
return unless nlink > 1
|
|
@@ -219,15 +204,9 @@ module Metaclean
|
|
|
219
204
|
"the other #{nlink - 1} still contain the original metadata."
|
|
220
205
|
end
|
|
221
206
|
|
|
222
|
-
# Dispatches to the right wrapper module. Returns a small Hash so the
|
|
223
|
-
# caller can summarize tool-by-tool success/failure.
|
|
224
207
|
def run_tool(tool, path)
|
|
225
208
|
case tool
|
|
226
209
|
when :exiftool
|
|
227
|
-
# :unsupported means ExifTool can read but not write this format (a
|
|
228
|
-
# ZIP-based document mat2 owns) — a soft skip, NOT a pipeline failure.
|
|
229
|
-
# Pass the privacy tag names so TIFF/DNG IFD0 tags `-all=` won't drop
|
|
230
|
-
# still get deleted (losslessly).
|
|
231
210
|
if Exiftool.strip!(path, also_delete: Strategy::PRIVACY_TAGS) == :unsupported
|
|
232
211
|
Display.info ' · exiftool (read-only for this format, skipped)'
|
|
233
212
|
{ tool: :exiftool, ok: false, skipped: true, note: :unsupported }
|
|
@@ -237,10 +216,6 @@ module Metaclean
|
|
|
237
216
|
end
|
|
238
217
|
when :mat2
|
|
239
218
|
result = Mat2.strip!(path)
|
|
240
|
-
# mat2 returns either `true` (success) or a symbol indicating a
|
|
241
|
-
# soft skip. `:unsupported` means the tool didn't actually run, so
|
|
242
|
-
# it must not count as a successful pass — otherwise a file can be
|
|
243
|
-
# reported as "Cleaned" while metadata is still embedded.
|
|
244
219
|
case result
|
|
245
220
|
when :unsupported
|
|
246
221
|
Display.info ' · mat2 (unsupported file type, skipped)'
|
|
@@ -257,36 +232,21 @@ module Metaclean
|
|
|
257
232
|
Display.info ' ✓ qpdf'
|
|
258
233
|
{ tool: :qpdf, ok: true }
|
|
259
234
|
when :ffmpeg
|
|
260
|
-
# Matroska remux. A failure raises and is caught below (→ not written).
|
|
261
235
|
Ffmpeg.strip!(path)
|
|
262
236
|
Display.info ' ✓ ffmpeg'
|
|
263
237
|
{ tool: :ffmpeg, ok: true }
|
|
264
238
|
end
|
|
265
239
|
rescue Error, SystemCallError => e
|
|
266
|
-
# One tool failing shouldn't abort the pipeline — we want to keep
|
|
267
|
-
# trying with the others. The `finalize_result` step decides whether
|
|
268
|
-
# the overall file counts as cleaned or failed. `SystemCallError`
|
|
269
|
-
# (Errno::*) covers a tool wrapper's internal FileUtils.mv/File.delete
|
|
270
|
-
# raising on permission/quota/disk errors — without it those would
|
|
271
|
-
# escape and crash the batch.
|
|
272
|
-
# Collapse whitespace and bound the length: some tools (notably mat2) dump a
|
|
273
|
-
# multi-line Python traceback on failure, which would otherwise flood the
|
|
274
|
-
# diff. One readable line is enough — re-run the tool directly to debug.
|
|
275
240
|
msg = Display.truncate(e.message.gsub(/\s+/, ' ').strip, 200)
|
|
276
241
|
Display.warning " ✗ #{tool}: #{msg} — continuing"
|
|
277
242
|
{ tool: tool, ok: false, error: e.message }
|
|
278
243
|
end
|
|
279
244
|
|
|
280
|
-
# :cleaned needs ALL of: a tool genuinely ran, no privacy residual survived,
|
|
281
|
-
# no pipeline tool errored, AND — for a format where mat2 owns coverage
|
|
282
|
-
# ExifTool can't re-read (Office/PDF doc internals) — mat2 actually ran. A
|
|
283
|
-
# tool that errored, or an absent mat2 on a document format, means the
|
|
284
|
-
# pipeline didn't fully complete and the residual check is partly blind, so
|
|
285
|
-
# the result is :unverified, not a confident :cleaned. `file` is needed only
|
|
286
|
-
# for that mat2-coverage check.
|
|
287
245
|
def finalize_result(tool_results, before, after, residual, file: nil)
|
|
288
246
|
removed = removed_embedded_count(before, after)
|
|
289
|
-
status = if !
|
|
247
|
+
status = if !tools_succeeded?(tool_results)
|
|
248
|
+
tool_errored?(tool_results) || residual.any? ? :failed : :unsupported
|
|
249
|
+
elsif !residual.empty?
|
|
290
250
|
:failed
|
|
291
251
|
elsif !tool_errored?(tool_results) && !mat2_coverage_gap?(tool_results, file)
|
|
292
252
|
:cleaned
|
|
@@ -296,41 +256,20 @@ module Metaclean
|
|
|
296
256
|
{ status: status, removed: removed, residual: residual.size }
|
|
297
257
|
end
|
|
298
258
|
|
|
299
|
-
# mat2 is essential for this format (Office/PDF internals ExifTool can't
|
|
300
|
-
# strip or fully re-read) but did NOT actually run and strip — absent,
|
|
301
|
-
# unsupported soft-skip, or errored. The residual check can't confirm the
|
|
302
|
-
# clean, so don't report a confident :cleaned.
|
|
303
259
|
def mat2_coverage_gap?(tool_results, file)
|
|
304
260
|
return false unless file && Strategy.mat2_essential?(file)
|
|
305
261
|
|
|
306
262
|
tool_results.none? { |r| r[:tool] == :mat2 && r[:ok] && !r[:skipped] }
|
|
307
263
|
end
|
|
308
264
|
|
|
309
|
-
# A file is genuinely cleaned only when at least one tool actually ran
|
|
310
|
-
# (not just a mat2 :unsupported soft-skip) AND no privacy-relevant tag
|
|
311
|
-
# survived. Both the commit gate and the final status use this ONE
|
|
312
|
-
# predicate, so they can never disagree — we never write a "_clean" copy
|
|
313
|
-
# (or overwrite an original in place) and then report it :failed. Silently
|
|
314
|
-
# marking a file clean while sensitive metadata is still present is the
|
|
315
|
-
# worst possible outcome for a privacy tool.
|
|
316
|
-
def cleaned?(tool_results, residual)
|
|
317
|
-
tools_succeeded?(tool_results) && residual.empty?
|
|
318
|
-
end
|
|
319
|
-
|
|
320
|
-
# Did at least one tool genuinely run (not a mat2 :unsupported soft-skip)?
|
|
321
265
|
def tools_succeeded?(tool_results)
|
|
322
266
|
tool_results.any? { |r| r[:ok] && !r[:skipped] }
|
|
323
267
|
end
|
|
324
268
|
|
|
325
|
-
# Did a tool that was meant to run error out (not a mat2 :unsupported
|
|
326
|
-
# soft-skip)? Even with an empty residual that means the pipeline didn't
|
|
327
|
-
# fully complete, so the clean can't be reported as a confident :cleaned.
|
|
328
269
|
def tool_errored?(tool_results)
|
|
329
270
|
tool_results.any? { |r| !r[:ok] && !r[:skipped] }
|
|
330
271
|
end
|
|
331
272
|
|
|
332
|
-
# Read metadata for the before/after diff. ensure_tools! guarantees exiftool
|
|
333
|
-
# is present before any run.
|
|
334
273
|
def read_metadata(path)
|
|
335
274
|
Exiftool.read(path)
|
|
336
275
|
end
|
|
@@ -339,142 +278,14 @@ module Metaclean
|
|
|
339
278
|
before.keys.count { |key| Display.embedded_key?(key) && !after.key?(key) }
|
|
340
279
|
end
|
|
341
280
|
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
def
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
# File.rename, NOT FileUtils.mv. staging is in the same dir as final_path
|
|
351
|
-
# (staging_path_for), so this is always an atomic same-fs swap. FileUtils.mv
|
|
352
|
-
# would, on EPERM (e.g. a sticky /tmp file not owned by us) or EXDEV, fall
|
|
353
|
-
# back to a TRUNCATING copy of the original — and an interrupt mid-copy
|
|
354
|
-
# would corrupt the original while the rescue below deleted the only backup.
|
|
355
|
-
# File.rename raises BEFORE touching final_path, so the rescue's
|
|
356
|
-
# "staging still exists ⇒ original intact" assumption always holds.
|
|
357
|
-
File.rename(staging, final_path)
|
|
358
|
-
committed = true
|
|
359
|
-
return final_path
|
|
360
|
-
end
|
|
361
|
-
|
|
362
|
-
link_with_collision_safe_name(staging, final_path)
|
|
363
|
-
rescue SystemCallError, Interrupt
|
|
364
|
-
# The rename failed (disk full, read-only fs, cross-device) OR the user hit
|
|
365
|
-
# Ctrl-C in the window after the .bak was written but before the mv. Either
|
|
366
|
-
# way the original is untouched, so the .bak is a redundant copy of it —
|
|
367
|
-
# remove it instead of leaving a stray file behind, then re-raise (a
|
|
368
|
-
# SystemCallError is reported per-file as failed; an Interrupt propagates to
|
|
369
|
-
# the CLI's exit-130 handler).
|
|
370
|
-
File.delete(backup) if backup && !committed && File.exist?(staging) && File.exist?(backup)
|
|
371
|
-
raise
|
|
372
|
-
end
|
|
373
|
-
|
|
374
|
-
def link_with_collision_safe_name(staging, preferred)
|
|
375
|
-
target = preferred
|
|
376
|
-
loop do
|
|
377
|
-
File.link(staging, target)
|
|
378
|
-
File.delete(staging)
|
|
379
|
-
return target
|
|
380
|
-
rescue Errno::EEXIST
|
|
381
|
-
target = collision_safe(target)
|
|
382
|
-
rescue Errno::EACCES, Errno::EPERM, Errno::ENOTSUP, NotImplementedError
|
|
383
|
-
# The filesystem can't hard-link (Linux returns EPERM; macOS/BSD on
|
|
384
|
-
# exFAT/FAT/SMB returns ENOTSUP/EOPNOTSUPP — same Errno class). Fall back
|
|
385
|
-
# to a plain exclusive copy so removable/network drives still clean.
|
|
386
|
-
target = copy_with_collision_safe_name(staging, target)
|
|
387
|
-
File.delete(staging)
|
|
388
|
-
return target
|
|
389
|
-
end
|
|
390
|
-
end
|
|
391
|
-
|
|
392
|
-
def copy_with_collision_safe_name(src, preferred)
|
|
393
|
-
target = preferred
|
|
394
|
-
loop do
|
|
395
|
-
copy_file_exclusive(src, target, preserve: true)
|
|
396
|
-
return target
|
|
397
|
-
rescue Errno::EEXIST
|
|
398
|
-
target = collision_safe(target)
|
|
399
|
-
end
|
|
400
|
-
end
|
|
401
|
-
|
|
402
|
-
def copy_file_exclusive(src, dest, preserve: false)
|
|
403
|
-
src_stat = File.lstat(src)
|
|
404
|
-
raise Error, "#{src} is a symlink — refusing to copy it" if src_stat.symlink?
|
|
405
|
-
|
|
406
|
-
mode = src_stat.mode & 0o7777
|
|
407
|
-
created = false
|
|
408
|
-
File.open(dest, File::WRONLY | File::CREAT | File::EXCL, mode) do |out|
|
|
409
|
-
created = true
|
|
410
|
-
File.open(src, 'rb') do |input|
|
|
411
|
-
opened = input.stat
|
|
412
|
-
unless opened.dev == src_stat.dev && opened.ino == src_stat.ino
|
|
413
|
-
raise Error, "#{src} changed while opening — refusing to copy it"
|
|
414
|
-
end
|
|
415
|
-
|
|
416
|
-
IO.copy_stream(input, out)
|
|
417
|
-
end
|
|
418
|
-
end
|
|
419
|
-
return unless preserve
|
|
420
|
-
|
|
421
|
-
# Best-effort: the bytes are already fully copied, so a failed mode/timestamp
|
|
422
|
-
# restore (e.g. utime/chmod on a FAT/exFAT mount) must NOT discard an
|
|
423
|
-
# otherwise-complete file by falling into the delete-on-error rescue below.
|
|
424
|
-
begin
|
|
425
|
-
File.chmod(mode, dest)
|
|
426
|
-
File.utime(src_stat.atime, src_stat.mtime, dest)
|
|
427
|
-
rescue SystemCallError
|
|
428
|
-
nil
|
|
429
|
-
end
|
|
430
|
-
rescue StandardError, Interrupt
|
|
431
|
-
File.delete(dest) if created && dest && File.exist?(dest)
|
|
432
|
-
raise
|
|
433
|
-
end
|
|
434
|
-
|
|
435
|
-
def resolve_final_path(file)
|
|
436
|
-
return file if @options[:in_place]
|
|
437
|
-
|
|
438
|
-
# Default: write `<name>_clean.<ext>` next to the original. If it
|
|
439
|
-
# already exists, `collision_safe` appends `_1`, `_2`, …
|
|
440
|
-
collision_safe(build_clean_path(file))
|
|
441
|
-
end
|
|
442
|
-
|
|
443
|
-
def build_clean_path(file)
|
|
444
|
-
ext = File.extname(file)
|
|
445
|
-
base = File.basename(file, ext)
|
|
446
|
-
File.join(File.dirname(file), "#{base}#{Metaclean::CLEAN_SUFFIX}#{ext}")
|
|
447
|
-
end
|
|
448
|
-
|
|
449
|
-
# Staging path lives in the same directory as the destination so that
|
|
450
|
-
# `File.rename`/`FileUtils.mv` is an atomic same-filesystem operation.
|
|
451
|
-
# PID + random number prevent collisions between simultaneous runs.
|
|
452
|
-
# The original extension is preserved as the LAST segment so tools like
|
|
453
|
-
# mat2 — which dispatch on file extension — see the real type.
|
|
454
|
-
def staging_path_for(final_path)
|
|
455
|
-
dir = File.dirname(final_path)
|
|
456
|
-
ext = File.extname(final_path)
|
|
457
|
-
# SecureRandom (not rand) makes the staging name unpredictable, so a
|
|
458
|
-
# hostile process in the same directory can't pre-create it as a symlink
|
|
459
|
-
# that `FileUtils.cp` would copy the (still-sensitive) original through.
|
|
460
|
-
File.join(dir, "#{Metaclean::TMP_MARKER}#{Process.pid}.#{SecureRandom.hex(8)}#{ext}")
|
|
461
|
-
end
|
|
462
|
-
|
|
463
|
-
# If `path` is taken, try `path_1`, `path_2`, … until one is free.
|
|
464
|
-
def collision_safe(path)
|
|
465
|
-
return path unless File.exist?(path)
|
|
466
|
-
|
|
467
|
-
ext = File.extname(path)
|
|
468
|
-
base = File.basename(path, ext)
|
|
469
|
-
dir = File.dirname(path)
|
|
470
|
-
i = 1
|
|
471
|
-
loop do
|
|
472
|
-
candidate = File.join(dir, "#{base}_#{i}#{ext}")
|
|
473
|
-
return candidate unless File.exist?(candidate)
|
|
474
|
-
|
|
475
|
-
i += 1
|
|
476
|
-
end
|
|
477
|
-
end
|
|
281
|
+
def commit!(...) = @committer.commit!(...)
|
|
282
|
+
def resolve_final_path(...) = @committer.resolve_final_path(...)
|
|
283
|
+
def staging_path_for(...) = @committer.staging_path_for(...)
|
|
284
|
+
def cleanup_staging(...) = @committer.cleanup_staging(...)
|
|
285
|
+
def build_clean_path(...) = @committer.build_clean_path(...)
|
|
286
|
+
def collision_safe(...) = @committer.collision_safe(...)
|
|
287
|
+
def copy_file_exclusive(...) = @committer.copy_file_exclusive(...)
|
|
288
|
+
def link_with_collision_safe_name(...) = @committer.link_with_collision_safe_name(...)
|
|
478
289
|
|
|
479
290
|
def print_summary(summary)
|
|
480
291
|
Display.header 'Summary'
|
|
@@ -482,119 +293,28 @@ module Metaclean
|
|
|
482
293
|
if summary[:unverified].positive?
|
|
483
294
|
Display.warning "Unverified (clean could not be confirmed): #{summary[:unverified]} file(s)"
|
|
484
295
|
end
|
|
296
|
+
if summary[:unsupported].positive?
|
|
297
|
+
Display.warning "Unsupported (not cleaned): #{summary[:unsupported]} file(s)"
|
|
298
|
+
end
|
|
485
299
|
puts Display.error("Failed: #{summary[:failed]}") if summary[:failed].positive?
|
|
486
300
|
Display.info "Total embedded tags removed: #{summary[:removed_total]}"
|
|
487
301
|
if summary[:residual_files].positive?
|
|
488
302
|
Display.warning "Files with privacy residual: #{summary[:residual_files]}"
|
|
489
303
|
end
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
# File discovery — turning the user's paths into a flat list.
|
|
493
|
-
|
|
494
|
-
def expand_files(paths)
|
|
495
|
-
explicit = []
|
|
496
|
-
discovered = []
|
|
497
|
-
paths.each do |p|
|
|
498
|
-
# Symlinks are always skipped — avoids cleaning something through a link
|
|
499
|
-
# that points outside the intended scope.
|
|
500
|
-
if File.symlink?(p)
|
|
501
|
-
Display.warning "Skipping symlink: #{p}"
|
|
502
|
-
next
|
|
503
|
-
end
|
|
504
|
-
if File.directory?(p)
|
|
505
|
-
collect_dir(p, discovered)
|
|
506
|
-
elsif File.file?(p)
|
|
507
|
-
# Explicit file argument — never apply skip?, the user asked for
|
|
508
|
-
# this exact path. (Skip filters exist to avoid re-cleaning our
|
|
509
|
-
# own outputs during recursion, not to override the CLI.)
|
|
510
|
-
explicit << p
|
|
511
|
-
else
|
|
512
|
-
Display.warning "Not found: #{p}"
|
|
513
|
-
end
|
|
514
|
-
end
|
|
515
|
-
discovered.reject! { |f| skip?(f) }
|
|
516
|
-
dedupe_by_realpath(explicit + discovered)
|
|
517
|
-
end
|
|
518
|
-
|
|
519
|
-
# Same file via two different paths (or via symlink + direct path) should
|
|
520
|
-
# be cleaned once. Comparing by realpath catches both cases. If realpath
|
|
521
|
-
# raises (broken symlink, permission denied), fall back to the raw path.
|
|
522
|
-
def dedupe_by_realpath(paths)
|
|
523
|
-
seen = {}
|
|
524
|
-
paths.each_with_object([]) do |p, acc|
|
|
525
|
-
key = safe_realpath(p)
|
|
526
|
-
next if seen[key]
|
|
527
|
-
|
|
528
|
-
seen[key] = true
|
|
529
|
-
acc << p
|
|
530
|
-
end
|
|
531
|
-
end
|
|
532
|
-
|
|
533
|
-
# File.realpath, falling back to the raw path when it can't resolve
|
|
534
|
-
# (broken symlink, permission denied) instead of raising.
|
|
535
|
-
def safe_realpath(path)
|
|
536
|
-
File.realpath(path)
|
|
537
|
-
rescue StandardError
|
|
538
|
-
path
|
|
539
|
-
end
|
|
540
|
-
|
|
541
|
-
def collect_dir(dir, out)
|
|
542
|
-
if @options[:recursive]
|
|
543
|
-
walk_recursive(dir, out)
|
|
544
|
-
else
|
|
545
|
-
# Non-recursive: just the immediate children of `dir`. Use Dir.children,
|
|
546
|
-
# NOT Dir.glob("#{dir}/*") — glob interprets the WHOLE pattern, so a
|
|
547
|
-
# directory name containing glob metacharacters (e.g. "Holiday [2024]")
|
|
548
|
-
# matches nothing and the entire folder is silently skipped. Dir.children
|
|
549
|
-
# surfaces dotfiles too; skip? filters them later, same as walk_recursive.
|
|
550
|
-
Dir.children(dir).each do |entry|
|
|
551
|
-
sub = File.join(dir, entry)
|
|
552
|
-
next if File.symlink?(sub)
|
|
553
|
-
|
|
554
|
-
out << sub if File.file?(sub)
|
|
555
|
-
end
|
|
304
|
+
if @scan_errors.positive?
|
|
305
|
+
Display.warning "Paths skipped during discovery (not found or unreadable): #{@scan_errors}"
|
|
556
306
|
end
|
|
557
|
-
rescue SystemCallError => e
|
|
558
|
-
# Any Errno (EACCES/ENOENT/ENOTDIR from a dir replaced mid-scan, EIO, …):
|
|
559
|
-
# warn and skip this directory so one bad entry doesn't abort discovery of
|
|
560
|
-
# the rest of the batch.
|
|
561
|
-
Display.warning "Skipping #{dir}: #{e.message}"
|
|
562
307
|
end
|
|
563
308
|
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
next if File.symlink?(sub)
|
|
570
|
-
|
|
571
|
-
if File.directory?(sub)
|
|
572
|
-
walk_recursive(sub, out)
|
|
573
|
-
elsif File.file?(sub)
|
|
574
|
-
out << sub
|
|
575
|
-
end
|
|
576
|
-
end
|
|
577
|
-
rescue SystemCallError => e
|
|
578
|
-
# Any Errno (EACCES/ENOENT/ENOTDIR from a dir replaced mid-scan, EIO, …):
|
|
579
|
-
# warn and skip this directory so one bad entry doesn't abort discovery of
|
|
580
|
-
# the rest of the batch.
|
|
581
|
-
Display.warning "Skipping #{dir}: #{e.message}"
|
|
309
|
+
def expand_files(paths)
|
|
310
|
+
files = @discovery.expand(paths)
|
|
311
|
+
@scan_errors = @discovery.scan_errors
|
|
312
|
+
@path_guards = @discovery.guards
|
|
313
|
+
files
|
|
582
314
|
end
|
|
583
315
|
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
# `metaclean .hidden.jpg`, they meant it. Hidden files (dot-prefixed)
|
|
587
|
-
# might be system metadata; .bak/_clean/.metaclean.tmp.* are our own
|
|
588
|
-
# outputs, so skipping them prevents loops on re-runs.
|
|
589
|
-
def skip?(file)
|
|
590
|
-
base = File.basename(file)
|
|
591
|
-
return true if base.start_with?('.')
|
|
592
|
-
return true if base.end_with?('.bak')
|
|
593
|
-
return true if base =~ Metaclean::CLEAN_OUTPUT_RE
|
|
594
|
-
# Matches our staging temps regardless of the pid/random suffix format.
|
|
595
|
-
return true if base.include?(Metaclean::TMP_MARKER)
|
|
596
|
-
|
|
597
|
-
false
|
|
316
|
+
def guard_for(file)
|
|
317
|
+
@path_guards[file] || FileOps.path_guard!(file)
|
|
598
318
|
end
|
|
599
319
|
end
|
|
600
320
|
end
|