metaclean 1.0.2 → 4.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/LICENSE +1 -1
- data/README.md +143 -47
- data/bin/metaclean +1 -22
- data/lib/metaclean/cli.rb +42 -92
- data/lib/metaclean/display.rb +59 -40
- data/lib/metaclean/exiftool.rb +72 -91
- data/lib/metaclean/ffmpeg.rb +84 -0
- data/lib/metaclean/mat2.rb +43 -40
- data/lib/metaclean/qpdf.rb +30 -26
- data/lib/metaclean/runner.rb +345 -170
- data/lib/metaclean/strategy.rb +118 -39
- data/lib/metaclean/version.rb +1 -3
- data/lib/metaclean.rb +151 -26
- metadata +11 -8
data/lib/metaclean/runner.rb
CHANGED
|
@@ -1,56 +1,62 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
# ───────────────────────────────────────────────────────────────────────────
|
|
4
3
|
# The orchestrator. Given a list of paths and parsed CLI options, this class:
|
|
5
4
|
#
|
|
6
|
-
# 1. Expands paths into a flat list of files (handling directories
|
|
7
|
-
# recursion
|
|
5
|
+
# 1. Expands paths into a flat list of files (handling directories and
|
|
6
|
+
# recursion; symlinks are skipped).
|
|
8
7
|
# 2. Asks the user for confirmation (unless --force).
|
|
9
8
|
# 3. For each file, runs the strategy pipeline (mat2 / exiftool / qpdf)
|
|
10
9
|
# using the "atomic write" pattern so a crash never leaves a
|
|
11
10
|
# half-cleaned file.
|
|
12
11
|
# 4. Prints a before/after diff and a final summary.
|
|
13
|
-
# ───────────────────────────────────────────────────────────────────────────
|
|
14
12
|
|
|
15
13
|
require 'fileutils'
|
|
16
|
-
require '
|
|
17
|
-
require 'set'
|
|
18
|
-
require 'tmpdir'
|
|
14
|
+
require 'securerandom'
|
|
19
15
|
|
|
20
16
|
module Metaclean
|
|
21
17
|
class Runner
|
|
22
|
-
# Constructor — just stashes the options Hash. The CLI builds it.
|
|
23
18
|
def initialize(options)
|
|
24
19
|
@options = options
|
|
20
|
+
# Paths that couldn't be read during discovery (missing arg, unreadable
|
|
21
|
+
# directory). Tracked so a partially-scanned batch exits non-zero instead
|
|
22
|
+
# of letting automation mistake "some files cleaned" for "everything done".
|
|
23
|
+
@scan_errors = 0
|
|
25
24
|
end
|
|
26
25
|
|
|
27
|
-
# ─────────────────────────────────────────────────────────────────
|
|
28
26
|
# Public entry points: one for `--inspect`, one for the cleaning flow.
|
|
29
|
-
# ─────────────────────────────────────────────────────────────────
|
|
30
27
|
|
|
31
28
|
def inspect_paths(paths)
|
|
32
29
|
files = expand_files(paths)
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
if @options[:format] == :json
|
|
37
|
-
out = files.map { |f| { file: f, metadata: Exiftool.read(f) } }
|
|
38
|
-
puts JSON.pretty_generate(out)
|
|
39
|
-
return
|
|
30
|
+
if files.empty?
|
|
31
|
+
Display.warning('No files to inspect.')
|
|
32
|
+
exit 1
|
|
40
33
|
end
|
|
41
|
-
|
|
42
|
-
# Human output: pretty header + grouped table per file.
|
|
34
|
+
failed = 0
|
|
43
35
|
files.each do |file|
|
|
44
36
|
Display.header "📄 #{file}"
|
|
45
37
|
meta = Exiftool.read(file)
|
|
46
38
|
Display.section "Metadata (#{Display.count_embedded(meta)} embedded tags)"
|
|
47
39
|
Display.metadata_table(meta)
|
|
40
|
+
rescue Error, SystemCallError => e
|
|
41
|
+
# One unreadable/odd file shouldn't abort inspecting the rest — mirrors
|
|
42
|
+
# the per-file rescue in the clean batch.
|
|
43
|
+
warn Display.error("#{file}: #{e.message}")
|
|
44
|
+
failed += 1
|
|
48
45
|
end
|
|
46
|
+
|
|
47
|
+
# A file we couldn't read is a non-zero condition: a script using --inspect
|
|
48
|
+
# as a gate must not mistake "couldn't read it" for "no metadata". Discovery
|
|
49
|
+
# errors (missing paths / unreadable dirs) count too.
|
|
50
|
+
exit 1 if failed.positive? || @scan_errors.positive?
|
|
49
51
|
end
|
|
50
52
|
|
|
51
53
|
def clean_paths(paths)
|
|
52
54
|
files = expand_files(paths)
|
|
53
|
-
|
|
55
|
+
# See inspect_paths: nothing to act on is a non-zero condition, not success.
|
|
56
|
+
if files.empty?
|
|
57
|
+
Display.warning('No files to process.')
|
|
58
|
+
exit 1
|
|
59
|
+
end
|
|
54
60
|
|
|
55
61
|
announce_tools
|
|
56
62
|
|
|
@@ -59,79 +65,81 @@ module Metaclean
|
|
|
59
65
|
unless @options[:force] || @options[:dry_run]
|
|
60
66
|
action = @options[:in_place] ? 'OVERWRITE' : 'create cleaned copies of'
|
|
61
67
|
puts Display.c("About to #{action} #{files.size} file(s).", :yellow)
|
|
62
|
-
if @options[:in_place]
|
|
63
|
-
|
|
68
|
+
if @options[:in_place]
|
|
69
|
+
# The .bak IS the original — metadata intact. Say so plainly: a user who
|
|
70
|
+
# shares the "cleaned" folder would otherwise leak it via the backup.
|
|
71
|
+
puts Display.c('Each <file>.bak is the ORIGINAL, with all metadata still in it — ' \
|
|
72
|
+
'delete or move the .bak files before sharing the folder.', :yellow)
|
|
64
73
|
end
|
|
65
74
|
print Display.c('Proceed? [y/N] ', :bold)
|
|
66
|
-
|
|
67
|
-
# (
|
|
68
|
-
|
|
69
|
-
|
|
75
|
+
ans = $stdin.gets&.strip&.downcase # gets → nil on Ctrl-D
|
|
76
|
+
# Abort (no/blank/EOF) is a non-zero exit, not silent success — a
|
|
77
|
+
# non-interactive caller must not read "Aborted." as "files were cleaned".
|
|
78
|
+
unless %w[y yes].include?(ans)
|
|
79
|
+
Display.warning('Aborted.')
|
|
80
|
+
exit 1
|
|
81
|
+
end
|
|
70
82
|
end
|
|
71
83
|
|
|
72
|
-
summary = { cleaned: 0, failed: 0, removed_total: 0, residual_files: 0 }
|
|
84
|
+
summary = { cleaned: 0, unverified: 0, failed: 0, removed_total: 0, residual_files: 0 }
|
|
73
85
|
|
|
74
|
-
#
|
|
75
|
-
# to `clean_one` so it can render "[3/47]" in batch mode.
|
|
86
|
+
# index/total let clean_one render "[3/47]" in batch mode.
|
|
76
87
|
files.each_with_index do |file, idx|
|
|
77
88
|
result = clean_one(file, index: idx + 1, total: files.size)
|
|
78
89
|
summary[result[:status]] += 1
|
|
79
90
|
summary[:removed_total] += result[:removed].to_i
|
|
80
91
|
summary[:residual_files] += 1 if result[:residual].to_i.positive?
|
|
81
|
-
rescue Error => e
|
|
82
|
-
#
|
|
83
|
-
#
|
|
84
|
-
#
|
|
92
|
+
rescue Error, SystemCallError => e
|
|
93
|
+
# One bad file shouldn't abort the whole batch. SystemCallError
|
|
94
|
+
# (Errno::*: disk full, permission denied, read-only fs) is a SIBLING
|
|
95
|
+
# of our Error, not a subclass, so it must be named explicitly or it
|
|
96
|
+
# would escape this rescue and crash the run with a raw backtrace.
|
|
85
97
|
warn Display.error("#{file}: #{e.message}")
|
|
86
98
|
summary[:failed] += 1
|
|
87
99
|
end
|
|
88
100
|
|
|
89
101
|
print_summary(summary)
|
|
90
102
|
|
|
91
|
-
# Non-zero exit
|
|
92
|
-
|
|
93
|
-
exit 1 if summary[:failed].positive?
|
|
103
|
+
# Non-zero exit so CI/scripts can detect a failed or not-fully-verified file,
|
|
104
|
+
# OR a batch that was never fully discovered (a path/dir we couldn't read).
|
|
105
|
+
exit 1 if summary[:failed].positive? || summary[:unverified].positive? || @scan_errors.positive?
|
|
94
106
|
end
|
|
95
107
|
|
|
96
108
|
private
|
|
97
109
|
|
|
98
|
-
# ─────────────────────────────────────────────────────────────────
|
|
99
110
|
# Output helpers
|
|
100
|
-
# ─────────────────────────────────────────────────────────────────
|
|
101
111
|
|
|
102
112
|
def announce_tools
|
|
103
113
|
have = []
|
|
104
|
-
have << "exiftool #{Exiftool.version}"
|
|
105
|
-
have << "mat2 #{Mat2.version}"
|
|
106
|
-
have << "qpdf #{Qpdf.version
|
|
114
|
+
have << "exiftool #{Exiftool.version}" if Exiftool.available?
|
|
115
|
+
have << "mat2 #{Mat2.version}" if Mat2.available?
|
|
116
|
+
have << "qpdf #{Qpdf.version}" if Qpdf.available?
|
|
117
|
+
have << "ffmpeg #{Ffmpeg.version}" if Ffmpeg.available?
|
|
107
118
|
Display.info "Tools detected: #{have.join(', ')}"
|
|
108
119
|
Display.info '(dry-run — no files will be modified)' if @options[:dry_run]
|
|
109
120
|
end
|
|
110
121
|
|
|
111
|
-
#
|
|
112
|
-
# Cleaning a single file — the heart of the program.
|
|
113
|
-
# ─────────────────────────────────────────────────────────────────
|
|
114
|
-
|
|
122
|
+
# Cleaning a single file.
|
|
115
123
|
def clean_one(file, index:, total:)
|
|
116
124
|
prefix = total > 1 ? "[#{index}/#{total}] " : ''
|
|
117
125
|
Display.header "#{prefix}📄 #{file}"
|
|
118
126
|
|
|
119
127
|
# Read the "before" metadata FIRST — once we start cleaning, this is
|
|
120
128
|
# gone forever and we'd have nothing to diff against.
|
|
121
|
-
before =
|
|
129
|
+
before = read_metadata(file)
|
|
122
130
|
Display.section "Before (#{Display.count_embedded(before)} embedded tags)"
|
|
123
131
|
Display.metadata_table(before, only_embedded: true)
|
|
124
132
|
|
|
125
|
-
# Ask the strategy module which tools to run
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
133
|
+
# Ask the strategy module which tools to run for this file type.
|
|
134
|
+
tools = Strategy.tools_for(file)
|
|
135
|
+
# Warn when the stricter tool for a document format won't run: ExifTool
|
|
136
|
+
# alone leaves (and can't fully verify) document-internal metadata.
|
|
137
|
+
if Strategy.mat2_essential?(file) && !tools.include?(:mat2)
|
|
138
|
+
Display.warning 'mat2 will not run for this format — document-internal metadata may remain and cannot be verified.'
|
|
131
139
|
end
|
|
132
140
|
Display.info "Pipeline: #{tools.join(' → ')}"
|
|
133
141
|
|
|
134
|
-
#
|
|
142
|
+
# Atomic write setup:
|
|
135
143
|
# `final_path` = where the cleaned file will end up.
|
|
136
144
|
# `staging` = a temp file we mutate. After all tools succeed, we
|
|
137
145
|
# rename staging → final_path. If anything goes wrong
|
|
@@ -140,22 +148,32 @@ module Metaclean
|
|
|
140
148
|
final_path = resolve_final_path(file)
|
|
141
149
|
staging = staging_path_for(final_path)
|
|
142
150
|
|
|
143
|
-
FileUtils.cp(file, staging)
|
|
144
151
|
tool_results = []
|
|
145
152
|
begin
|
|
153
|
+
# TOCTOU guard: the path was a regular file at discovery (expand_files), but
|
|
154
|
+
# it could have been swapped for a symlink in the window since. Re-check
|
|
155
|
+
# right before we read/copy/back it up, so we never copy — or take a .bak —
|
|
156
|
+
# THROUGH a link pointing outside the intended scope. Bails to :failed.
|
|
157
|
+
raise Error, "#{file} became a symlink since discovery — refusing to clean it" if File.symlink?(file)
|
|
158
|
+
|
|
159
|
+
# The staging copy lives INSIDE the begin so the ensure below cleans up a
|
|
160
|
+
# partial temp if cp is interrupted (Ctrl-C) or fails mid-copy (disk full,
|
|
161
|
+
# read-only fs). cp only ever reads the original, so the source is intact
|
|
162
|
+
# regardless.
|
|
163
|
+
copy_file_exclusive(file, staging)
|
|
146
164
|
tools.each do |tool|
|
|
147
165
|
tool_results << run_tool(tool, staging)
|
|
148
166
|
end
|
|
149
167
|
|
|
150
168
|
# Re-read metadata of the cleaned staging file for the diff.
|
|
151
|
-
after =
|
|
169
|
+
after = read_metadata(staging)
|
|
152
170
|
Display.section "After (#{Display.count_embedded(after)} embedded tags)"
|
|
153
171
|
Display.metadata_table(after, only_embedded: true)
|
|
154
172
|
|
|
155
173
|
Display.section 'Diff'
|
|
156
174
|
Display.diff(before, after)
|
|
157
175
|
|
|
158
|
-
#
|
|
176
|
+
# Anything privacy-relevant that survived the strip.
|
|
159
177
|
residual = Strategy.privacy_residual(after)
|
|
160
178
|
if residual.any?
|
|
161
179
|
Display.warning "Privacy-relevant tags still present (#{residual.size}):"
|
|
@@ -166,14 +184,42 @@ module Metaclean
|
|
|
166
184
|
if @options[:dry_run]
|
|
167
185
|
File.delete(staging) if File.exist?(staging)
|
|
168
186
|
Display.info '(dry-run: nothing was written)'
|
|
169
|
-
return finalize_result(tool_results, before, after, residual)
|
|
187
|
+
return finalize_result(tool_results, before, after, residual, file: file)
|
|
170
188
|
end
|
|
171
189
|
|
|
172
|
-
#
|
|
173
|
-
|
|
174
|
-
|
|
190
|
+
# Never write output unless the file is genuinely clean: at least one
|
|
191
|
+
# tool ran AND no privacy-relevant tag survived. Otherwise the staging
|
|
192
|
+
# file — committed as a "_clean" copy or an in-place overwrite — would
|
|
193
|
+
# not actually be clean, the exact false-clean this tool exists to
|
|
194
|
+
# prevent. Bail to :failed and let the ensure block delete staging,
|
|
195
|
+
# leaving the original untouched.
|
|
196
|
+
unless cleaned?(tool_results, residual)
|
|
197
|
+
reason = tools_succeeded?(tool_results) ? 'Privacy-relevant tags survived' : 'All tools failed'
|
|
198
|
+
Display.warning "#{reason} — not writing output."
|
|
199
|
+
return finalize_result(tool_results, before, after, residual, file: file)
|
|
200
|
+
end
|
|
175
201
|
|
|
176
|
-
|
|
202
|
+
# Preserve the original's permission bits onto the cleaned output. cp and
|
|
203
|
+
# the tools' temp renames otherwise leave it at the umask default, which
|
|
204
|
+
# could widen a locked-down 0600 file to 0644 — a leak for a privacy tool.
|
|
205
|
+
File.chmod(File.stat(file).mode, staging)
|
|
206
|
+
|
|
207
|
+
# In-place clean of a hard-linked file only re-points THIS name (rename) at
|
|
208
|
+
# the freshly-cleaned inode; the file's other names still point at the
|
|
209
|
+
# original, metadata-bearing inode. This name is genuinely clean, but warn
|
|
210
|
+
# so the user knows the other links aren't covered by the run.
|
|
211
|
+
warn_if_hardlinked(file) if @options[:in_place]
|
|
212
|
+
|
|
213
|
+
# Commit: move/link staging → final_path (backing up the original in place).
|
|
214
|
+
final_path = commit!(staging, final_path)
|
|
215
|
+
result = finalize_result(tool_results, before, after, residual, file: file)
|
|
216
|
+
if result[:status] == :unverified
|
|
217
|
+
reason = tool_errored?(tool_results) ? 'a tool in the pipeline failed' : 'mat2 did not run on this format'
|
|
218
|
+
Display.warning "→ #{final_path} (unverified — #{reason})"
|
|
219
|
+
else
|
|
220
|
+
Display.success "→ #{final_path}"
|
|
221
|
+
end
|
|
222
|
+
result
|
|
177
223
|
ensure
|
|
178
224
|
# Last-resort cleanup. If `commit!` already moved the staging file,
|
|
179
225
|
# `File.exist?(staging)` is false and this is a no-op. The path-
|
|
@@ -183,16 +229,32 @@ module Metaclean
|
|
|
183
229
|
end
|
|
184
230
|
end
|
|
185
231
|
|
|
232
|
+
# Warn when an in-place target has more than one hard link: a rename only
|
|
233
|
+
# cleans the named link, leaving the others pointing at the original metadata.
|
|
234
|
+
def warn_if_hardlinked(file)
|
|
235
|
+
nlink = File.stat(file).nlink
|
|
236
|
+
return unless nlink > 1
|
|
237
|
+
|
|
238
|
+
Display.warning "#{file} has #{nlink} hard links — only this name is cleaned; " \
|
|
239
|
+
"the other #{nlink - 1} still contain the original metadata."
|
|
240
|
+
end
|
|
241
|
+
|
|
186
242
|
# Dispatches to the right wrapper module. Returns a small Hash so the
|
|
187
243
|
# caller can summarize tool-by-tool success/failure.
|
|
188
244
|
def run_tool(tool, path)
|
|
189
245
|
case tool
|
|
190
246
|
when :exiftool
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
247
|
+
# :unsupported means ExifTool can read but not write this format (a
|
|
248
|
+
# ZIP-based document mat2 owns) — a soft skip, NOT a pipeline failure.
|
|
249
|
+
# Pass the privacy tag names so TIFF/DNG IFD0 tags `-all=` won't drop
|
|
250
|
+
# still get deleted (losslessly).
|
|
251
|
+
if Exiftool.strip!(path, also_delete: Strategy::PRIVACY_TAGS) == :unsupported
|
|
252
|
+
Display.info ' · exiftool (read-only for this format, skipped)'
|
|
253
|
+
{ tool: :exiftool, ok: false, skipped: true, note: :unsupported }
|
|
254
|
+
else
|
|
255
|
+
Display.info ' ✓ exiftool'
|
|
256
|
+
{ tool: :exiftool, ok: true }
|
|
257
|
+
end
|
|
196
258
|
when :mat2
|
|
197
259
|
result = Mat2.strip!(path)
|
|
198
260
|
# mat2 returns either `true` (success) or a symbol indicating a
|
|
@@ -214,63 +276,183 @@ module Metaclean
|
|
|
214
276
|
Qpdf.rebuild!(path)
|
|
215
277
|
Display.info ' ✓ qpdf'
|
|
216
278
|
{ tool: :qpdf, ok: true }
|
|
279
|
+
when :ffmpeg
|
|
280
|
+
# Matroska remux. A failure raises and is caught below (→ not written).
|
|
281
|
+
Ffmpeg.strip!(path)
|
|
282
|
+
Display.info ' ✓ ffmpeg'
|
|
283
|
+
{ tool: :ffmpeg, ok: true }
|
|
217
284
|
end
|
|
218
|
-
rescue Error => e
|
|
285
|
+
rescue Error, SystemCallError => e
|
|
219
286
|
# One tool failing shouldn't abort the pipeline — we want to keep
|
|
220
287
|
# trying with the others. The `finalize_result` step decides whether
|
|
221
|
-
# the overall file counts as cleaned or failed.
|
|
222
|
-
|
|
288
|
+
# the overall file counts as cleaned or failed. `SystemCallError`
|
|
289
|
+
# (Errno::*) covers a tool wrapper's internal FileUtils.mv/File.delete
|
|
290
|
+
# raising on permission/quota/disk errors — without it those would
|
|
291
|
+
# escape and crash the batch.
|
|
292
|
+
# Collapse whitespace and bound the length: some tools (notably mat2) dump a
|
|
293
|
+
# multi-line Python traceback on failure, which would otherwise flood the
|
|
294
|
+
# diff. One readable line is enough — re-run the tool directly to debug.
|
|
295
|
+
msg = Display.truncate(e.message.gsub(/\s+/, ' ').strip, 200)
|
|
296
|
+
Display.warning " ✗ #{tool}: #{msg} — continuing"
|
|
223
297
|
{ tool: tool, ok: false, error: e.message }
|
|
224
298
|
end
|
|
225
299
|
|
|
226
|
-
|
|
300
|
+
# :cleaned needs ALL of: a tool genuinely ran, no privacy residual survived,
|
|
301
|
+
# no pipeline tool errored, AND — for a format where mat2 owns coverage
|
|
302
|
+
# ExifTool can't re-read (Office/PDF doc internals) — mat2 actually ran. A
|
|
303
|
+
# tool that errored, or an absent mat2 on a document format, means the
|
|
304
|
+
# pipeline didn't fully complete and the residual check is partly blind, so
|
|
305
|
+
# the result is :unverified, not a confident :cleaned. `file` is needed only
|
|
306
|
+
# for that mat2-coverage check.
|
|
307
|
+
def finalize_result(tool_results, before, after, residual, file: nil)
|
|
227
308
|
removed = removed_embedded_count(before, after)
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
{ status: status,
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
309
|
+
status = if !cleaned?(tool_results, residual)
|
|
310
|
+
:failed
|
|
311
|
+
elsif !tool_errored?(tool_results) && !mat2_coverage_gap?(tool_results, file)
|
|
312
|
+
:cleaned
|
|
313
|
+
else
|
|
314
|
+
:unverified
|
|
315
|
+
end
|
|
316
|
+
{ status: status, removed: removed, residual: residual.size }
|
|
317
|
+
end
|
|
318
|
+
|
|
319
|
+
# mat2 is essential for this format (Office/PDF internals ExifTool can't
|
|
320
|
+
# strip or fully re-read) but did NOT actually run and strip — absent,
|
|
321
|
+
# unsupported soft-skip, or errored. The residual check can't confirm the
|
|
322
|
+
# clean, so don't report a confident :cleaned.
|
|
323
|
+
def mat2_coverage_gap?(tool_results, file)
|
|
324
|
+
return false unless file && Strategy.mat2_essential?(file)
|
|
325
|
+
|
|
326
|
+
tool_results.none? { |r| r[:tool] == :mat2 && r[:ok] && !r[:skipped] }
|
|
327
|
+
end
|
|
328
|
+
|
|
329
|
+
# A file is genuinely cleaned only when at least one tool actually ran
|
|
330
|
+
# (not just a mat2 :unsupported soft-skip) AND no privacy-relevant tag
|
|
331
|
+
# survived. Both the commit gate and the final status use this ONE
|
|
332
|
+
# predicate, so they can never disagree — we never write a "_clean" copy
|
|
333
|
+
# (or overwrite an original in place) and then report it :failed. Silently
|
|
334
|
+
# marking a file clean while sensitive metadata is still present is the
|
|
335
|
+
# worst possible outcome for a privacy tool.
|
|
336
|
+
def cleaned?(tool_results, residual)
|
|
337
|
+
tools_succeeded?(tool_results) && residual.empty?
|
|
338
|
+
end
|
|
339
|
+
|
|
340
|
+
# Did at least one tool genuinely run (not a mat2 :unsupported soft-skip)?
|
|
341
|
+
def tools_succeeded?(tool_results)
|
|
342
|
+
tool_results.any? { |r| r[:ok] && !r[:skipped] }
|
|
343
|
+
end
|
|
344
|
+
|
|
345
|
+
# Did a tool that was meant to run error out (not a mat2 :unsupported
|
|
346
|
+
# soft-skip)? Even with an empty residual that means the pipeline didn't
|
|
347
|
+
# fully complete, so the clean can't be reported as a confident :cleaned.
|
|
348
|
+
def tool_errored?(tool_results)
|
|
349
|
+
tool_results.any? { |r| !r[:ok] && !r[:skipped] }
|
|
350
|
+
end
|
|
351
|
+
|
|
352
|
+
# Read metadata for the before/after diff. ensure_tools! guarantees exiftool
|
|
353
|
+
# is present before any run.
|
|
354
|
+
def read_metadata(path)
|
|
355
|
+
Exiftool.read(path)
|
|
239
356
|
end
|
|
240
357
|
|
|
241
358
|
def removed_embedded_count(before, after)
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
359
|
+
before.keys.count { |key| Display.embedded_key?(key) && !after.key?(key) }
|
|
360
|
+
end
|
|
361
|
+
|
|
362
|
+
# Path helpers — figuring out where to stage and where to commit.
|
|
246
363
|
|
|
247
|
-
|
|
364
|
+
def commit!(staging, final_path)
|
|
365
|
+
committed = false
|
|
366
|
+
# Make a backup of the original BEFORE we overwrite it. The order matters:
|
|
367
|
+
# if the rename below fails, the backup still exists.
|
|
368
|
+
if @options[:in_place]
|
|
369
|
+
backup = copy_with_collision_safe_name(final_path, "#{final_path}.bak")
|
|
370
|
+
# File.rename, NOT FileUtils.mv. staging is in the same dir as final_path
|
|
371
|
+
# (staging_path_for), so this is always an atomic same-fs swap. FileUtils.mv
|
|
372
|
+
# would, on EPERM (e.g. a sticky /tmp file not owned by us) or EXDEV, fall
|
|
373
|
+
# back to a TRUNCATING copy of the original — and an interrupt mid-copy
|
|
374
|
+
# would corrupt the original while the rescue below deleted the only backup.
|
|
375
|
+
# File.rename raises BEFORE touching final_path, so the rescue's
|
|
376
|
+
# "staging still exists ⇒ original intact" assumption always holds.
|
|
377
|
+
File.rename(staging, final_path)
|
|
378
|
+
committed = true
|
|
379
|
+
return final_path
|
|
248
380
|
end
|
|
381
|
+
|
|
382
|
+
link_with_collision_safe_name(staging, final_path)
|
|
383
|
+
rescue SystemCallError, Interrupt
|
|
384
|
+
# The rename failed (disk full, read-only fs, cross-device) OR the user hit
|
|
385
|
+
# Ctrl-C in the window after the .bak was written but before the mv. Either
|
|
386
|
+
# way the original is untouched, so the .bak is a redundant copy of it —
|
|
387
|
+
# remove it instead of leaving a stray file behind, then re-raise (a
|
|
388
|
+
# SystemCallError is reported per-file as failed; an Interrupt propagates to
|
|
389
|
+
# the CLI's exit-130 handler).
|
|
390
|
+
File.delete(backup) if backup && !committed && File.exist?(staging) && File.exist?(backup)
|
|
391
|
+
raise
|
|
249
392
|
end
|
|
250
393
|
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
394
|
+
def link_with_collision_safe_name(staging, preferred)
|
|
395
|
+
target = preferred
|
|
396
|
+
loop do
|
|
397
|
+
File.link(staging, target)
|
|
398
|
+
File.delete(staging)
|
|
399
|
+
return target
|
|
400
|
+
rescue Errno::EEXIST
|
|
401
|
+
target = collision_safe(target)
|
|
402
|
+
rescue Errno::EACCES, Errno::EPERM, Errno::ENOTSUP, NotImplementedError
|
|
403
|
+
# The filesystem can't hard-link (Linux returns EPERM; macOS/BSD on
|
|
404
|
+
# exFAT/FAT/SMB returns ENOTSUP/EOPNOTSUPP — same Errno class). Fall back
|
|
405
|
+
# to a plain exclusive copy so removable/network drives still clean.
|
|
406
|
+
target = copy_with_collision_safe_name(staging, target)
|
|
407
|
+
File.delete(staging)
|
|
408
|
+
return target
|
|
409
|
+
end
|
|
410
|
+
end
|
|
411
|
+
|
|
412
|
+
def copy_with_collision_safe_name(src, preferred)
|
|
413
|
+
target = preferred
|
|
414
|
+
loop do
|
|
415
|
+
copy_file_exclusive(src, target, preserve: true)
|
|
416
|
+
return target
|
|
417
|
+
rescue Errno::EEXIST
|
|
418
|
+
target = collision_safe(target)
|
|
419
|
+
end
|
|
420
|
+
end
|
|
421
|
+
|
|
422
|
+
def copy_file_exclusive(src, dest, preserve: false)
|
|
423
|
+
src_stat = File.lstat(src)
|
|
424
|
+
raise Error, "#{src} is a symlink — refusing to copy it" if src_stat.symlink?
|
|
425
|
+
|
|
426
|
+
mode = src_stat.mode & 0o7777
|
|
427
|
+
created = false
|
|
428
|
+
File.open(dest, File::WRONLY | File::CREAT | File::EXCL, mode) do |out|
|
|
429
|
+
created = true
|
|
430
|
+
File.open(src, 'rb') do |input|
|
|
431
|
+
opened = input.stat
|
|
432
|
+
unless opened.dev == src_stat.dev && opened.ino == src_stat.ino
|
|
433
|
+
raise Error, "#{src} changed while opening — refusing to copy it"
|
|
434
|
+
end
|
|
435
|
+
|
|
436
|
+
IO.copy_stream(input, out)
|
|
437
|
+
end
|
|
438
|
+
end
|
|
439
|
+
return unless preserve
|
|
440
|
+
|
|
441
|
+
# Best-effort: the bytes are already fully copied, so a failed mode/timestamp
|
|
442
|
+
# restore (e.g. utime/chmod on a FAT/exFAT mount) must NOT discard an
|
|
443
|
+
# otherwise-complete file by falling into the delete-on-error rescue below.
|
|
444
|
+
begin
|
|
445
|
+
File.chmod(mode, dest)
|
|
446
|
+
File.utime(src_stat.atime, src_stat.mtime, dest)
|
|
447
|
+
rescue SystemCallError
|
|
448
|
+
nil
|
|
265
449
|
end
|
|
266
|
-
|
|
450
|
+
rescue StandardError, Interrupt
|
|
451
|
+
File.delete(dest) if created && dest && File.exist?(dest)
|
|
452
|
+
raise
|
|
267
453
|
end
|
|
268
454
|
|
|
269
455
|
def resolve_final_path(file)
|
|
270
|
-
# When following a symlink with --in-place, we want to overwrite the
|
|
271
|
-
# *target* of the link, not replace the link itself with a regular
|
|
272
|
-
# file. `realpath` resolves through the link.
|
|
273
|
-
return File.realpath(file) if @options[:in_place] && File.symlink?(file)
|
|
274
456
|
return file if @options[:in_place]
|
|
275
457
|
|
|
276
458
|
# Default: write `<name>_clean.<ext>` next to the original. If it
|
|
@@ -281,7 +463,7 @@ module Metaclean
|
|
|
281
463
|
def build_clean_path(file)
|
|
282
464
|
ext = File.extname(file)
|
|
283
465
|
base = File.basename(file, ext)
|
|
284
|
-
File.join(File.dirname(file), "#{base}
|
|
466
|
+
File.join(File.dirname(file), "#{base}#{Metaclean::CLEAN_SUFFIX}#{ext}")
|
|
285
467
|
end
|
|
286
468
|
|
|
287
469
|
# Staging path lives in the same directory as the destination so that
|
|
@@ -290,13 +472,15 @@ module Metaclean
|
|
|
290
472
|
# The original extension is preserved as the LAST segment so tools like
|
|
291
473
|
# mat2 — which dispatch on file extension — see the real type.
|
|
292
474
|
def staging_path_for(final_path)
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
475
|
+
dir = File.dirname(final_path)
|
|
476
|
+
ext = File.extname(final_path)
|
|
477
|
+
# SecureRandom (not rand) makes the staging name unpredictable, so a
|
|
478
|
+
# hostile process in the same directory can't pre-create it as a symlink
|
|
479
|
+
# that `FileUtils.cp` would copy the (still-sensitive) original through.
|
|
480
|
+
File.join(dir, "#{Metaclean::TMP_MARKER}#{Process.pid}.#{SecureRandom.hex(8)}#{ext}")
|
|
296
481
|
end
|
|
297
482
|
|
|
298
|
-
# If `path` is taken,
|
|
299
|
-
# one. `loop do … end` runs forever; we `return` out of it.
|
|
483
|
+
# If `path` is taken, try `path_1`, `path_2`, … until one is free.
|
|
300
484
|
def collision_safe(path)
|
|
301
485
|
return path unless File.exist?(path)
|
|
302
486
|
|
|
@@ -312,38 +496,32 @@ module Metaclean
|
|
|
312
496
|
end
|
|
313
497
|
end
|
|
314
498
|
|
|
315
|
-
# Translates the on/off CLI flags into a "prefer" hash that Strategy
|
|
316
|
-
# understands. Keeping this as one method makes the wiring obvious.
|
|
317
|
-
def tool_prefs
|
|
318
|
-
{
|
|
319
|
-
mat2: !@options[:no_mat2] && !@options[:exiftool_only],
|
|
320
|
-
qpdf: !@options[:no_qpdf] && !@options[:exiftool_only],
|
|
321
|
-
exiftool: !@options[:no_exiftool]
|
|
322
|
-
}
|
|
323
|
-
end
|
|
324
|
-
|
|
325
499
|
def print_summary(summary)
|
|
326
500
|
Display.header 'Summary'
|
|
327
501
|
Display.success "Cleaned: #{summary[:cleaned]} file(s)"
|
|
502
|
+
if summary[:unverified].positive?
|
|
503
|
+
Display.warning "Unverified (clean could not be confirmed): #{summary[:unverified]} file(s)"
|
|
504
|
+
end
|
|
328
505
|
puts Display.error("Failed: #{summary[:failed]}") if summary[:failed].positive?
|
|
329
506
|
Display.info "Total embedded tags removed: #{summary[:removed_total]}"
|
|
330
507
|
if summary[:residual_files].positive?
|
|
331
508
|
Display.warning "Files with privacy residual: #{summary[:residual_files]}"
|
|
332
509
|
end
|
|
510
|
+
if @scan_errors.positive?
|
|
511
|
+
Display.warning "Paths skipped during discovery (not found or unreadable): #{@scan_errors}"
|
|
512
|
+
end
|
|
333
513
|
end
|
|
334
514
|
|
|
335
|
-
# ─────────────────────────────────────────────────────────────────
|
|
336
515
|
# File discovery — turning the user's paths into a flat list.
|
|
337
|
-
# ─────────────────────────────────────────────────────────────────
|
|
338
516
|
|
|
339
517
|
def expand_files(paths)
|
|
340
518
|
explicit = []
|
|
341
519
|
discovered = []
|
|
342
520
|
paths.each do |p|
|
|
343
|
-
# Symlinks are skipped
|
|
344
|
-
#
|
|
345
|
-
if File.symlink?(p)
|
|
346
|
-
Display.warning "Skipping symlink: #{p}
|
|
521
|
+
# Symlinks are always skipped — avoids cleaning something through a link
|
|
522
|
+
# that points outside the intended scope.
|
|
523
|
+
if File.symlink?(p)
|
|
524
|
+
Display.warning "Skipping symlink: #{p}"
|
|
347
525
|
next
|
|
348
526
|
end
|
|
349
527
|
if File.directory?(p)
|
|
@@ -355,12 +533,11 @@ module Metaclean
|
|
|
355
533
|
explicit << p
|
|
356
534
|
else
|
|
357
535
|
Display.warning "Not found: #{p}"
|
|
536
|
+
@scan_errors += 1
|
|
358
537
|
end
|
|
359
538
|
end
|
|
360
539
|
discovered.reject! { |f| skip?(f) }
|
|
361
|
-
|
|
362
|
-
result.select! { |f| type_allowed?(f) } if @options[:types]
|
|
363
|
-
dedupe_by_realpath(result)
|
|
540
|
+
dedupe_by_realpath(explicit + discovered)
|
|
364
541
|
end
|
|
365
542
|
|
|
366
543
|
# Same file via two different paths (or via symlink + direct path) should
|
|
@@ -369,11 +546,7 @@ module Metaclean
|
|
|
369
546
|
def dedupe_by_realpath(paths)
|
|
370
547
|
seen = {}
|
|
371
548
|
paths.each_with_object([]) do |p, acc|
|
|
372
|
-
key =
|
|
373
|
-
File.realpath(p)
|
|
374
|
-
rescue StandardError
|
|
375
|
-
p
|
|
376
|
-
end
|
|
549
|
+
key = safe_realpath(p)
|
|
377
550
|
next if seen[key]
|
|
378
551
|
|
|
379
552
|
seen[key] = true
|
|
@@ -381,51 +554,57 @@ module Metaclean
|
|
|
381
554
|
end
|
|
382
555
|
end
|
|
383
556
|
|
|
557
|
+
# File.realpath, falling back to the raw path when it can't resolve
|
|
558
|
+
# (broken symlink, permission denied) instead of raising.
|
|
559
|
+
def safe_realpath(path)
|
|
560
|
+
File.realpath(path)
|
|
561
|
+
rescue StandardError
|
|
562
|
+
path
|
|
563
|
+
end
|
|
564
|
+
|
|
384
565
|
def collect_dir(dir, out)
|
|
385
566
|
if @options[:recursive]
|
|
386
|
-
walk_recursive(dir, out
|
|
567
|
+
walk_recursive(dir, out)
|
|
387
568
|
else
|
|
388
|
-
# Non-recursive: just the immediate children of `dir`.
|
|
389
|
-
Dir.glob(
|
|
390
|
-
|
|
569
|
+
# Non-recursive: just the immediate children of `dir`. Use Dir.children,
|
|
570
|
+
# NOT Dir.glob("#{dir}/*") — glob interprets the WHOLE pattern, so a
|
|
571
|
+
# directory name containing glob metacharacters (e.g. "Holiday [2024]")
|
|
572
|
+
# matches nothing and the entire folder is silently skipped. Dir.children
|
|
573
|
+
# surfaces dotfiles too; skip? filters them later, same as walk_recursive.
|
|
574
|
+
Dir.children(dir).each do |entry|
|
|
575
|
+
sub = File.join(dir, entry)
|
|
576
|
+
next if File.symlink?(sub)
|
|
391
577
|
|
|
392
578
|
out << sub if File.file?(sub)
|
|
393
579
|
end
|
|
394
580
|
end
|
|
581
|
+
rescue SystemCallError => e
|
|
582
|
+
# Any Errno (EACCES/ENOENT/ENOTDIR from a dir replaced mid-scan, EIO, …):
|
|
583
|
+
# warn and skip this directory so one bad entry doesn't abort discovery of
|
|
584
|
+
# the rest of the batch.
|
|
585
|
+
Display.warning "Skipping #{dir}: #{e.message}"
|
|
586
|
+
@scan_errors += 1
|
|
395
587
|
end
|
|
396
588
|
|
|
397
|
-
# Manual recursive walker.
|
|
398
|
-
#
|
|
399
|
-
|
|
400
|
-
# eventually points at one of its ancestors.
|
|
401
|
-
def walk_recursive(dir, out, visited)
|
|
402
|
-
real = begin
|
|
403
|
-
File.realpath(dir)
|
|
404
|
-
rescue StandardError
|
|
405
|
-
dir
|
|
406
|
-
end
|
|
407
|
-
return if visited.include?(real)
|
|
408
|
-
|
|
409
|
-
visited << real
|
|
410
|
-
|
|
589
|
+
# Manual recursive walker. Symlinks are always skipped (never followed), so
|
|
590
|
+
# the real directory tree is acyclic and no loop-guard is needed.
|
|
591
|
+
def walk_recursive(dir, out)
|
|
411
592
|
Dir.each_child(dir) do |entry|
|
|
412
593
|
sub = File.join(dir, entry)
|
|
413
|
-
if File.symlink?(sub)
|
|
414
|
-
next unless @options[:follow_symlinks]
|
|
594
|
+
next if File.symlink?(sub)
|
|
415
595
|
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
elsif File.file?(sub)
|
|
419
|
-
out << sub
|
|
420
|
-
end
|
|
421
|
-
elsif File.directory?(sub)
|
|
422
|
-
walk_recursive(sub, out, visited)
|
|
596
|
+
if File.directory?(sub)
|
|
597
|
+
walk_recursive(sub, out)
|
|
423
598
|
elsif File.file?(sub)
|
|
424
599
|
out << sub
|
|
425
600
|
end
|
|
426
601
|
end
|
|
427
|
-
rescue
|
|
602
|
+
rescue SystemCallError => e
|
|
603
|
+
# Any Errno (EACCES/ENOENT/ENOTDIR from a dir replaced mid-scan, EIO, …):
|
|
604
|
+
# warn and skip this directory so one bad entry doesn't abort discovery of
|
|
605
|
+
# the rest of the batch.
|
|
428
606
|
Display.warning "Skipping #{dir}: #{e.message}"
|
|
607
|
+
@scan_errors += 1
|
|
429
608
|
end
|
|
430
609
|
|
|
431
610
|
# Files we never touch when DISCOVERED via directory scanning. This is
|
|
@@ -437,15 +616,11 @@ module Metaclean
|
|
|
437
616
|
base = File.basename(file)
|
|
438
617
|
return true if base.start_with?('.')
|
|
439
618
|
return true if base.end_with?('.bak')
|
|
440
|
-
return true if base =~
|
|
441
|
-
|
|
619
|
+
return true if base =~ Metaclean::CLEAN_OUTPUT_RE
|
|
620
|
+
# Matches our staging temps regardless of the pid/random suffix format.
|
|
621
|
+
return true if base.include?(Metaclean::TMP_MARKER)
|
|
442
622
|
|
|
443
623
|
false
|
|
444
624
|
end
|
|
445
|
-
|
|
446
|
-
def type_allowed?(file)
|
|
447
|
-
ext = File.extname(file).downcase.delete('.')
|
|
448
|
-
@options[:types].include?(ext)
|
|
449
|
-
end
|
|
450
625
|
end
|
|
451
626
|
end
|