metaclean 4.1.0 → 4.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +91 -29
- data/SECURITY.md +59 -0
- data/bin/metaclean +0 -1
- data/lib/metaclean/cli.rb +46 -43
- data/lib/metaclean/committer.rb +130 -0
- data/lib/metaclean/discovery.rb +127 -0
- data/lib/metaclean/display.rb +73 -90
- data/lib/metaclean/exiftool.rb +34 -65
- data/lib/metaclean/ffmpeg.rb +96 -50
- data/lib/metaclean/file_ops.rb +214 -0
- data/lib/metaclean/mat2.rb +28 -77
- data/lib/metaclean/qpdf.rb +46 -46
- data/lib/metaclean/runner.rb +126 -432
- data/lib/metaclean/strategy.rb +75 -100
- data/lib/metaclean/version.rb +1 -5
- data/lib/metaclean.rb +53 -73
- metadata +7 -1
data/lib/metaclean/runner.rb
CHANGED
|
@@ -1,30 +1,22 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
# The orchestrator. Given a list of paths and parsed CLI options, this class:
|
|
4
|
-
#
|
|
5
|
-
# 1. Expands paths into a flat list of files (handling directories and
|
|
6
|
-
# recursion; symlinks are skipped).
|
|
7
|
-
# 2. Asks the user for confirmation (unless --force).
|
|
8
|
-
# 3. For each file, runs the strategy pipeline (mat2 / exiftool / qpdf)
|
|
9
|
-
# using the "atomic write" pattern so a crash never leaves a
|
|
10
|
-
# half-cleaned file.
|
|
11
|
-
# 4. Prints a before/after diff and a final summary.
|
|
12
|
-
|
|
13
|
-
require 'fileutils'
|
|
14
|
-
require 'securerandom'
|
|
15
|
-
|
|
16
3
|
module Metaclean
|
|
17
4
|
class Runner
|
|
18
5
|
def initialize(options)
|
|
19
6
|
@options = options
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
7
|
+
Display.configure(
|
|
8
|
+
quiet: options.fetch(:quiet, false),
|
|
9
|
+
redact_values: options.fetch(:redact_values, !$stdout.tty?)
|
|
10
|
+
)
|
|
23
11
|
@scan_errors = 0
|
|
12
|
+
@path_guards = {}
|
|
13
|
+
@discovery = Discovery.new(recursive: options.fetch(:recursive, false))
|
|
14
|
+
@committer = Committer.new(
|
|
15
|
+
in_place: options.fetch(:in_place, false),
|
|
16
|
+
dry_run: options.fetch(:dry_run, false)
|
|
17
|
+
)
|
|
24
18
|
end
|
|
25
19
|
|
|
26
|
-
# Public entry points: one for `--inspect`, one for the cleaning flow.
|
|
27
|
-
|
|
28
20
|
def inspect_paths(paths)
|
|
29
21
|
files = expand_files(paths)
|
|
30
22
|
if files.empty?
|
|
@@ -34,25 +26,20 @@ module Metaclean
|
|
|
34
26
|
failed = 0
|
|
35
27
|
files.each do |file|
|
|
36
28
|
Display.header "📄 #{file}"
|
|
29
|
+
FileOps.ensure_path_guard!(file, guard_for(file))
|
|
37
30
|
meta = Exiftool.read(file)
|
|
38
31
|
Display.section "Metadata (#{Display.count_embedded(meta)} embedded tags)"
|
|
39
32
|
Display.metadata_table(meta)
|
|
40
33
|
rescue Error, SystemCallError => e
|
|
41
|
-
# One unreadable/odd file shouldn't abort inspecting the rest — mirrors
|
|
42
|
-
# the per-file rescue in the clean batch.
|
|
43
34
|
warn Display.error("#{file}: #{e.message}")
|
|
44
35
|
failed += 1
|
|
45
36
|
end
|
|
46
37
|
|
|
47
|
-
# A file we couldn't read is a non-zero condition: a script using --inspect
|
|
48
|
-
# as a gate must not mistake "couldn't read it" for "no metadata". Discovery
|
|
49
|
-
# errors (missing paths / unreadable dirs) count too.
|
|
50
38
|
exit 1 if failed.positive? || @scan_errors.positive?
|
|
51
39
|
end
|
|
52
40
|
|
|
53
41
|
def clean_paths(paths)
|
|
54
42
|
files = expand_files(paths)
|
|
55
|
-
# See inspect_paths: nothing to act on is a non-zero condition, not success.
|
|
56
43
|
if files.empty?
|
|
57
44
|
Display.warning('No files to process.')
|
|
58
45
|
exit 1
|
|
@@ -60,54 +47,45 @@ module Metaclean
|
|
|
60
47
|
|
|
61
48
|
announce_tools
|
|
62
49
|
|
|
63
|
-
|
|
64
|
-
|
|
50
|
+
if @options[:in_place] && !@options[:dry_run]
|
|
51
|
+
Display.warning 'Each backup is the ORIGINAL with all metadata intact; move or delete it before sharing.'
|
|
52
|
+
end
|
|
53
|
+
|
|
65
54
|
unless @options[:force] || @options[:dry_run]
|
|
66
55
|
action = @options[:in_place] ? 'OVERWRITE' : 'create cleaned copies of'
|
|
67
56
|
puts Display.c("About to #{action} #{files.size} file(s).", :yellow)
|
|
68
|
-
if @options[:in_place]
|
|
69
|
-
# The .bak IS the original — metadata intact. Say so plainly: a user who
|
|
70
|
-
# shares the "cleaned" folder would otherwise leak it via the backup.
|
|
71
|
-
puts Display.c('Each <file>.bak is the ORIGINAL, with all metadata still in it — ' \
|
|
72
|
-
'delete or move the .bak files before sharing the folder.', :yellow)
|
|
73
|
-
end
|
|
74
57
|
print Display.c('Proceed? [y/N] ', :bold)
|
|
75
|
-
ans = $stdin.gets&.strip&.downcase
|
|
76
|
-
# Abort (no/blank/EOF) is a non-zero exit, not silent success — a
|
|
77
|
-
# non-interactive caller must not read "Aborted." as "files were cleaned".
|
|
58
|
+
ans = $stdin.gets&.strip&.downcase
|
|
78
59
|
unless %w[y yes].include?(ans)
|
|
79
60
|
Display.warning('Aborted.')
|
|
80
61
|
exit 1
|
|
81
62
|
end
|
|
82
63
|
end
|
|
83
64
|
|
|
84
|
-
summary = { cleaned: 0, unverified: 0, failed: 0, removed_total: 0, residual_files: 0 }
|
|
65
|
+
summary = { cleaned: 0, unverified: 0, unsupported: 0, failed: 0, removed_total: 0, residual_files: 0 }
|
|
85
66
|
|
|
86
|
-
# index/total let clean_one render "[3/47]" in batch mode.
|
|
87
67
|
files.each_with_index do |file, idx|
|
|
88
|
-
|
|
89
|
-
summary[result[:status]] += 1
|
|
90
|
-
summary[:removed_total] += result[:removed].to_i
|
|
91
|
-
summary[:residual_files] += 1 if result[:residual].to_i.positive?
|
|
68
|
+
tally(summary, clean_one(file, index: idx + 1, total: files.size))
|
|
92
69
|
rescue Error, SystemCallError => e
|
|
93
|
-
# One bad file shouldn't abort the whole batch. SystemCallError
|
|
94
|
-
# (Errno::*: disk full, permission denied, read-only fs) is a SIBLING
|
|
95
|
-
# of our Error, not a subclass, so it must be named explicitly or it
|
|
96
|
-
# would escape this rescue and crash the run with a raw backtrace.
|
|
97
70
|
warn Display.error("#{file}: #{e.message}")
|
|
98
71
|
summary[:failed] += 1
|
|
99
72
|
end
|
|
100
73
|
|
|
101
74
|
print_summary(summary)
|
|
102
75
|
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
76
|
+
if summary[:failed].positive? || summary[:unverified].positive? ||
|
|
77
|
+
summary[:unsupported].positive? || @scan_errors.positive?
|
|
78
|
+
exit 1
|
|
79
|
+
end
|
|
106
80
|
end
|
|
107
81
|
|
|
108
82
|
private
|
|
109
83
|
|
|
110
|
-
|
|
84
|
+
def tally(summary, result)
|
|
85
|
+
summary[result[:status]] += 1
|
|
86
|
+
summary[:removed_total] += result[:removed].to_i if result[:status] == :cleaned
|
|
87
|
+
summary[:residual_files] += 1 if result[:residual].to_i.positive?
|
|
88
|
+
end
|
|
111
89
|
|
|
112
90
|
def announce_tools
|
|
113
91
|
have = []
|
|
@@ -119,118 +97,105 @@ module Metaclean
|
|
|
119
97
|
Display.info '(dry-run — no files will be modified)' if @options[:dry_run]
|
|
120
98
|
end
|
|
121
99
|
|
|
122
|
-
# Cleaning a single file.
|
|
123
100
|
def clean_one(file, index:, total:)
|
|
124
101
|
prefix = total > 1 ? "[#{index}/#{total}] " : ''
|
|
125
102
|
Display.header "#{prefix}📄 #{file}"
|
|
103
|
+
path_guard = guard_for(file)
|
|
104
|
+
FileOps.ensure_path_guard!(file, path_guard)
|
|
126
105
|
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
before = read_metadata(file)
|
|
130
|
-
Display.section "Before (#{Display.count_embedded(before)} embedded tags)"
|
|
131
|
-
Display.metadata_table(before, only_embedded: true)
|
|
132
|
-
|
|
133
|
-
# Ask the strategy module which tools to run for this file type.
|
|
134
|
-
tools = Strategy.tools_for(file)
|
|
135
|
-
# Warn when the stricter tool for a document format won't run: ExifTool
|
|
136
|
-
# alone leaves (and can't fully verify) document-internal metadata.
|
|
137
|
-
if Strategy.mat2_essential?(file) && !tools.include?(:mat2)
|
|
138
|
-
Display.warning 'mat2 will not run for this format — document-internal metadata may remain and cannot be verified.'
|
|
139
|
-
end
|
|
140
|
-
Display.info "Pipeline: #{tools.join(' → ')}"
|
|
141
|
-
|
|
142
|
-
# Atomic write setup:
|
|
143
|
-
# `final_path` = where the cleaned file will end up.
|
|
144
|
-
# `staging` = a temp file we mutate. After all tools succeed, we
|
|
145
|
-
# rename staging → final_path. If anything goes wrong
|
|
146
|
-
# in the middle, we delete staging in the `ensure`
|
|
147
|
-
# block and the original is untouched.
|
|
106
|
+
before = inspect_before(file)
|
|
107
|
+
tools = selected_tools(file)
|
|
148
108
|
final_path = resolve_final_path(file)
|
|
149
109
|
staging = staging_path_for(final_path)
|
|
150
110
|
|
|
151
|
-
tool_results = []
|
|
152
111
|
begin
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
# partial temp if cp is interrupted (Ctrl-C) or fails mid-copy (disk full,
|
|
161
|
-
# read-only fs). cp only ever reads the original, so the source is intact
|
|
162
|
-
# regardless.
|
|
163
|
-
copy_file_exclusive(file, staging)
|
|
164
|
-
tools.each do |tool|
|
|
165
|
-
tool_results << run_tool(tool, staging)
|
|
166
|
-
end
|
|
112
|
+
FileOps.ensure_path_guard!(file, path_guard)
|
|
113
|
+
source_stat = copy_file_exclusive(file, staging)
|
|
114
|
+
tool_results = tools.map { |tool| run_tool(tool, staging) }
|
|
115
|
+
after = inspect_after(staging, before)
|
|
116
|
+
residual = report_residual(after)
|
|
117
|
+
result = finalize_result(tool_results, before, after, residual, file: file)
|
|
118
|
+
return result if discard_result?(result)
|
|
167
119
|
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
120
|
+
commit_cleaned(file, staging, final_path, source_stat, path_guard)
|
|
121
|
+
result
|
|
122
|
+
ensure
|
|
123
|
+
cleanup_staging(staging)
|
|
124
|
+
end
|
|
125
|
+
end
|
|
172
126
|
|
|
173
|
-
|
|
174
|
-
|
|
127
|
+
def inspect_before(file)
|
|
128
|
+
metadata = read_metadata(file)
|
|
129
|
+
Display.section "Before (#{Display.count_embedded(metadata)} embedded tags)"
|
|
130
|
+
Display.metadata_table(metadata, only_embedded: true)
|
|
131
|
+
metadata
|
|
132
|
+
end
|
|
175
133
|
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
134
|
+
def selected_tools(file)
|
|
135
|
+
tools = Strategy.tools_for(file)
|
|
136
|
+
if Strategy.mat2_essential?(file) && !tools.include?(:mat2)
|
|
137
|
+
Display.warning 'mat2 will not run for this format — document-internal metadata may remain and cannot be verified.'
|
|
138
|
+
end
|
|
139
|
+
Display.info "Pipeline: #{tools.join(' → ')}"
|
|
140
|
+
tools
|
|
141
|
+
end
|
|
182
142
|
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
143
|
+
def inspect_after(staging, before)
|
|
144
|
+
metadata = read_metadata(staging)
|
|
145
|
+
Display.section "After (#{Display.count_embedded(metadata)} embedded tags)"
|
|
146
|
+
Display.metadata_table(metadata, only_embedded: true)
|
|
147
|
+
Display.section 'Diff'
|
|
148
|
+
Display.diff(before, metadata)
|
|
149
|
+
metadata
|
|
150
|
+
end
|
|
189
151
|
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
152
|
+
def report_residual(metadata)
|
|
153
|
+
residual = Strategy.privacy_residual(
|
|
154
|
+
metadata,
|
|
155
|
+
allow_icc_metadata: @options.fetch(:allow_icc_metadata, false)
|
|
156
|
+
)
|
|
157
|
+
return residual if residual.empty?
|
|
158
|
+
|
|
159
|
+
Display.warning "Privacy-relevant tags still present (#{residual.size}):"
|
|
160
|
+
unless Display.quiet?
|
|
161
|
+
residual.each do |key, value|
|
|
162
|
+
puts " #{Display.c(key, :yellow)} = #{Display.truncate(Display.visible_value(value), 60)}"
|
|
200
163
|
end
|
|
164
|
+
end
|
|
165
|
+
residual
|
|
166
|
+
end
|
|
201
167
|
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
168
|
+
def discard_result?(result)
|
|
169
|
+
if @options[:dry_run]
|
|
170
|
+
Display.info '(dry-run: temporary copy removed; no output was kept)'
|
|
171
|
+
return true
|
|
172
|
+
end
|
|
173
|
+
return false if result[:status] == :cleaned
|
|
206
174
|
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
175
|
+
reason = case result[:status]
|
|
176
|
+
when :unsupported then 'No cleaning tool supports this format'
|
|
177
|
+
when :unverified then 'The complete cleaning pipeline could not be verified'
|
|
178
|
+
else 'Privacy-relevant metadata survived or all tools failed'
|
|
179
|
+
end
|
|
180
|
+
Display.warning "#{reason} — not writing output."
|
|
181
|
+
true
|
|
182
|
+
end
|
|
212
183
|
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
end
|
|
222
|
-
result
|
|
223
|
-
ensure
|
|
224
|
-
# Last-resort cleanup. If `commit!` already moved the staging file,
|
|
225
|
-
# `File.exist?(staging)` is false and this is a no-op. The path-
|
|
226
|
-
# comparison protects against deleting the final file by accident
|
|
227
|
-
# in the (impossible) case where staging == final.
|
|
228
|
-
File.delete(staging) if File.exist?(staging) && File.expand_path(staging) != File.expand_path(final_path)
|
|
184
|
+
def commit_cleaned(file, staging, final_path, source_stat, path_guard)
|
|
185
|
+
FileOps.ensure_same_file!(file, source_stat)
|
|
186
|
+
FileOps.ensure_path_guard!(file, path_guard)
|
|
187
|
+
if @options[:in_place]
|
|
188
|
+
FileOps.prepare_in_place_commit!(staging, file, source_stat)
|
|
189
|
+
warn_if_hardlinked(file)
|
|
190
|
+
else
|
|
191
|
+
FileOps.restore_metadata(staging, source_stat)
|
|
229
192
|
end
|
|
193
|
+
|
|
194
|
+
committed = commit!(staging, final_path, source_stat: source_stat)
|
|
195
|
+
Display.success "→ #{committed[:path]}"
|
|
196
|
+
Display.warning "Backup with original metadata: #{committed[:backup]}" if committed[:backup]
|
|
230
197
|
end
|
|
231
198
|
|
|
232
|
-
# Warn when an in-place target has more than one hard link: a rename only
|
|
233
|
-
# cleans the named link, leaving the others pointing at the original metadata.
|
|
234
199
|
def warn_if_hardlinked(file)
|
|
235
200
|
nlink = File.stat(file).nlink
|
|
236
201
|
return unless nlink > 1
|
|
@@ -239,15 +204,9 @@ module Metaclean
|
|
|
239
204
|
"the other #{nlink - 1} still contain the original metadata."
|
|
240
205
|
end
|
|
241
206
|
|
|
242
|
-
# Dispatches to the right wrapper module. Returns a small Hash so the
|
|
243
|
-
# caller can summarize tool-by-tool success/failure.
|
|
244
207
|
def run_tool(tool, path)
|
|
245
208
|
case tool
|
|
246
209
|
when :exiftool
|
|
247
|
-
# :unsupported means ExifTool can read but not write this format (a
|
|
248
|
-
# ZIP-based document mat2 owns) — a soft skip, NOT a pipeline failure.
|
|
249
|
-
# Pass the privacy tag names so TIFF/DNG IFD0 tags `-all=` won't drop
|
|
250
|
-
# still get deleted (losslessly).
|
|
251
210
|
if Exiftool.strip!(path, also_delete: Strategy::PRIVACY_TAGS) == :unsupported
|
|
252
211
|
Display.info ' · exiftool (read-only for this format, skipped)'
|
|
253
212
|
{ tool: :exiftool, ok: false, skipped: true, note: :unsupported }
|
|
@@ -257,10 +216,6 @@ module Metaclean
|
|
|
257
216
|
end
|
|
258
217
|
when :mat2
|
|
259
218
|
result = Mat2.strip!(path)
|
|
260
|
-
# mat2 returns either `true` (success) or a symbol indicating a
|
|
261
|
-
# soft skip. `:unsupported` means the tool didn't actually run, so
|
|
262
|
-
# it must not count as a successful pass — otherwise a file can be
|
|
263
|
-
# reported as "Cleaned" while metadata is still embedded.
|
|
264
219
|
case result
|
|
265
220
|
when :unsupported
|
|
266
221
|
Display.info ' · mat2 (unsupported file type, skipped)'
|
|
@@ -277,36 +232,21 @@ module Metaclean
|
|
|
277
232
|
Display.info ' ✓ qpdf'
|
|
278
233
|
{ tool: :qpdf, ok: true }
|
|
279
234
|
when :ffmpeg
|
|
280
|
-
# Matroska remux. A failure raises and is caught below (→ not written).
|
|
281
235
|
Ffmpeg.strip!(path)
|
|
282
236
|
Display.info ' ✓ ffmpeg'
|
|
283
237
|
{ tool: :ffmpeg, ok: true }
|
|
284
238
|
end
|
|
285
239
|
rescue Error, SystemCallError => e
|
|
286
|
-
# One tool failing shouldn't abort the pipeline — we want to keep
|
|
287
|
-
# trying with the others. The `finalize_result` step decides whether
|
|
288
|
-
# the overall file counts as cleaned or failed. `SystemCallError`
|
|
289
|
-
# (Errno::*) covers a tool wrapper's internal FileUtils.mv/File.delete
|
|
290
|
-
# raising on permission/quota/disk errors — without it those would
|
|
291
|
-
# escape and crash the batch.
|
|
292
|
-
# Collapse whitespace and bound the length: some tools (notably mat2) dump a
|
|
293
|
-
# multi-line Python traceback on failure, which would otherwise flood the
|
|
294
|
-
# diff. One readable line is enough — re-run the tool directly to debug.
|
|
295
240
|
msg = Display.truncate(e.message.gsub(/\s+/, ' ').strip, 200)
|
|
296
241
|
Display.warning " ✗ #{tool}: #{msg} — continuing"
|
|
297
242
|
{ tool: tool, ok: false, error: e.message }
|
|
298
243
|
end
|
|
299
244
|
|
|
300
|
-
# :cleaned needs ALL of: a tool genuinely ran, no privacy residual survived,
|
|
301
|
-
# no pipeline tool errored, AND — for a format where mat2 owns coverage
|
|
302
|
-
# ExifTool can't re-read (Office/PDF doc internals) — mat2 actually ran. A
|
|
303
|
-
# tool that errored, or an absent mat2 on a document format, means the
|
|
304
|
-
# pipeline didn't fully complete and the residual check is partly blind, so
|
|
305
|
-
# the result is :unverified, not a confident :cleaned. `file` is needed only
|
|
306
|
-
# for that mat2-coverage check.
|
|
307
245
|
def finalize_result(tool_results, before, after, residual, file: nil)
|
|
308
246
|
removed = removed_embedded_count(before, after)
|
|
309
|
-
status = if !
|
|
247
|
+
status = if !tools_succeeded?(tool_results)
|
|
248
|
+
tool_errored?(tool_results) || residual.any? ? :failed : :unsupported
|
|
249
|
+
elsif !residual.empty?
|
|
310
250
|
:failed
|
|
311
251
|
elsif !tool_errored?(tool_results) && !mat2_coverage_gap?(tool_results, file)
|
|
312
252
|
:cleaned
|
|
@@ -316,41 +256,20 @@ module Metaclean
|
|
|
316
256
|
{ status: status, removed: removed, residual: residual.size }
|
|
317
257
|
end
|
|
318
258
|
|
|
319
|
-
# mat2 is essential for this format (Office/PDF internals ExifTool can't
|
|
320
|
-
# strip or fully re-read) but did NOT actually run and strip — absent,
|
|
321
|
-
# unsupported soft-skip, or errored. The residual check can't confirm the
|
|
322
|
-
# clean, so don't report a confident :cleaned.
|
|
323
259
|
def mat2_coverage_gap?(tool_results, file)
|
|
324
260
|
return false unless file && Strategy.mat2_essential?(file)
|
|
325
261
|
|
|
326
262
|
tool_results.none? { |r| r[:tool] == :mat2 && r[:ok] && !r[:skipped] }
|
|
327
263
|
end
|
|
328
264
|
|
|
329
|
-
# A file is genuinely cleaned only when at least one tool actually ran
|
|
330
|
-
# (not just a mat2 :unsupported soft-skip) AND no privacy-relevant tag
|
|
331
|
-
# survived. Both the commit gate and the final status use this ONE
|
|
332
|
-
# predicate, so they can never disagree — we never write a "_clean" copy
|
|
333
|
-
# (or overwrite an original in place) and then report it :failed. Silently
|
|
334
|
-
# marking a file clean while sensitive metadata is still present is the
|
|
335
|
-
# worst possible outcome for a privacy tool.
|
|
336
|
-
def cleaned?(tool_results, residual)
|
|
337
|
-
tools_succeeded?(tool_results) && residual.empty?
|
|
338
|
-
end
|
|
339
|
-
|
|
340
|
-
# Did at least one tool genuinely run (not a mat2 :unsupported soft-skip)?
|
|
341
265
|
def tools_succeeded?(tool_results)
|
|
342
266
|
tool_results.any? { |r| r[:ok] && !r[:skipped] }
|
|
343
267
|
end
|
|
344
268
|
|
|
345
|
-
# Did a tool that was meant to run error out (not a mat2 :unsupported
|
|
346
|
-
# soft-skip)? Even with an empty residual that means the pipeline didn't
|
|
347
|
-
# fully complete, so the clean can't be reported as a confident :cleaned.
|
|
348
269
|
def tool_errored?(tool_results)
|
|
349
270
|
tool_results.any? { |r| !r[:ok] && !r[:skipped] }
|
|
350
271
|
end
|
|
351
272
|
|
|
352
|
-
# Read metadata for the before/after diff. ensure_tools! guarantees exiftool
|
|
353
|
-
# is present before any run.
|
|
354
273
|
def read_metadata(path)
|
|
355
274
|
Exiftool.read(path)
|
|
356
275
|
end
|
|
@@ -359,142 +278,14 @@ module Metaclean
|
|
|
359
278
|
before.keys.count { |key| Display.embedded_key?(key) && !after.key?(key) }
|
|
360
279
|
end
|
|
361
280
|
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
def
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
# File.rename, NOT FileUtils.mv. staging is in the same dir as final_path
|
|
371
|
-
# (staging_path_for), so this is always an atomic same-fs swap. FileUtils.mv
|
|
372
|
-
# would, on EPERM (e.g. a sticky /tmp file not owned by us) or EXDEV, fall
|
|
373
|
-
# back to a TRUNCATING copy of the original — and an interrupt mid-copy
|
|
374
|
-
# would corrupt the original while the rescue below deleted the only backup.
|
|
375
|
-
# File.rename raises BEFORE touching final_path, so the rescue's
|
|
376
|
-
# "staging still exists ⇒ original intact" assumption always holds.
|
|
377
|
-
File.rename(staging, final_path)
|
|
378
|
-
committed = true
|
|
379
|
-
return final_path
|
|
380
|
-
end
|
|
381
|
-
|
|
382
|
-
link_with_collision_safe_name(staging, final_path)
|
|
383
|
-
rescue SystemCallError, Interrupt
|
|
384
|
-
# The rename failed (disk full, read-only fs, cross-device) OR the user hit
|
|
385
|
-
# Ctrl-C in the window after the .bak was written but before the mv. Either
|
|
386
|
-
# way the original is untouched, so the .bak is a redundant copy of it —
|
|
387
|
-
# remove it instead of leaving a stray file behind, then re-raise (a
|
|
388
|
-
# SystemCallError is reported per-file as failed; an Interrupt propagates to
|
|
389
|
-
# the CLI's exit-130 handler).
|
|
390
|
-
File.delete(backup) if backup && !committed && File.exist?(staging) && File.exist?(backup)
|
|
391
|
-
raise
|
|
392
|
-
end
|
|
393
|
-
|
|
394
|
-
def link_with_collision_safe_name(staging, preferred)
|
|
395
|
-
target = preferred
|
|
396
|
-
loop do
|
|
397
|
-
File.link(staging, target)
|
|
398
|
-
File.delete(staging)
|
|
399
|
-
return target
|
|
400
|
-
rescue Errno::EEXIST
|
|
401
|
-
target = collision_safe(target)
|
|
402
|
-
rescue Errno::EACCES, Errno::EPERM, Errno::ENOTSUP, NotImplementedError
|
|
403
|
-
# The filesystem can't hard-link (Linux returns EPERM; macOS/BSD on
|
|
404
|
-
# exFAT/FAT/SMB returns ENOTSUP/EOPNOTSUPP — same Errno class). Fall back
|
|
405
|
-
# to a plain exclusive copy so removable/network drives still clean.
|
|
406
|
-
target = copy_with_collision_safe_name(staging, target)
|
|
407
|
-
File.delete(staging)
|
|
408
|
-
return target
|
|
409
|
-
end
|
|
410
|
-
end
|
|
411
|
-
|
|
412
|
-
def copy_with_collision_safe_name(src, preferred)
|
|
413
|
-
target = preferred
|
|
414
|
-
loop do
|
|
415
|
-
copy_file_exclusive(src, target, preserve: true)
|
|
416
|
-
return target
|
|
417
|
-
rescue Errno::EEXIST
|
|
418
|
-
target = collision_safe(target)
|
|
419
|
-
end
|
|
420
|
-
end
|
|
421
|
-
|
|
422
|
-
def copy_file_exclusive(src, dest, preserve: false)
|
|
423
|
-
src_stat = File.lstat(src)
|
|
424
|
-
raise Error, "#{src} is a symlink — refusing to copy it" if src_stat.symlink?
|
|
425
|
-
|
|
426
|
-
mode = src_stat.mode & 0o7777
|
|
427
|
-
created = false
|
|
428
|
-
File.open(dest, File::WRONLY | File::CREAT | File::EXCL, mode) do |out|
|
|
429
|
-
created = true
|
|
430
|
-
File.open(src, 'rb') do |input|
|
|
431
|
-
opened = input.stat
|
|
432
|
-
unless opened.dev == src_stat.dev && opened.ino == src_stat.ino
|
|
433
|
-
raise Error, "#{src} changed while opening — refusing to copy it"
|
|
434
|
-
end
|
|
435
|
-
|
|
436
|
-
IO.copy_stream(input, out)
|
|
437
|
-
end
|
|
438
|
-
end
|
|
439
|
-
return unless preserve
|
|
440
|
-
|
|
441
|
-
# Best-effort: the bytes are already fully copied, so a failed mode/timestamp
|
|
442
|
-
# restore (e.g. utime/chmod on a FAT/exFAT mount) must NOT discard an
|
|
443
|
-
# otherwise-complete file by falling into the delete-on-error rescue below.
|
|
444
|
-
begin
|
|
445
|
-
File.chmod(mode, dest)
|
|
446
|
-
File.utime(src_stat.atime, src_stat.mtime, dest)
|
|
447
|
-
rescue SystemCallError
|
|
448
|
-
nil
|
|
449
|
-
end
|
|
450
|
-
rescue StandardError, Interrupt
|
|
451
|
-
File.delete(dest) if created && dest && File.exist?(dest)
|
|
452
|
-
raise
|
|
453
|
-
end
|
|
454
|
-
|
|
455
|
-
def resolve_final_path(file)
|
|
456
|
-
return file if @options[:in_place]
|
|
457
|
-
|
|
458
|
-
# Default: write `<name>_clean.<ext>` next to the original. If it
|
|
459
|
-
# already exists, `collision_safe` appends `_1`, `_2`, …
|
|
460
|
-
collision_safe(build_clean_path(file))
|
|
461
|
-
end
|
|
462
|
-
|
|
463
|
-
def build_clean_path(file)
|
|
464
|
-
ext = File.extname(file)
|
|
465
|
-
base = File.basename(file, ext)
|
|
466
|
-
File.join(File.dirname(file), "#{base}#{Metaclean::CLEAN_SUFFIX}#{ext}")
|
|
467
|
-
end
|
|
468
|
-
|
|
469
|
-
# Staging path lives in the same directory as the destination so that
|
|
470
|
-
# `File.rename`/`FileUtils.mv` is an atomic same-filesystem operation.
|
|
471
|
-
# PID + random number prevent collisions between simultaneous runs.
|
|
472
|
-
# The original extension is preserved as the LAST segment so tools like
|
|
473
|
-
# mat2 — which dispatch on file extension — see the real type.
|
|
474
|
-
def staging_path_for(final_path)
|
|
475
|
-
dir = File.dirname(final_path)
|
|
476
|
-
ext = File.extname(final_path)
|
|
477
|
-
# SecureRandom (not rand) makes the staging name unpredictable, so a
|
|
478
|
-
# hostile process in the same directory can't pre-create it as a symlink
|
|
479
|
-
# that `FileUtils.cp` would copy the (still-sensitive) original through.
|
|
480
|
-
File.join(dir, "#{Metaclean::TMP_MARKER}#{Process.pid}.#{SecureRandom.hex(8)}#{ext}")
|
|
481
|
-
end
|
|
482
|
-
|
|
483
|
-
# If `path` is taken, try `path_1`, `path_2`, … until one is free.
|
|
484
|
-
def collision_safe(path)
|
|
485
|
-
return path unless File.exist?(path)
|
|
486
|
-
|
|
487
|
-
ext = File.extname(path)
|
|
488
|
-
base = File.basename(path, ext)
|
|
489
|
-
dir = File.dirname(path)
|
|
490
|
-
i = 1
|
|
491
|
-
loop do
|
|
492
|
-
candidate = File.join(dir, "#{base}_#{i}#{ext}")
|
|
493
|
-
return candidate unless File.exist?(candidate)
|
|
494
|
-
|
|
495
|
-
i += 1
|
|
496
|
-
end
|
|
497
|
-
end
|
|
281
|
+
def commit!(...) = @committer.commit!(...)
|
|
282
|
+
def resolve_final_path(...) = @committer.resolve_final_path(...)
|
|
283
|
+
def staging_path_for(...) = @committer.staging_path_for(...)
|
|
284
|
+
def cleanup_staging(...) = @committer.cleanup_staging(...)
|
|
285
|
+
def build_clean_path(...) = @committer.build_clean_path(...)
|
|
286
|
+
def collision_safe(...) = @committer.collision_safe(...)
|
|
287
|
+
def copy_file_exclusive(...) = @committer.copy_file_exclusive(...)
|
|
288
|
+
def link_with_collision_safe_name(...) = @committer.link_with_collision_safe_name(...)
|
|
498
289
|
|
|
499
290
|
def print_summary(summary)
|
|
500
291
|
Display.header 'Summary'
|
|
@@ -502,6 +293,9 @@ module Metaclean
|
|
|
502
293
|
if summary[:unverified].positive?
|
|
503
294
|
Display.warning "Unverified (clean could not be confirmed): #{summary[:unverified]} file(s)"
|
|
504
295
|
end
|
|
296
|
+
if summary[:unsupported].positive?
|
|
297
|
+
Display.warning "Unsupported (not cleaned): #{summary[:unsupported]} file(s)"
|
|
298
|
+
end
|
|
505
299
|
puts Display.error("Failed: #{summary[:failed]}") if summary[:failed].positive?
|
|
506
300
|
Display.info "Total embedded tags removed: #{summary[:removed_total]}"
|
|
507
301
|
if summary[:residual_files].positive?
|
|
@@ -512,115 +306,15 @@ module Metaclean
|
|
|
512
306
|
end
|
|
513
307
|
end
|
|
514
308
|
|
|
515
|
-
# File discovery — turning the user's paths into a flat list.
|
|
516
|
-
|
|
517
309
|
def expand_files(paths)
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
# that points outside the intended scope.
|
|
523
|
-
if File.symlink?(p)
|
|
524
|
-
Display.warning "Skipping symlink: #{p}"
|
|
525
|
-
next
|
|
526
|
-
end
|
|
527
|
-
if File.directory?(p)
|
|
528
|
-
collect_dir(p, discovered)
|
|
529
|
-
elsif File.file?(p)
|
|
530
|
-
# Explicit file argument — never apply skip?, the user asked for
|
|
531
|
-
# this exact path. (Skip filters exist to avoid re-cleaning our
|
|
532
|
-
# own outputs during recursion, not to override the CLI.)
|
|
533
|
-
explicit << p
|
|
534
|
-
else
|
|
535
|
-
Display.warning "Not found: #{p}"
|
|
536
|
-
@scan_errors += 1
|
|
537
|
-
end
|
|
538
|
-
end
|
|
539
|
-
discovered.reject! { |f| skip?(f) }
|
|
540
|
-
dedupe_by_realpath(explicit + discovered)
|
|
541
|
-
end
|
|
542
|
-
|
|
543
|
-
# Same file via two different paths (or via symlink + direct path) should
|
|
544
|
-
# be cleaned once. Comparing by realpath catches both cases. If realpath
|
|
545
|
-
# raises (broken symlink, permission denied), fall back to the raw path.
|
|
546
|
-
def dedupe_by_realpath(paths)
|
|
547
|
-
seen = {}
|
|
548
|
-
paths.each_with_object([]) do |p, acc|
|
|
549
|
-
key = safe_realpath(p)
|
|
550
|
-
next if seen[key]
|
|
551
|
-
|
|
552
|
-
seen[key] = true
|
|
553
|
-
acc << p
|
|
554
|
-
end
|
|
555
|
-
end
|
|
556
|
-
|
|
557
|
-
# File.realpath, falling back to the raw path when it can't resolve
|
|
558
|
-
# (broken symlink, permission denied) instead of raising.
|
|
559
|
-
def safe_realpath(path)
|
|
560
|
-
File.realpath(path)
|
|
561
|
-
rescue StandardError
|
|
562
|
-
path
|
|
563
|
-
end
|
|
564
|
-
|
|
565
|
-
def collect_dir(dir, out)
|
|
566
|
-
if @options[:recursive]
|
|
567
|
-
walk_recursive(dir, out)
|
|
568
|
-
else
|
|
569
|
-
# Non-recursive: just the immediate children of `dir`. Use Dir.children,
|
|
570
|
-
# NOT Dir.glob("#{dir}/*") — glob interprets the WHOLE pattern, so a
|
|
571
|
-
# directory name containing glob metacharacters (e.g. "Holiday [2024]")
|
|
572
|
-
# matches nothing and the entire folder is silently skipped. Dir.children
|
|
573
|
-
# surfaces dotfiles too; skip? filters them later, same as walk_recursive.
|
|
574
|
-
Dir.children(dir).each do |entry|
|
|
575
|
-
sub = File.join(dir, entry)
|
|
576
|
-
next if File.symlink?(sub)
|
|
577
|
-
|
|
578
|
-
out << sub if File.file?(sub)
|
|
579
|
-
end
|
|
580
|
-
end
|
|
581
|
-
rescue SystemCallError => e
|
|
582
|
-
# Any Errno (EACCES/ENOENT/ENOTDIR from a dir replaced mid-scan, EIO, …):
|
|
583
|
-
# warn and skip this directory so one bad entry doesn't abort discovery of
|
|
584
|
-
# the rest of the batch.
|
|
585
|
-
Display.warning "Skipping #{dir}: #{e.message}"
|
|
586
|
-
@scan_errors += 1
|
|
587
|
-
end
|
|
588
|
-
|
|
589
|
-
# Manual recursive walker. Symlinks are always skipped (never followed), so
|
|
590
|
-
# the real directory tree is acyclic and no loop-guard is needed.
|
|
591
|
-
def walk_recursive(dir, out)
|
|
592
|
-
Dir.each_child(dir) do |entry|
|
|
593
|
-
sub = File.join(dir, entry)
|
|
594
|
-
next if File.symlink?(sub)
|
|
595
|
-
|
|
596
|
-
if File.directory?(sub)
|
|
597
|
-
walk_recursive(sub, out)
|
|
598
|
-
elsif File.file?(sub)
|
|
599
|
-
out << sub
|
|
600
|
-
end
|
|
601
|
-
end
|
|
602
|
-
rescue SystemCallError => e
|
|
603
|
-
# Any Errno (EACCES/ENOENT/ENOTDIR from a dir replaced mid-scan, EIO, …):
|
|
604
|
-
# warn and skip this directory so one bad entry doesn't abort discovery of
|
|
605
|
-
# the rest of the batch.
|
|
606
|
-
Display.warning "Skipping #{dir}: #{e.message}"
|
|
607
|
-
@scan_errors += 1
|
|
310
|
+
files = @discovery.expand(paths)
|
|
311
|
+
@scan_errors = @discovery.scan_errors
|
|
312
|
+
@path_guards = @discovery.guards
|
|
313
|
+
files
|
|
608
314
|
end
|
|
609
315
|
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
# `metaclean .hidden.jpg`, they meant it. Hidden files (dot-prefixed)
|
|
613
|
-
# might be system metadata; .bak/_clean/.metaclean.tmp.* are our own
|
|
614
|
-
# outputs, so skipping them prevents loops on re-runs.
|
|
615
|
-
def skip?(file)
|
|
616
|
-
base = File.basename(file)
|
|
617
|
-
return true if base.start_with?('.')
|
|
618
|
-
return true if base.end_with?('.bak')
|
|
619
|
-
return true if base =~ Metaclean::CLEAN_OUTPUT_RE
|
|
620
|
-
# Matches our staging temps regardless of the pid/random suffix format.
|
|
621
|
-
return true if base.include?(Metaclean::TMP_MARKER)
|
|
622
|
-
|
|
623
|
-
false
|
|
316
|
+
def guard_for(file)
|
|
317
|
+
@path_guards[file] || FileOps.path_guard!(file)
|
|
624
318
|
end
|
|
625
319
|
end
|
|
626
320
|
end
|