metaclean 4.1.0 → 4.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +95 -30
- data/SECURITY.md +59 -0
- data/bin/metaclean +0 -1
- data/lib/metaclean/cli.rb +48 -45
- data/lib/metaclean/committer.rb +130 -0
- data/lib/metaclean/discovery.rb +127 -0
- data/lib/metaclean/display.rb +81 -95
- data/lib/metaclean/exiftool.rb +34 -65
- data/lib/metaclean/ffmpeg.rb +96 -50
- data/lib/metaclean/file_ops.rb +214 -0
- data/lib/metaclean/mat2.rb +28 -77
- data/lib/metaclean/qpdf.rb +46 -46
- data/lib/metaclean/runner.rb +130 -433
- data/lib/metaclean/strategy.rb +75 -100
- data/lib/metaclean/version.rb +1 -5
- data/lib/metaclean.rb +53 -73
- metadata +8 -1
data/lib/metaclean/ffmpeg.rb
CHANGED
|
@@ -1,34 +1,30 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
#
|
|
5
|
-
# ffmpeg is used for ONE job the other tools can't do: the Matroska containers
|
|
6
|
-
# (mkv/webm). ExifTool is read-only for Matroska, and mat2 has no Matroska
|
|
7
|
-
# parser, so without ffmpeg those formats can't be cleaned at all. ffmpeg
|
|
8
|
-
# rewrites the container while copying every stream verbatim (`-c copy`) — no
|
|
9
|
-
# re-encode, so the audio/video is bit-identical and only the metadata is gone.
|
|
10
|
-
#
|
|
11
|
-
# Like mat2, ffmpeg can't edit in place: it muxes to a new file. We write to a
|
|
12
|
-
# SecureRandom-named sibling and move it back over the source on success.
|
|
13
|
-
|
|
14
|
-
require 'open3'
|
|
15
|
-
require 'fileutils'
|
|
16
|
-
require 'securerandom'
|
|
3
|
+
require 'json'
|
|
17
4
|
|
|
18
5
|
module Metaclean
|
|
19
6
|
module Ffmpeg
|
|
7
|
+
NESTED_IMAGE_CODECS = %w[bmp gif mjpeg mjpegb png tiff webp].freeze
|
|
8
|
+
|
|
20
9
|
module_function
|
|
21
10
|
|
|
22
|
-
# Memoized PATH check (same pattern as the other wrappers).
|
|
23
11
|
def available?
|
|
24
12
|
return @available if defined?(@available)
|
|
25
13
|
|
|
26
|
-
out, _err, status =
|
|
27
|
-
|
|
28
|
-
|
|
14
|
+
out, _err, status = Metaclean.capture3(
|
|
15
|
+
'ffmpeg', '-version',
|
|
16
|
+
timeout: Metaclean::PROBE_TIMEOUT,
|
|
17
|
+
max_output: Metaclean::PROBE_MAX_OUTPUT_BYTES
|
|
18
|
+
)
|
|
19
|
+
_probe_out, _probe_err, probe_status = Metaclean.capture3(
|
|
20
|
+
'ffprobe', '-version',
|
|
21
|
+
timeout: Metaclean::PROBE_TIMEOUT,
|
|
22
|
+
max_output: Metaclean::PROBE_MAX_OUTPUT_BYTES
|
|
23
|
+
)
|
|
24
|
+
@available = status.success? && probe_status.success?
|
|
29
25
|
@version = @available ? out.lines.first.to_s.split[2] : nil
|
|
30
26
|
@available
|
|
31
|
-
rescue Errno::ENOENT
|
|
27
|
+
rescue Errno::ENOENT, Error
|
|
32
28
|
@version = nil
|
|
33
29
|
@available = false
|
|
34
30
|
end
|
|
@@ -37,44 +33,94 @@ module Metaclean
|
|
|
37
33
|
available? ? @version : nil
|
|
38
34
|
end
|
|
39
35
|
|
|
40
|
-
# Strips all metadata from `path` in place, losslessly. Returns true on
|
|
41
|
-
# success, raises Metaclean::Error on failure.
|
|
42
|
-
#
|
|
43
|
-
# -map 0 keep every stream (video, audio, subtitles)
|
|
44
|
-
# -map_metadata -1 drop global/container metadata
|
|
45
|
-
# -map_chapters -1 drop chapter markers (they can carry titles)
|
|
46
|
-
# -c copy remux without re-encoding — bit-identical streams
|
|
47
36
|
def strip!(path)
|
|
48
37
|
raise Error, 'ffmpeg not available' unless available?
|
|
49
38
|
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
39
|
+
source = FileOps.regular_stat!(path)
|
|
40
|
+
structure = probe_structure(path)
|
|
41
|
+
reject_unverifiable_streams!(structure[:streams])
|
|
42
|
+
FileOps.with_private_workspace(path, 'ffmpeg') do |workspace|
|
|
43
|
+
tmp = File.join(workspace, "output#{File.extname(path)}")
|
|
44
|
+
args = [
|
|
45
|
+
'ffmpeg', '-y', '-v', 'error', '-nostdin', '-i', file_url(path),
|
|
46
|
+
'-map', '0', '-map_metadata', '-1',
|
|
47
|
+
'-map_chapters', structure[:chapters].positive? ? '0' : '-1'
|
|
48
|
+
]
|
|
49
|
+
args.concat(['-metadata:c', 'title=']) if structure[:chapters].positive?
|
|
50
|
+
append_safe_stream_languages(args, structure[:streams])
|
|
51
|
+
args.concat(['-c', 'copy', file_url(tmp)])
|
|
52
|
+
|
|
53
|
+
_out, err, status = Metaclean.capture3(*args)
|
|
54
|
+
raise Error, "ffmpeg failed: #{err.strip}" unless status.success? && File.exist?(tmp)
|
|
53
55
|
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
56
|
+
FileOps.validate_output!(tmp)
|
|
57
|
+
ensure_structure_preserved!(structure, probe_structure(tmp))
|
|
58
|
+
FileOps.ensure_same_file!(path, source)
|
|
59
|
+
FileOps.restore_metadata(tmp, source)
|
|
60
|
+
File.rename(tmp, path)
|
|
61
|
+
true
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def probe_structure(path)
|
|
66
|
+
out, err, status = Metaclean.capture3(
|
|
67
|
+
'ffprobe', '-v', 'error', '-show_streams', '-show_chapters',
|
|
68
|
+
'-of', 'json', file_url(path)
|
|
58
69
|
)
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
70
|
+
raise Error, "ffprobe failed: #{err.strip}" unless status.success?
|
|
71
|
+
|
|
72
|
+
data = JSON.parse(out)
|
|
73
|
+
streams = data.fetch('streams', [])
|
|
74
|
+
chapters = data.fetch('chapters', [])
|
|
75
|
+
raise TypeError unless streams.is_a?(Array) && chapters.is_a?(Array)
|
|
76
|
+
|
|
77
|
+
{ streams: streams, chapters: chapters.size }
|
|
78
|
+
rescue JSON::ParserError, KeyError, TypeError => e
|
|
79
|
+
raise Error, "Could not parse ffprobe output: #{e.message}"
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
def reject_unverifiable_streams!(streams)
|
|
83
|
+
video_streams = streams.count { |stream| stream['codec_type'] == 'video' }
|
|
84
|
+
nested = streams.any? do |stream|
|
|
85
|
+
tags = stream['tags'].is_a?(Hash) ? stream['tags'] : {}
|
|
86
|
+
disposition = stream['disposition'].is_a?(Hash) ? stream['disposition'] : {}
|
|
87
|
+
nested_image = stream['codec_type'] == 'video' &&
|
|
88
|
+
NESTED_IMAGE_CODECS.include?(stream['codec_name']) &&
|
|
89
|
+
(video_streams > 1 || streams.any? { |candidate| candidate['codec_type'] == 'audio' })
|
|
90
|
+
stream['codec_type'] == 'attachment' ||
|
|
91
|
+
nested_image ||
|
|
92
|
+
disposition['attached_pic'].to_i == 1 ||
|
|
93
|
+
!tags['filename'].to_s.empty? ||
|
|
94
|
+
!tags['mimetype'].to_s.empty?
|
|
95
|
+
end
|
|
96
|
+
return unless nested
|
|
97
|
+
|
|
98
|
+
raise Error, 'Matroska contains attachments or cover art whose nested metadata cannot be verified'
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
def ensure_structure_preserved!(before, after)
|
|
102
|
+
before_signatures = stream_signatures(before[:streams])
|
|
103
|
+
after_signatures = stream_signatures(after[:streams])
|
|
104
|
+
unless before_signatures == after_signatures && before[:chapters] == after[:chapters]
|
|
105
|
+
raise Error, 'ffmpeg output changed the stream or chapter structure'
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
def stream_signatures(streams)
|
|
110
|
+
streams.map { |stream| [stream['codec_type'], stream['codec_name']] }
|
|
69
111
|
end
|
|
70
112
|
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
113
|
+
def append_safe_stream_languages(args, streams)
|
|
114
|
+
streams.each do |stream|
|
|
115
|
+
index = Integer(stream.fetch('index'))
|
|
116
|
+
tags = stream['tags'].is_a?(Hash) ? stream['tags'] : {}
|
|
117
|
+
language = tags['language'].to_s.downcase
|
|
118
|
+
if language.match?(/\A[a-z]{2,3}(?:-[a-z0-9]{2,8})*\z/)
|
|
119
|
+
args.concat(["-metadata:s:#{index}", "language=#{language}"])
|
|
120
|
+
end
|
|
121
|
+
end
|
|
122
|
+
rescue KeyError, TypeError, ArgumentError => e
|
|
123
|
+
raise Error, "Unexpected ffprobe stream data: #{e.message}"
|
|
78
124
|
end
|
|
79
125
|
|
|
80
126
|
def file_url(path)
|
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'fileutils'
|
|
4
|
+
require 'tmpdir'
|
|
5
|
+
|
|
6
|
+
module Metaclean
|
|
7
|
+
module FileOps
|
|
8
|
+
module_function
|
|
9
|
+
|
|
10
|
+
def path_guard!(path)
|
|
11
|
+
components = component_identities(path)
|
|
12
|
+
realpath = File.realpath(path)
|
|
13
|
+
raise Error, "#{path} changed while its path was being verified" unless components == component_identities(path)
|
|
14
|
+
|
|
15
|
+
{ realpath: realpath, components: components }
|
|
16
|
+
rescue SystemCallError => e
|
|
17
|
+
raise Error, "#{path} cannot be resolved safely: #{e.message}"
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def ensure_path_guard!(path, expected)
|
|
21
|
+
current = path_guard!(path)
|
|
22
|
+
return if current == expected
|
|
23
|
+
|
|
24
|
+
raise Error, "#{path} or one of its parent directories changed during cleaning"
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def component_identities(path)
|
|
28
|
+
components = path_components(path)
|
|
29
|
+
components.map.with_index do |component, index|
|
|
30
|
+
stat = File.lstat(component)
|
|
31
|
+
if stat.symlink? && (index == components.length - 1 || !trusted_system_symlink?(component, stat))
|
|
32
|
+
raise Error, "#{path} contains a symlink component (#{component}) — refusing to use it"
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
[component, stat.dev, stat.ino]
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def path_components(path)
|
|
40
|
+
expanded = File.expand_path(path)
|
|
41
|
+
current = File::SEPARATOR
|
|
42
|
+
expanded.split(File::SEPARATOR).reject(&:empty?).map do |part|
|
|
43
|
+
current = File.join(current, part)
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def trusted_system_symlink?(path, stat)
|
|
48
|
+
parent = File.lstat(File.dirname(path))
|
|
49
|
+
stat.uid.zero? && parent.uid.zero? && parent.mode.nobits?(0o022)
|
|
50
|
+
rescue SystemCallError
|
|
51
|
+
false
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def regular_stat!(path)
|
|
55
|
+
stat = File.lstat(path)
|
|
56
|
+
raise Error, "#{path} is a symlink — refusing to use it" if stat.symlink?
|
|
57
|
+
raise Error, "#{path} is not a regular file — refusing to use it" unless stat.file?
|
|
58
|
+
|
|
59
|
+
stat
|
|
60
|
+
rescue SystemCallError => e
|
|
61
|
+
raise Error, "#{path} cannot be used: #{e.message}"
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def ensure_same_file!(path, expected)
|
|
65
|
+
current = regular_stat!(path)
|
|
66
|
+
return current if same_file?(current, expected)
|
|
67
|
+
|
|
68
|
+
raise Error, "#{path} changed during cleaning — refusing to commit"
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def same_file?(left, right)
|
|
72
|
+
left.dev == right.dev && left.ino == right.ino &&
|
|
73
|
+
left.size == right.size && left.mtime == right.mtime && left.ctime == right.ctime
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def same_identity?(left, right)
|
|
77
|
+
left.dev == right.dev && left.ino == right.ino &&
|
|
78
|
+
left.size == right.size && left.mtime == right.mtime
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
def copy_exclusive(src, dest, preserve: false, expected: nil)
|
|
82
|
+
source = regular_stat!(src)
|
|
83
|
+
if expected && !same_file?(source, expected)
|
|
84
|
+
raise Error, "#{src} changed before it could be copied"
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
mode = source.mode & 0o7777
|
|
88
|
+
created = false
|
|
89
|
+
read_flags = File::RDONLY
|
|
90
|
+
read_flags |= File::NOFOLLOW if defined?(File::NOFOLLOW)
|
|
91
|
+
|
|
92
|
+
File.open(dest, File::WRONLY | File::CREAT | File::EXCL, mode) do |out|
|
|
93
|
+
created = true
|
|
94
|
+
File.open(src, read_flags) do |input|
|
|
95
|
+
opened = input.stat
|
|
96
|
+
raise Error, "#{src} changed while opening — refusing to copy it" unless same_file?(opened, source)
|
|
97
|
+
|
|
98
|
+
size = opened.size
|
|
99
|
+
mtime = opened.mtime
|
|
100
|
+
ctime = opened.ctime
|
|
101
|
+
IO.copy_stream(input, out)
|
|
102
|
+
finished = input.stat
|
|
103
|
+
unless finished.size == size && finished.mtime == mtime && finished.ctime == ctime
|
|
104
|
+
raise Error, "#{src} changed while it was being copied"
|
|
105
|
+
end
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
restore_metadata(dest, source) if preserve
|
|
110
|
+
source
|
|
111
|
+
rescue StandardError, Interrupt
|
|
112
|
+
File.delete(dest) if created && dest && File.exist?(dest)
|
|
113
|
+
raise
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
def restore_metadata(path, source_stat, strict: false)
|
|
117
|
+
File.chmod(source_stat.mode & 0o7777, path)
|
|
118
|
+
File.utime(source_stat.atime, source_stat.mtime, path)
|
|
119
|
+
rescue SystemCallError => e
|
|
120
|
+
raise Error, "Could not restore file permissions/timestamps: #{e.message}" if strict
|
|
121
|
+
|
|
122
|
+
nil
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
def prepare_in_place_commit!(cleaned, source, expected)
|
|
126
|
+
ensure_same_file!(source, expected)
|
|
127
|
+
prepared = "#{cleaned}.metadata"
|
|
128
|
+
native_metadata_copy!(source, prepared)
|
|
129
|
+
copied = regular_stat!(prepared)
|
|
130
|
+
unless copied.uid == expected.uid && copied.gid == expected.gid
|
|
131
|
+
raise Error, 'Could not preserve source ownership for in-place output'
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
clean_stat = regular_stat!(cleaned)
|
|
135
|
+
File.chmod((expected.mode & 0o7777) | 0o200, prepared)
|
|
136
|
+
write_flags = File::WRONLY | File::TRUNC
|
|
137
|
+
write_flags |= File::NOFOLLOW if defined?(File::NOFOLLOW)
|
|
138
|
+
File.open(prepared, write_flags) do |out|
|
|
139
|
+
File.open(cleaned, File::RDONLY) { |input| IO.copy_stream(input, out) }
|
|
140
|
+
end
|
|
141
|
+
ensure_same_file!(cleaned, clean_stat)
|
|
142
|
+
ensure_same_file!(source, expected)
|
|
143
|
+
restore_metadata(prepared, expected, strict: true)
|
|
144
|
+
validate_output!(prepared)
|
|
145
|
+
File.rename(prepared, cleaned)
|
|
146
|
+
cleaned
|
|
147
|
+
rescue StandardError, Interrupt
|
|
148
|
+
File.delete(prepared) if prepared && lexist?(prepared)
|
|
149
|
+
raise
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
def native_metadata_copy!(source, destination)
|
|
153
|
+
cp = find_executable('cp')
|
|
154
|
+
raise Error, 'cp is required to preserve in-place filesystem metadata' unless cp
|
|
155
|
+
|
|
156
|
+
args = if RUBY_PLATFORM.include?('darwin')
|
|
157
|
+
[cp, '-p', '--', source, destination]
|
|
158
|
+
elsif RUBY_PLATFORM.include?('linux')
|
|
159
|
+
[cp, '--preserve=all', '--', source, destination]
|
|
160
|
+
else
|
|
161
|
+
raise Error, "In-place metadata preservation is unsupported on #{RUBY_PLATFORM}"
|
|
162
|
+
end
|
|
163
|
+
_out, err, status = Metaclean.capture3(*args)
|
|
164
|
+
return if status.success?
|
|
165
|
+
|
|
166
|
+
raise Error, "Could not preserve filesystem metadata for in-place output: #{err.strip}"
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
def find_executable(name)
|
|
170
|
+
ENV.fetch('PATH', '').split(File::PATH_SEPARATOR).each do |directory|
|
|
171
|
+
next if directory.empty?
|
|
172
|
+
|
|
173
|
+
candidate = File.join(directory, name)
|
|
174
|
+
return candidate if File.file?(candidate) && File.executable?(candidate)
|
|
175
|
+
end
|
|
176
|
+
nil
|
|
177
|
+
end
|
|
178
|
+
|
|
179
|
+
def metadata_copy_supported?
|
|
180
|
+
(RUBY_PLATFORM.include?('darwin') || RUBY_PLATFORM.include?('linux')) &&
|
|
181
|
+
!find_executable('cp').nil?
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
def lexist?(path)
|
|
185
|
+
File.symlink?(path) || File.exist?(path)
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
def validate_output!(path)
|
|
189
|
+
stat = regular_stat!(path)
|
|
190
|
+
raise Error, "#{path} is empty — refusing to use tool output" unless stat.size.positive?
|
|
191
|
+
|
|
192
|
+
stat
|
|
193
|
+
end
|
|
194
|
+
|
|
195
|
+
def with_private_workspace(path, label)
|
|
196
|
+
parent = File.dirname(File.expand_path(path))
|
|
197
|
+
prefix = "#{TMP_MARKER}#{label}.#{Process.pid}."
|
|
198
|
+
Dir.mktmpdir(prefix, parent) do |dir|
|
|
199
|
+
secure_workspace!(dir)
|
|
200
|
+
yield dir
|
|
201
|
+
end
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
def secure_workspace!(directory)
|
|
205
|
+
File.chmod(0o700, directory)
|
|
206
|
+
mode = File.stat(directory).mode & 0o777
|
|
207
|
+
return true if mode.nobits?(0o077)
|
|
208
|
+
|
|
209
|
+
raise Error, "#{directory} cannot enforce a private workspace (mode #{mode.to_s(8)})"
|
|
210
|
+
rescue SystemCallError => e
|
|
211
|
+
raise Error, "#{directory} cannot enforce a private workspace: #{e.message}"
|
|
212
|
+
end
|
|
213
|
+
end
|
|
214
|
+
end
|
data/lib/metaclean/mat2.rb
CHANGED
|
@@ -1,55 +1,31 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
# Wrapper around the external `mat2` (Metadata Anonymisation Toolkit 2).
|
|
4
|
-
#
|
|
5
|
-
# mat2 is stricter than ExifTool on certain formats (DOCX/PDF/PNG): instead
|
|
6
|
-
# of blacklisting known tags, it rebuilds the file from scratch keeping only
|
|
7
|
-
# the bytes it understands. The trade-off is that mat2 supports fewer formats.
|
|
8
|
-
#
|
|
9
|
-
# mat2's CLI quirk: it does NOT overwrite the original. It writes a new file
|
|
10
|
-
# named `<name>.cleaned.<ext>` next to it. We adapt by renaming that result
|
|
11
|
-
# back over the source after a successful run.
|
|
12
|
-
|
|
13
|
-
require 'open3'
|
|
14
|
-
require 'fileutils'
|
|
15
|
-
|
|
16
3
|
module Metaclean
|
|
17
4
|
module Mat2
|
|
18
|
-
# File extensions we know mat2 can handle. Keep this list conservative —
|
|
19
|
-
# if mat2 doesn't actually support an extension, the call will fail
|
|
20
|
-
# gracefully via UNSUPPORTED_RE below, but we'd rather not even try.
|
|
21
|
-
# Deliberately ABSENT: Matroska (mkv/webm) — mat2 has no parser for it; ffmpeg
|
|
22
|
-
# owns those (Strategy::FFMPEG_FORMATS). QuickTime/MP4-audio (mov/m4a) — mat2
|
|
23
|
-
# can't write them and ExifTool already cleans them, so listing them only
|
|
24
|
-
# caused a wasted mat2 spawn that always soft-skipped. WMV (ASF) IS here on
|
|
25
|
-
# purpose: mat2 CAN write it but ExifTool can't, so mat2 is the only tool that
|
|
26
|
-
# cleans .wmv — dropping it would make every .wmv permanently :failed.
|
|
27
5
|
SUPPORTED_EXTS = %w[
|
|
28
|
-
pdf
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
6
|
+
epub pdf odc odf odg odi odp ods odt pptx xlsx docx torrent ncx tar
|
|
7
|
+
xht xhtml zip flac m3a mp2a mp3 mp2 m2a mpga oga spx ogg opus
|
|
8
|
+
aifc aiff aif wav bmp gif heic jpg jpe jpeg png svgz svg tiff tif
|
|
9
|
+
webp ppm css htm html text txt in def list log conf
|
|
10
|
+
mp4 mp4v mpg4 m4v wmv avi
|
|
33
11
|
].freeze
|
|
34
12
|
|
|
35
|
-
# Matches the messages mat2 prints when it can't handle a file — lets us
|
|
36
|
-
# distinguish a soft skip from a real error.
|
|
37
13
|
UNSUPPORTED_RE = /(not supported|isn't supported|cannot be cleaned|unsupported file)/i.freeze
|
|
38
14
|
|
|
39
15
|
module_function
|
|
40
16
|
|
|
41
|
-
# Memoized PATH check (same pattern as Exiftool.available?).
|
|
42
17
|
def available?
|
|
43
18
|
return @available if defined?(@available)
|
|
44
19
|
|
|
45
|
-
out, _err, status =
|
|
20
|
+
out, _err, status = Metaclean.capture3(
|
|
21
|
+
'mat2', '--version',
|
|
22
|
+
timeout: Metaclean::PROBE_TIMEOUT,
|
|
23
|
+
max_output: Metaclean::PROBE_MAX_OUTPUT_BYTES
|
|
24
|
+
)
|
|
46
25
|
@available = status.success?
|
|
47
|
-
# `mat2 --version` prints "mat2 0.14.0" — `.split.last` grabs the
|
|
48
|
-
# version number regardless of whatever prefix appears. Captured here
|
|
49
|
-
# so `version` reuses it instead of re-spawning the binary.
|
|
50
26
|
@version = @available ? out.strip.split.last : nil
|
|
51
27
|
@available
|
|
52
|
-
rescue Errno::ENOENT
|
|
28
|
+
rescue Errno::ENOENT, Error
|
|
53
29
|
@version = nil
|
|
54
30
|
@available = false
|
|
55
31
|
end
|
|
@@ -58,64 +34,39 @@ module Metaclean
|
|
|
58
34
|
available? ? @version : nil
|
|
59
35
|
end
|
|
60
36
|
|
|
61
|
-
# Quick check before we even try mat2 on a file. Used by Strategy to
|
|
62
|
-
# decide whether to add :mat2 to the pipeline.
|
|
63
37
|
def supports?(path)
|
|
64
38
|
return false unless available?
|
|
65
39
|
|
|
66
40
|
SUPPORTED_EXTS.include?(Metaclean.ext_of(path))
|
|
67
41
|
end
|
|
68
42
|
|
|
69
|
-
# Strips metadata from `path` in place. Returns:
|
|
70
|
-
# true — stripped successfully
|
|
71
|
-
# :no_metadata — mat2 ran but found nothing to strip
|
|
72
|
-
# :unsupported — mat2 cannot handle this file type
|
|
73
|
-
# Raises Metaclean::Error on hard failure.
|
|
74
|
-
#
|
|
75
|
-
# We return symbols (instead of always raising) so the runner can show a
|
|
76
|
-
# friendly "skipped" message and continue with the next tool.
|
|
77
43
|
def strip!(path)
|
|
78
44
|
raise Error, 'mat2 not available' unless available?
|
|
79
45
|
|
|
80
|
-
|
|
81
|
-
|
|
46
|
+
source = FileOps.regular_stat!(path)
|
|
47
|
+
FileOps.with_private_workspace(path, 'mat2') do |workspace|
|
|
48
|
+
work = File.join(workspace, "input#{File.extname(path)}")
|
|
49
|
+
FileOps.copy_exclusive(path, work, preserve: true, expected: source)
|
|
50
|
+
cleaned = cleaned_path_for(work)
|
|
51
|
+
out, err, status = Metaclean.capture3('mat2', Metaclean.safe_path(work))
|
|
82
52
|
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
File.delete(cleaned) if File.exist?(cleaned)
|
|
53
|
+
if status.success?
|
|
54
|
+
return :no_metadata unless File.exist?(cleaned)
|
|
86
55
|
|
|
87
|
-
|
|
56
|
+
FileOps.validate_output!(cleaned)
|
|
57
|
+
FileOps.ensure_same_file!(path, source)
|
|
58
|
+
FileOps.restore_metadata(cleaned, source)
|
|
59
|
+
File.rename(cleaned, path)
|
|
60
|
+
return true
|
|
61
|
+
end
|
|
88
62
|
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
# nothing to remove. We check exit status BEFORE the "unsupported"
|
|
92
|
-
# message so a successful run that merely warns about one embedded
|
|
93
|
-
# stream isn't misreported as a soft skip.
|
|
94
|
-
if status.success?
|
|
95
|
-
return :no_metadata unless File.exist?(cleaned)
|
|
63
|
+
combined = "#{out}\n#{err}"
|
|
64
|
+
return :unsupported if combined.match?(UNSUPPORTED_RE)
|
|
96
65
|
|
|
97
|
-
|
|
98
|
-
return true
|
|
66
|
+
raise Error, "mat2 failed: #{err.strip.empty? ? out.strip : err.strip}"
|
|
99
67
|
end
|
|
100
|
-
|
|
101
|
-
# Failure path. A "not supported" message means a soft skip we report
|
|
102
|
-
# so the runner can continue with the next tool, not a hard error.
|
|
103
|
-
combined = "#{out}\n#{err}"
|
|
104
|
-
return :unsupported if combined.match?(UNSUPPORTED_RE)
|
|
105
|
-
|
|
106
|
-
# `err.strip.empty? ? out.strip : err.strip` picks whichever stream
|
|
107
|
-
# has actual content — some tools log to stdout, others to stderr.
|
|
108
|
-
raise Error, "mat2 failed: #{err.strip.empty? ? out.strip : err.strip}"
|
|
109
|
-
ensure
|
|
110
|
-
# Interrupt-safety: if we were killed (Ctrl-C) between mat2 writing
|
|
111
|
-
# `<name>.cleaned.<ext>` and the rename, don't leave the orphan behind.
|
|
112
|
-
# On the success path it's already moved, so this is a no-op.
|
|
113
|
-
File.delete(cleaned) if cleaned && File.exist?(cleaned)
|
|
114
68
|
end
|
|
115
69
|
|
|
116
|
-
# Builds the path mat2 will write to: `name.cleaned.ext`.
|
|
117
|
-
# We use File.dirname/basename/join instead of string concatenation so
|
|
118
|
-
# this works on Windows (\ separator) too.
|
|
119
70
|
def cleaned_path_for(path)
|
|
120
71
|
dir = File.dirname(path)
|
|
121
72
|
ext = File.extname(path)
|
data/lib/metaclean/qpdf.rb
CHANGED
|
@@ -1,15 +1,6 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
#
|
|
5
|
-
# Why qpdf on top of mat2/ExifTool? PDFs hide metadata in places those two
|
|
6
|
-
# don't always reach: orphaned objects, unused image streams, old revisions.
|
|
7
|
-
# qpdf rebuilds the PDF using only referenced objects — a final pass after the
|
|
8
|
-
# other tools have stripped the obvious metadata.
|
|
9
|
-
|
|
10
|
-
require 'open3'
|
|
11
|
-
require 'fileutils'
|
|
12
|
-
require 'securerandom'
|
|
3
|
+
require 'json'
|
|
13
4
|
|
|
14
5
|
module Metaclean
|
|
15
6
|
module Qpdf
|
|
@@ -18,15 +9,15 @@ module Metaclean
|
|
|
18
9
|
def available?
|
|
19
10
|
return @available if defined?(@available)
|
|
20
11
|
|
|
21
|
-
out, _err, status =
|
|
12
|
+
out, _err, status = Metaclean.capture3(
|
|
13
|
+
'qpdf', '--version',
|
|
14
|
+
timeout: Metaclean::PROBE_TIMEOUT,
|
|
15
|
+
max_output: Metaclean::PROBE_MAX_OUTPUT_BYTES
|
|
16
|
+
)
|
|
22
17
|
@available = status.success?
|
|
23
|
-
# `qpdf --version` prints "qpdf version 11.9.0" on its first line. We
|
|
24
|
-
# keep just the bare number (`.split.last`) so callers don't each have
|
|
25
|
-
# to post-process it — matching Exiftool.version / Mat2.version. Captured
|
|
26
|
-
# here so `version` reuses it instead of re-spawning the binary.
|
|
27
18
|
@version = @available ? out.lines.first.to_s.strip.split.last : nil
|
|
28
19
|
@available
|
|
29
|
-
rescue Errno::ENOENT
|
|
20
|
+
rescue Errno::ENOENT, Error
|
|
30
21
|
@version = nil
|
|
31
22
|
@available = false
|
|
32
23
|
end
|
|
@@ -35,45 +26,54 @@ module Metaclean
|
|
|
35
26
|
available? ? @version : nil
|
|
36
27
|
end
|
|
37
28
|
|
|
38
|
-
# Rebuilds a PDF in place. The qpdf flags here:
|
|
39
|
-
# --linearize → optimize for streaming/web
|
|
40
|
-
# --object-streams=generate → bundle objects efficiently
|
|
41
|
-
# --remove-unreferenced-resources=yes → drop unused content (the
|
|
42
|
-
# privacy-relevant part!)
|
|
43
|
-
#
|
|
44
|
-
# qpdf can't write back to the same file, so we use the standard
|
|
45
|
-
# "atomic write" pattern: write to a temp file, then rename it on top of
|
|
46
|
-
# the original. `File.rename` (used internally by `FileUtils.mv` for
|
|
47
|
-
# same-filesystem moves) is atomic on POSIX — either the swap completes
|
|
48
|
-
# or nothing changes. No "half-written" state is ever visible.
|
|
49
29
|
def rebuild!(path)
|
|
50
30
|
raise Error, 'qpdf not available' unless available?
|
|
51
31
|
|
|
52
|
-
|
|
53
|
-
|
|
32
|
+
source = FileOps.regular_stat!(path)
|
|
33
|
+
ensure_no_attachments!(path)
|
|
34
|
+
FileOps.with_private_workspace(path, 'qpdf') do |workspace|
|
|
35
|
+
tmp = File.join(workspace, 'output.pdf')
|
|
36
|
+
_out, err, status = Metaclean.capture3(
|
|
37
|
+
'qpdf', '--linearize', '--object-streams=generate',
|
|
38
|
+
'--remove-unreferenced-resources=yes',
|
|
39
|
+
Metaclean.safe_path(path), Metaclean.safe_path(tmp)
|
|
40
|
+
)
|
|
54
41
|
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
42
|
+
success = status.success? || status.exitstatus == 3
|
|
43
|
+
raise Error, "qpdf failed: #{err.strip}" unless success && File.exist?(tmp)
|
|
44
|
+
|
|
45
|
+
FileOps.validate_output!(tmp)
|
|
46
|
+
validate_pdf!(tmp)
|
|
47
|
+
FileOps.ensure_same_file!(path, source)
|
|
48
|
+
FileOps.restore_metadata(tmp, source)
|
|
49
|
+
File.rename(tmp, path)
|
|
50
|
+
true
|
|
51
|
+
end
|
|
52
|
+
end
|
|
59
53
|
|
|
60
|
-
|
|
61
|
-
|
|
54
|
+
def ensure_no_attachments!(path)
|
|
55
|
+
out, err, status = Metaclean.capture3(
|
|
56
|
+
'qpdf', '--json', '--json-key=attachments', Metaclean.safe_path(path)
|
|
57
|
+
)
|
|
62
58
|
success = status.success? || status.exitstatus == 3
|
|
63
|
-
raise Error, "qpdf
|
|
59
|
+
raise Error, "qpdf could not inspect PDF attachments: #{err.strip}" unless success
|
|
60
|
+
|
|
61
|
+
attachments = JSON.parse(out).fetch('attachments')
|
|
62
|
+
raise Error, 'Unexpected qpdf attachment inventory' unless attachments.is_a?(Hash)
|
|
63
|
+
return if attachments.empty?
|
|
64
64
|
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
# Interrupt-safety: drop the temp if we died (or failed) before the
|
|
69
|
-
# rename. On success it's already moved, so this is a no-op.
|
|
70
|
-
File.delete(tmp) if tmp && File.exist?(tmp)
|
|
65
|
+
raise Error, 'PDF contains embedded attachments whose nested metadata cannot be verified'
|
|
66
|
+
rescue JSON::ParserError, KeyError => e
|
|
67
|
+
raise Error, "Could not parse qpdf attachment inventory: #{e.message}"
|
|
71
68
|
end
|
|
72
69
|
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
70
|
+
def validate_pdf!(path)
|
|
71
|
+
_out, err, status = Metaclean.capture3(
|
|
72
|
+
'qpdf', '--check', Metaclean.safe_path(path)
|
|
73
|
+
)
|
|
74
|
+
return true if status.success? || status.exitstatus == 3
|
|
75
|
+
|
|
76
|
+
raise Error, "qpdf produced an invalid PDF: #{err.strip}"
|
|
77
77
|
end
|
|
78
78
|
end
|
|
79
79
|
end
|