metaclean 4.1.0 → 4.2.0

data/lib/metaclean/strategy.rb

@@ -1,45 +1,23 @@
 # frozen_string_literal: true
 
-# The "policy" module: which tools to run for which file, and what counts as
-# privacy-relevant if it survives a clean.
-#
-# Keeping this logic in its own file means the runner doesn't need to know
-# about formats — it just asks Strategy.tools_for(path) and runs whatever
-# comes back.
-
 module Metaclean
   module Strategy
-    # Group-name PREFIXES treated as privacy-bearing. Matching whole families by
-    # prefix keeps the residual check fail-closed instead of an exact allowlist
-    # that silently misses variants:
-    #   GPS*       — GPS plus any sub-group
-    #   XMP-*      — every XMP namespace (XMP-exif GPS, XMP-mwg-rs face/person
-    #                names, XMP-xmpMM DocumentID, XMP-iptcExt, …)
-    #   MakerNotes*, IPTC*
-    #   IFD1       — the embedded thumbnail IFD; a surviving thumbnail can carry
-    #                the original's full EXIF+GPS
-    # Over-flagging here is deliberate: for a privacy tool a false "still
-    # present" is far cheaper than a false "Cleaned". (ICC colour-profile groups
-    # are intentionally NOT flagged — a colour profile isn't PII; any genuinely
-    # identifying field such as Copyright is still caught by PRIVACY_TAGS below.)
     PRIVACY_GROUP_PREFIXES = %w[GPS XMP- MakerNotes IPTC IFD1].freeze
 
-    # Formats ExifTool can't WRITE, so it leaves document-internal metadata only
-    # mat2's rebuild removes (and can't re-read to verify). If mat2 won't run for
-    # one of these, the runner warns coverage is reduced rather than reporting a
-    # confident "Cleaned". (PDF is NOT here: ExifTool writes PDF metadata and qpdf
-    # rebuilds the file, so PDF is fully handled and verifiable without mat2.)
-    MAT2_ESSENTIAL = %w[docx xlsx pptx odt ods odp odg odf epub].freeze
-
-    # Specific tag NAMES (regardless of group) we never want to leak.
-    # If exiftool reports e.g. "EXIF:Artist" we still flag it because of the
-    # tag-name match, not the group. exiftool's `-all=` normally strips these,
-    # so this list is a fail-closed BACKSTOP: if any survive a strip we'd rather
-    # over-warn than report a confident "Cleaned".
+    MAT2_ESSENTIAL = %w[
+      docx xlsx pptx odt ods odp odg odf odc odi epub
+      zip tar torrent html htm xhtml xht ncx
+    ].freeze
+
     PRIVACY_TAGS = %w[
       Artist Author Creator Copyright Rights
       By-line By-lineTitle Credit Source Contact OwnerName
       CameraOwnerName SerialNumber InternalSerialNumber LensSerialNumber
+      CameraSerialNumber UniqueCameraModel LocalizedCameraModel
+      RawDataUniqueID OriginalRawFileName OriginalRawFileData
+      OriginalRawFileDigest RawImageDigest NewRawImageDigest
+      CameraCalibrationSignature ProfileCalibrationSignature
+      ProfileName ProfileCopyright AsShotProfileName
       Software HostComputer ProcessingSoftware
       ImageDescription UserComment
       LastModifiedBy LastSavedBy LastAuthor
@@ -49,62 +27,49 @@ module Metaclean
       XPAuthor XPComment XPSubject XPKeywords XPTitle Comment
     ].freeze
 
-    # File extensions where mat2 is meaningfully stricter than ExifTool and
-    # should run first. For other formats, ExifTool is the broader expert.
-    # (mkv/webm are NOT here — see FFMPEG_FORMATS; no mat2/ExifTool path writes
-    # Matroska.)
     MAT2_PREFERRED = %w[
-      docx xlsx pptx odt ods odp odg odf epub png svg
+      docx xlsx pptx odt ods odp odg odf odc odi epub png svg svgz
       mp4 avi
     ].freeze
 
-    # Matroska containers. ExifTool is read-only for them and mat2 has no
-    # Matroska parser, so neither can strip mkv/webm. ffmpeg is the only tool in
-    # the set that can — it remuxes the container dropping all metadata while
-    # copying every stream verbatim (lossless, no re-encode).
+    RESIDUAL_IGNORED_GROUPS = %w[System ExifTool Composite].freeze
+
+    ICC_PRIVACY_TAGS = %w[
+      ProfileDescription ProfileDescriptionML ProfileCopyright ProfileCreator
+      ProfileCMMType DeviceManufacturer DeviceModel DeviceMfgDesc DeviceModelDesc
+    ].freeze
+
+    ICC_STANDARD_DESCRIPTIONS = [
+      'srgb iec61966-2.1', 'srgb', 'srgb iec61966-2-1 black scaled',
+      'srgb iec61966-2-1 no black scaling', 'display p3', 'dci-p3', 'p3-d65 (display)',
+      'adobe rgb (1998)', 'prophoto rgb', 'romm rgb: iso 22028-2:2013',
+      'apple rgb', 'colormatch rgb', 'camera rgb profile',
+      'apple wide color sharing profile', 'linear rgb profile',
+      'generic rgb profile', 'generic gray profile', 'generic gray gamma 2.2 profile',
+      'generic lab profile', 'generic cmyk profile', 'generic xyz profile',
+      'gray gamma 2.2', 'dot gain 20%',
+      'rec. itu-r bt.709-5', 'hdtv (rec. 709)', 'sdtv (rec. 601)'
+    ].freeze
+
     FFMPEG_FORMATS = %w[mkv webm].freeze
 
-    # Raster formats mat2 cannot strip without DAMAGING the file: it rebuilds via
-    # Pillow, which recompresses JPEG/WebP (visible quality loss — a clean
-    # wallpaper drops ~65% in size with no metadata to remove) and downconverts
-    # TIFF (16-bit → 8-bit). ExifTool strips all of these completely and IN PLACE
-    # (pixels byte-identical), so ExifTool owns them and mat2 is skipped —
-    # cleaning metadata must never silently damage the file.
-    MAT2_DEGRADES = %w[jpg jpeg webp tif tiff].freeze
+    MAT2_DEGRADES = %w[jpg jpe jpeg webp tif tiff heic].freeze
 
     module_function
 
-    # Returns an ordered list of tool symbols (e.g. `[:mat2, :exiftool, :qpdf]`)
-    # to run on `path`. The runner executes them in order; if one fails or
-    # is skipped, the next still runs. The tools are always used together
-    # for maximum coverage — there is no per-tool opt-out; a tool that isn't
-    # installed is simply left out (the `.available?`/`.supports?` checks).
     def tools_for(path)
       ext = Metaclean.ext_of(path)
       tools = []
 
       if ext == 'pdf'
-        # mat2 cleans PDFs by RASTERIZING every page (text → images): it destroys
-        # the text layer and balloons the file (~35×). So PDFs skip mat2 and use:
-        #   exiftool → strips the Info dictionary + XMP (Author, Title, Producer…)
-        #   qpdf → rebuilds the file, dropping unreferenced objects / old revisions
-        # Both are lossless and leave the text intact. (PDF JS/macros are out of
-        # scope — see README.)
         tools << :exiftool
         tools << :qpdf if Qpdf.available?
       elsif FFMPEG_FORMATS.include?(ext)
-        # Matroska (mkv/webm): ffmpeg is the ONLY tool that can clean these.
-        # ExifTool still re-reads the result afterwards, so the residual check
-        # (the false-clean backstop) is not blind.
         tools << :ffmpeg if Ffmpeg.available?
       elsif MAT2_PREFERRED.include?(ext) && Mat2.available?
-        # Office docs, modern image/video containers — mat2 leads.
         tools << :mat2
         tools << :exiftool
       else
-        # Everything else (JPEG, MP3, RAW, …) — ExifTool has the broadest coverage.
-        # mat2 still adds coverage for many, but NOT for rasters it would damage
-        # (MAT2_DEGRADES) — there ExifTool's in-place strip is complete and lossless.
         tools << :exiftool
         tools << :mat2 if Mat2.supports?(path) && !MAT2_DEGRADES.include?(ext)
       end
@@ -112,62 +77,72 @@ module Metaclean
       tools
     end
 
-    # Looks at metadata read AFTER cleaning and returns the entries that
-    # still look privacy-relevant. The runner uses this for the "still
-    # present" warning at the end of each file.
-    #
-    # Why both group-match and tag-match? Tag names can appear under
-    # different groups depending on the format (e.g. "Author" in PDF vs
-    # "Artist" in EXIF). Combining the two keeps coverage broad without
-    # having to enumerate every {group, tag} pair.
-    def privacy_residual(meta)
+    def privacy_residual(meta, allow_icc_metadata: false)
+      standard_icc = standard_icc_profile?(meta)
       meta.select do |k, v|
-        # Skip SourceFile and the System/File/etc. groups — not user metadata.
-        next false unless Display.embedded_key?(k)
+        key = k.to_s
+        next false if key == 'SourceFile'
 
-        # ExifTool keys look like "GPS:GPSLatitude". Split on the first ":";
-        # no "Group:" prefix means the whole key is the tag name.
-        group, tag = k.to_s.split(':', 2)
+        group, tag = key.split(':', 2)
         name = tag.nil? ? group.to_s : tag
+        next false if RESIDUAL_IGNORED_GROUPS.include?(group.to_s)
+
+        icc_privacy = group.to_s.start_with?('ICC') && ICC_PRIVACY_TAGS.include?(name)
+        if group.to_s.start_with?('ICC')
+          next false if allow_icc_metadata || standard_icc
+          next false unless icc_privacy
+        end
 
-        # A zeroed/empty value is not a leak for un-removable container atoms like
-        # QuickTime:CreateDate (deletable only by zeroing, "0000:00:00 …") — without
-        # this every video would fail the gate on an already-zeroed date. GPS is the
-        # exception: 0,0 is a REAL location (Null Island) and a coordinate ExifTool
-        # reports as 0 (or null) must still be caught, so the blank exemption NEVER
-        # applies to GPS-family entries — the whole point of the fail-closed backstop.
         gps = group.to_s.start_with?('GPS') || name.start_with?('GPS')
-        next false if !gps && blank_value?(v)
+        next false if !gps && exempt_blank?(name, v)
 
-        privacy_group?(group) || privacy_tag?(name)
+        icc_privacy || privacy_group?(group) || privacy_tag?(name)
       end
     end
 
-    # True when a value carries no information: empty, or only zeros plus date/time
-    # punctuation and the "Z" (UTC) marker — e.g. "0000:00:00 00:00:00", or the ASF
-    # variant "0000:00:00 00:00:00Z" that mat2 writes into WMV's mandatory date
-    # field. Only the digit 0 is stripped (never 1-9), so a real value like
-    # "59.9139", "Jane Doe", or a real "2024:..." date keeps other characters and
-    # is NOT blank. (GPS is exempt from this check entirely — see privacy_residual.)
+    def standard_icc_profile?(meta)
+      meta.any? do |k, v|
+        key = k.to_s
+        next false unless key.start_with?('ICC') && key.split(':', 2).last == 'ProfileDescription'
+
+        ICC_STANDARD_DESCRIPTIONS.include?(normalize_icc_value(v))
+      end
+    end
+
+    def normalize_icc_value(value)
+      value.to_s.strip.downcase.gsub(/\s+/, ' ')
+    end
+
+    def exempt_blank?(name, value)
+      return true if value.to_s.strip.empty?
+
+      date_like_tag?(name) && blank_value?(value)
+    end
+
+    def date_like_tag?(name)
+      n = name.to_s
+      n.end_with?('Date') || n.start_with?('DateTime')
+    end
+
     def blank_value?(value)
       s = value.to_s
       s.strip.empty? || s.gsub(/[Z0\s:.+-]/, '').empty?
     end
 
-    # A group is privacy-bearing if it matches one of the family prefixes
-    # (GPS, XMP-, MakerNotes, IPTC, IFD1).
     def privacy_group?(group)
       PRIVACY_GROUP_PREFIXES.any? { |p| group.to_s.start_with?(p) }
     end
 
-    # A tag is privacy-bearing if it's in the exact list OR is any GPS* tag
-    # (GPSLatitude/GPSLongitude/GPSPosition/… regardless of group).
     def privacy_tag?(tag)
       t = tag.to_s
-      PRIVACY_TAGS.include?(t) || t.start_with?('GPS')
+      return true if PRIVACY_TAGS.include?(t) || t.start_with?('GPS')
+      return true if t.match?(/SerialNumber\z/i)
+      return true if t.match?(/\AOriginalRawFile(?:Name|Data|Digest)\z/i)
+      return true if t.match?(/\A(?:RawDataUniqueID|RawImageDigest|NewRawImageDigest)\z/i)
+
+      false
     end
 
-    # Does this path need mat2 for adequate coverage? (See MAT2_ESSENTIAL.)
     def mat2_essential?(path)
       MAT2_ESSENTIAL.include?(Metaclean.ext_of(path))
     end

data/lib/metaclean/version.rb

@@ -1,9 +1,5 @@
 # frozen_string_literal: true
 
-# Single source of truth for the program's version.
-# Both the gemspec and `metaclean --version` read from here, so we only have
-# one place to bump.
-
 module Metaclean
-  VERSION = '4.1.0'
+  VERSION = '4.2.0'
 end

data/lib/metaclean.rb

@@ -1,90 +1,89 @@
 # frozen_string_literal: true
 
-# Library entry point. require order matters: dependencies before dependents.
-
 require 'open3'
 
 require 'metaclean/version'
 require 'metaclean/display'
+require 'metaclean/file_ops'
 require 'metaclean/exiftool'
 require 'metaclean/mat2'
 require 'metaclean/qpdf'
 require 'metaclean/ffmpeg'
 require 'metaclean/strategy'
+require 'metaclean/discovery'
+require 'metaclean/committer'
 require 'metaclean/runner'
 require 'metaclean/cli'
 
 module Metaclean
   class Error < StandardError; end
 
-  # Raised by ensure_tools! when any of the four required external tools is not
-  # on PATH. metaclean runs ExifTool, mat2, qpdf and ffmpeg together and refuses
-  # to run without all of them.
   class ToolsMissing < Error; end
 
-  # A path beginning with "-" is misread as an *option* by the tools we shell
-  # out to — e.g. exiftool's `-config FILE` loads and runs arbitrary Perl.
-  # Open3 argument arrays bypass the shell, but NOT the invoked tool's own
-  # option parser. Prefixing a leading-dash relative path with "./" makes it
-  # unambiguously a filename to every tool. Absolute paths and normal names
-  # pass through untouched. Used at every shell-out boundary.
   def self.safe_path(path)
     s = path.to_s
     s.start_with?('-') ? File.join('.', s) : s
   end
 
-  # External tools can hang, or run away producing endless output, on a corrupt
-  # or hostile file. Every OPERATIONAL shell-out (read/strip/rebuild) goes through
-  # this instead of Open3.capture3 so one bad file is bounded on BOTH axes — by
-  # wall-clock (COMMAND_TIMEOUT) and by captured bytes (MAX_OUTPUT_BYTES) — rather
-  # than hanging or exhausting memory and taking the whole batch with it. The
-  # quick availability probes (`-ver`/`--version`) stay on plain capture3: fixed
-  # args, no file input, nothing to hang on.
-  COMMAND_TIMEOUT = 120 # seconds
-  # Per stream (stdout AND stderr). Far above any legitimate output from the
-  # tools' invocations here (metadata JSON / `-q` strips / `-v error` muxes), so
-  # tripping it means a runaway, not a real result.
+  COMMAND_TIMEOUT = 120
   MAX_OUTPUT_BYTES = 64 * 1024 * 1024
+  PROBE_TIMEOUT = 10
+  PROBE_MAX_OUTPUT_BYTES = 1024 * 1024
   READ_CHUNK = 64 * 1024
 
-  # Drop-in replacement for Open3.capture3 that returns the same [out, err,
-  # status] triple but kills the command (and anything it spawned) if it runs
-  # past `timeout` OR floods more than `max_output` bytes on either stream.
-  def self.capture3(*cmd, timeout: COMMAND_TIMEOUT, max_output: MAX_OUTPUT_BYTES)
+  def self.command_timeout
+    override = ENV['METACLEAN_TIMEOUT'].to_i
+    override.positive? ? override : COMMAND_TIMEOUT
+  end
+
+  def self.capture3(*cmd, timeout: command_timeout, max_output: MAX_OUTPUT_BYTES)
     Open3.popen3(*cmd, pgroup: true) do |stdin, stdout, stderr, wait_thr|
-      stdin.close
-      # Drain both pipes concurrently: a tool that fills one pipe buffer would
-      # otherwise block forever before exiting, and `join` below would never see
-      # it finish even though it isn't actually hung.
-      out_t = read_capped(stdout, max_output, wait_thr)
-      err_t = read_capped(stderr, max_output, wait_thr)
-
-      if wait_thr.join(timeout).nil?
+      out_t = err_t = deadline = nil
+      begin
+        stdin.close
+        deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + timeout
+        out_t = read_capped(stdout, max_output, wait_thr)
+        err_t = read_capped(stderr, max_output, wait_thr)
+
+        if wait_thr.join(timeout).nil?
+          kill_group(wait_thr)
+          drain_readers(out_t, err_t, deadline)
+          raise Error, "#{cmd.first} timed out after #{timeout}s"
+        end
+
+        unless drain_readers(out_t, err_t, deadline)
+          kill_group(wait_thr)
+          drain_readers(out_t, err_t, deadline)
+          raise Error, "#{cmd.first} timed out after #{timeout}s"
+        end
+
+        out, out_over = out_t.value
+        err, err_over = err_t.value
+        raise Error, "#{cmd.first} exceeded the #{max_output}-byte output limit" if out_over || err_over
+
+        [out, err, wait_thr.value]
+      rescue Interrupt
         kill_group(wait_thr)
-        out_t.join(2)
-        err_t.join(2)
-        raise Error, "#{cmd.first} timed out after #{timeout}s"
+        drain_readers(out_t, err_t, deadline) if out_t && err_t && deadline
+        raise
       end
+    end
+  end
 
-      out, out_over = out_t.value
-      err, err_over = err_t.value
-      raise Error, "#{cmd.first} exceeded the #{max_output}-byte output limit" if out_over || err_over
-
-      [out, err, wait_thr.value]
+  def self.drain_readers(out_t, err_t, deadline)
+    [out_t, err_t].all? do |t|
+      remaining = deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      remaining = 0.1 if remaining < 0.1
+      !t.join(remaining).nil?
     end
   end
 
-  # Read an IO into a String in a thread, but stop accumulating once it passes
-  # `limit` bytes — and kill the command then, so a flooding stream is cut off
-  # promptly instead of waiting out the full timeout. After the cap is hit it
-  # keeps draining (discarding) so the dying child isn't blocked on a full pipe.
-  # Returns [string, overflowed?].
   def self.read_capped(io, limit, wait_thr)
     Thread.new do
       buf = +''
       over = false
       while (chunk = io.read(READ_CHUNK))
-        next if over # past the cap: drain & discard so the child can exit
+        next if over
 
         buf << chunk
         next unless buf.bytesize > limit
@@ -97,51 +96,30 @@ module Metaclean
     end
   end
 
-  # SIGTERM the child's whole process group — pgroup:true made the child the
-  # group leader, so any helpers it forked are signalled too — escalating to
-  # SIGKILL if it ignores TERM. A negative pid targets the group.
   def self.kill_group(wait_thr)
     Process.kill('-TERM', wait_thr.pid)
     Process.kill('-KILL', wait_thr.pid) unless wait_thr.join(2)
   rescue Errno::ESRCH, Errno::EPERM
-    nil # already gone, or not permitted to signal it — nothing more to do
+    nil
   end
 
-  # Lower-cased, dot-stripped extension used for FORMAT ROUTING decisions
-  # (Strategy#tools_for, Strategy#mat2_essential?, Mat2.supports?). One
-  # definition so every routing path normalizes the extension identically —
-  # a future tweak (double extensions, locale-safe downcasing) lands once.
   def self.ext_of(path)
     File.extname(path.to_s).downcase.delete('.')
   end
 
-  # Marker embedded in every staging-temp filename (Runner, Ffmpeg, Qpdf) and
-  # matched by Runner#skip?, so a leftover temp from an interrupted run is
-  # ignored on a later directory scan. One literal keeps the producers and the
-  # matcher from drifting (qpdf previously embedded a divergent
-  # ".metaclean.qpdf.tmp." that didn't contain this marker).
   TMP_MARKER = '.metaclean.tmp.'
 
-  # Suffix of the default "<name>_clean.<ext>" outputs. Runner#build_clean_path
-  # writes it; CLEAN_OUTPUT_RE derives the loop-prevention match from it so the
-  # producer and Runner#skip? can't disagree.
   CLEAN_SUFFIX = '_clean'
 
-  # Matches our own "<name>_clean.<ext>" outputs (with optional "_N" collision
-  # counter) so a recursive re-run doesn't re-clean them. Compiled once here,
-  # in the module body that runs after the requires, so CLEAN_SUFFIX exists.
-  CLEAN_OUTPUT_RE = /#{Regexp.escape(CLEAN_SUFFIX)}(_\d+)?\.[^.]+\z/
+  CLEAN_OUTPUT_RE = /#{Regexp.escape(CLEAN_SUFFIX)}(?:_\d+)?(?:\.[^.]+)?\z/
 
-  # Preflight: all four tools must be installed. We run them together for full
-  # coverage and to verify the strip, so a partial toolchain is not "good enough"
-  # — bail with one clear message naming what's missing and how to install
-  # everything. Called once by the CLI before any inspect/clean work.
-  def self.ensure_tools!
+  def self.ensure_tools!(in_place: false)
     missing = []
     missing << 'exiftool' unless Exiftool.available?
     missing << 'mat2'     unless Mat2.available?
     missing << 'qpdf'     unless Qpdf.available?
     missing << 'ffmpeg'   unless Ffmpeg.available?
+    missing << 'cp with metadata preservation' if in_place && !FileOps.metadata_copy_supported?
     return if missing.empty?
 
     raise ToolsMissing, <<~MSG
@@ -153,6 +131,8 @@ module Metaclean
         Fedora:         sudo dnf install perl-Image-ExifTool mat2 qpdf ffmpeg
         Arch:           sudo pacman -S perl-image-exiftool mat2 qpdf ffmpeg
         Windows:        use WSL2 (https://learn.microsoft.com/windows/wsl/install) + the Debian/Ubuntu line
+
+      --in-place additionally requires the POSIX cp utility supplied by macOS or coreutils on Linux.
     MSG
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: metaclean
 version: !ruby/object:Gem::Version
-  version: 4.1.0
+  version: 4.2.0
 platform: ruby
 authors:
 - Laurent Zogaj
@@ -21,12 +21,16 @@ extra_rdoc_files: []
 files:
 - LICENSE
 - README.md
+- SECURITY.md
 - bin/metaclean
 - lib/metaclean.rb
 - lib/metaclean/cli.rb
+- lib/metaclean/committer.rb
+- lib/metaclean/discovery.rb
 - lib/metaclean/display.rb
 - lib/metaclean/exiftool.rb
 - lib/metaclean/ffmpeg.rb
+- lib/metaclean/file_ops.rb
 - lib/metaclean/mat2.rb
 - lib/metaclean/qpdf.rb
 - lib/metaclean/runner.rb
@@ -38,6 +42,7 @@ licenses:
 metadata:
   allowed_push_host: https://rubygems.org
   bug_tracker_uri: https://github.com/26zl/metaclean/issues
+  documentation_uri: https://github.com/26zl/metaclean#readme
   source_code_uri: https://github.com/26zl/metaclean
   rubygems_mfa_required: 'true'
 rdoc_options: []
@@ -58,6 +63,7 @@ requirements:
 - mat2 (https://github.com/jvoisin/mat2) on PATH — required
 - qpdf (https://qpdf.sourceforge.io) on PATH — required
 - ffmpeg (https://ffmpeg.org) on PATH — required
+- POSIX cp with metadata-preservation support — required for --in-place
 rubygems_version: 3.6.9
 specification_version: 4
 summary: CLI that strips file metadata with ExifTool, mat2, qpdf and ffmpeg.