metaclean 4.0.1 → 4.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b514075fb238c6274263922e87d7218b6c27b86f46c07dfa6fe36e8a1387b382
-  data.tar.gz: 7eff78f7bd882717bc03cc6f98cb140faa4c9123d5b6a0806ccdb0e342cd3935
+  metadata.gz: 29c5b4aad77d530c77d0f4fbae6bbb1811dd6c801719e67999a1da802522a06a
+  data.tar.gz: cbeb0bc1eaf21557b922641fe255b1542dfc776e8833d20ae3fc1c74c9d378a5
 SHA512:
-  metadata.gz: b674107a22b4965a30bd5c990c90fec5d4944828a8ca755f27124029383de2a543fdf73fad42934b7fa6c4600d0670ffc79ff1602a9b23caf4da9bec7e239ccf
-  data.tar.gz: d265c366f6b7f0c76acd866e883dc8459ab4e35c27ef092abb4488262e4a5726f7b812d00bba5419282a47d05c80ac3e5e4b97254d2849f3bb4320d7cb19e5b8
+  metadata.gz: b33eebe165d0d484d8fd19c9ba8bcaa28a2767b5597c5a08df946fa329256fb23694b9df35507007d4301998b49ca2bf7599eae1f11b0979b1408ee46a71ee74
+  data.tar.gz: 48a238f59f367c4bd905455cc44bfe12863b02a73bda64b22391d528ef8523d513e5d668992a8d337fe593448ee6552746ca41cf1766fcac6dabbd540e36ba85

data/README.md

@@ -31,7 +31,9 @@ It wraps four battle-tested tools and routes each file to the right one:
 - **Verification-first:** it re-reads the cleaned file and refuses to write a
   result when known privacy metadata survives.
 - **Safer defaults:** it writes `*_clean` copies by default; `--in-place` keeps a
-  `.bak` and asks for confirmation unless `--force` is set.
+  `.bak` and asks for confirmation unless `--force` is set. Note the `.bak` is the
+  **original, with all its metadata** — delete or move the `.bak` files before
+  sharing an in-place-cleaned folder.
 - **Lossless routing:** it avoids mat2 paths that recompress JPEG/WebP or
   downconvert TIFF, and uses ffmpeg stream-copy for Matroska.
 - **Batch-friendly:** failed or unverified files exit non-zero, so scripts and CI
@@ -150,7 +152,7 @@ metaclean --dry-run photo.jpg
 | --- | --- |
 | `--inspect` | Read-only — print metadata, never write |
 | `--dry-run` | Simulate cleaning, show diff, write nothing |
-| `-i`, `--in-place` | Overwrite originals (keeps a `<file>.bak`) |
+| `-i`, `--in-place` | Overwrite originals (keeps a `<file>.bak` — the **original, metadata intact**; remove it before sharing) |
 | `-r`, `--recursive` | Recurse into directories |
 | `-f`, `--force` | Skip the confirmation prompt |
 | `-h`, `--help` | Show usage and exit |
@@ -166,8 +168,8 @@ secret**; the one-time prerequisite is registering this gem's Trusted Publisher
 on rubygems.org.
 
 ```bash
-git tag v3.0.0
-git push origin v3.0.0
+git tag v4.0.1
+git push origin v4.0.1
 ```
 
 ## Safety
@@ -177,6 +179,13 @@ git push origin v3.0.0
 - `--in-place` writes atomically: the file is built in a temp file and
   renamed into place, so a crash mid-run cannot leave a half-written original.
 - Symlinks are always skipped — metaclean never cleans through a link.
+- Folder and recursive (`-r`) scans deliberately skip **hidden files** (dot-prefixed,
+  e.g. `.secret.jpg`) and metaclean's own outputs (`*_clean.*`, `*.bak`), so a
+  directory run does not clean every file in the tree. To clean a hidden file,
+  name it explicitly: `metaclean .secret.jpg`.
+- A `<file>.bak` left by `--in-place` is the **untouched original** and still
+  contains all its metadata. It is reported in the run, but it is yours to delete
+  or move before sharing the folder.
 - Filename collisions (`photo_clean.jpg` already exists, `.bak` already
   exists) are resolved with `_1`, `_2`, … suffixes, including late collisions
   that appear while a file is being cleaned.

data/lib/metaclean/exiftool.rb

@@ -50,7 +50,7 @@ module Metaclean
     #   -n         Numeric values (no human formatting like "1/100 sec")
     #   -api largefilesupport=1   Allow files >4 GB
     def read(path)
-      out, err, status = Open3.capture3(
+      out, err, status = Metaclean.capture3(
         'exiftool', '-j', '-G1', '-a', '-u', '-s', '-n', '-api', 'largefilesupport=1',
         Metaclean.safe_path(path)
       )
@@ -111,7 +111,7 @@ module Metaclean
       also_delete.each { |tag| args << "-#{tag}=" }
       args.concat(['-overwrite_original', '-q', '-q', '-api', 'largefilesupport=1', Metaclean.safe_path(path)])
 
-      _out, err, status = Open3.capture3(*args)
+      _out, err, status = Metaclean.capture3(*args)
       return true if status.success?
       return :unsupported if err.match?(WRITE_UNSUPPORTED_RE)
 

data/lib/metaclean/ffmpeg.rb

@@ -51,7 +51,7 @@ module Metaclean
       # Clear any stale temp from an earlier crashed run before muxing.
       File.delete(tmp) if File.exist?(tmp)
 
-      _out, err, status = Open3.capture3(
+      _out, err, status = Metaclean.capture3(
         'ffmpeg', '-y', '-v', 'error', '-nostdin', '-i', file_url(path),
         '-map', '0', '-map_metadata', '-1', '-map_chapters', '-1', '-c', 'copy',
         file_url(tmp)

data/lib/metaclean/mat2.rb

@@ -84,7 +84,7 @@ module Metaclean
       # crashed run, remove it so we don't accidentally use old data.
       File.delete(cleaned) if File.exist?(cleaned)
 
-      out, err, status = Open3.capture3('mat2', safe)
+      out, err, status = Metaclean.capture3('mat2', safe)
 
       # Success path first. mat2 only creates `<name>.cleaned.<ext>` when it
       # actually stripped something; no file after exit 0 means there was

data/lib/metaclean/qpdf.rb

@@ -52,7 +52,7 @@ module Metaclean
       src = Metaclean.safe_path(path)
       tmp = tmp_path_for(path)
 
-      _out, err, status = Open3.capture3(
+      _out, err, status = Metaclean.capture3(
         'qpdf', '--linearize', '--object-streams=generate',
         '--remove-unreferenced-resources=yes', src, Metaclean.safe_path(tmp)
       )

data/lib/metaclean/runner.rb

@@ -17,6 +17,10 @@ module Metaclean
   class Runner
     def initialize(options)
       @options = options
+      # Paths that couldn't be read during discovery (missing arg, unreadable
+      # directory). Tracked so a partially-scanned batch exits non-zero instead
+      # of letting automation mistake "some files cleaned" for "everything done".
+      @scan_errors = 0
     end
 
     # Public entry points: one for `--inspect`, one for the cleaning flow.
@@ -27,6 +31,7 @@ module Metaclean
         Display.warning('No files to inspect.')
         exit 1
       end
+      failed = 0
       files.each do |file|
         Display.header "📄 #{file}"
         meta = Exiftool.read(file)
@@ -36,7 +41,13 @@ module Metaclean
         # One unreadable/odd file shouldn't abort inspecting the rest — mirrors
         # the per-file rescue in the clean batch.
         warn Display.error("#{file}: #{e.message}")
+        failed += 1
       end
+
+      # A file we couldn't read is a non-zero condition: a script using --inspect
+      # as a gate must not mistake "couldn't read it" for "no metadata". Discovery
+      # errors (missing paths / unreadable dirs) count too.
+      exit 1 if failed.positive? || @scan_errors.positive?
     end
 
     def clean_paths(paths)
@@ -55,11 +66,19 @@ module Metaclean
         action = @options[:in_place] ? 'OVERWRITE' : 'create cleaned copies of'
         puts Display.c("About to #{action} #{files.size} file(s).", :yellow)
         if @options[:in_place]
-          puts Display.c('Backups will be saved alongside as <file>.bak.', :gray)
+          # The .bak IS the original — metadata intact. Say so plainly: a user who
+          # shares the "cleaned" folder would otherwise leak it via the backup.
+          puts Display.c('Each <file>.bak is the ORIGINAL, with all metadata still in it — ' \
+                         'delete or move the .bak files before sharing the folder.', :yellow)
         end
         print Display.c('Proceed? [y/N] ', :bold)
         ans = $stdin.gets&.strip&.downcase # gets → nil on Ctrl-D
-        return Display.warning('Aborted.') unless %w[y yes].include?(ans)
+        # Abort (no/blank/EOF) is a non-zero exit, not silent success — a
+        # non-interactive caller must not read "Aborted." as "files were cleaned".
+        unless %w[y yes].include?(ans)
+          Display.warning('Aborted.')
+          exit 1
+        end
       end
 
       summary = { cleaned: 0, unverified: 0, failed: 0, removed_total: 0, residual_files: 0 }
@@ -81,8 +100,9 @@ module Metaclean
 
       print_summary(summary)
 
-      # Non-zero exit so CI/scripts can detect a failed or not-fully-verified file.
-      exit 1 if summary[:failed].positive? || summary[:unverified].positive?
+      # Non-zero exit so CI/scripts can detect a failed or not-fully-verified file,
+      # OR a batch that was never fully discovered (a path/dir we couldn't read).
+      exit 1 if summary[:failed].positive? || summary[:unverified].positive? || @scan_errors.positive?
     end
 
     private
@@ -487,6 +507,9 @@ module Metaclean
       if summary[:residual_files].positive?
         Display.warning "Files with privacy residual: #{summary[:residual_files]}"
       end
+      if @scan_errors.positive?
+        Display.warning "Paths skipped during discovery (not found or unreadable): #{@scan_errors}"
+      end
     end
 
     # File discovery — turning the user's paths into a flat list.
@@ -510,6 +533,7 @@ module Metaclean
           explicit << p
         else
           Display.warning "Not found: #{p}"
+          @scan_errors += 1
         end
       end
       discovered.reject! { |f| skip?(f) }
@@ -559,6 +583,7 @@ module Metaclean
       # warn and skip this directory so one bad entry doesn't abort discovery of
       # the rest of the batch.
       Display.warning "Skipping #{dir}: #{e.message}"
+      @scan_errors += 1
     end
 
     # Manual recursive walker. Symlinks are always skipped (never followed), so
@@ -579,6 +604,7 @@ module Metaclean
       # warn and skip this directory so one bad entry doesn't abort discovery of
       # the rest of the batch.
       Display.warning "Skipping #{dir}: #{e.message}"
+      @scan_errors += 1
     end
 
     # Files we never touch when DISCOVERED via directory scanning. This is

data/lib/metaclean/strategy.rb

@@ -76,7 +76,7 @@ module Metaclean
 
     # Returns an ordered list of tool symbols (e.g. `[:mat2, :exiftool, :qpdf]`)
     # to run on `path`. The runner executes them in order; if one fails or
-    # is skipped, the next still runs. The three tools are always used together
+    # is skipped, the next still runs. The tools are always used together
     # for maximum coverage — there is no per-tool opt-out; a tool that isn't
     # installed is simply left out (the `.available?`/`.supports?` checks).
     def tools_for(path)

data/lib/metaclean/version.rb

@@ -5,5 +5,5 @@
 # one place to bump.
 
 module Metaclean
-  VERSION = '4.0.1'
+  VERSION = '4.1.0'
 end

data/lib/metaclean.rb

@@ -2,6 +2,8 @@
 
 # Library entry point. require order matters: dependencies before dependents.
 
+require 'open3'
+
 require 'metaclean/version'
 require 'metaclean/display'
 require 'metaclean/exiftool'
@@ -31,6 +33,80 @@ module Metaclean
     s.start_with?('-') ? File.join('.', s) : s
   end
 
+  # External tools can hang, or run away producing endless output, on a corrupt
+  # or hostile file. Every OPERATIONAL shell-out (read/strip/rebuild) goes through
+  # this instead of Open3.capture3 so one bad file is bounded on BOTH axes — by
+  # wall-clock (COMMAND_TIMEOUT) and by captured bytes (MAX_OUTPUT_BYTES) — rather
+  # than hanging or exhausting memory and taking the whole batch with it. The
+  # quick availability probes (`-ver`/`--version`) stay on plain capture3: fixed
+  # args, no file input, nothing to hang on.
+  COMMAND_TIMEOUT = 120 # seconds
+  # Per stream (stdout AND stderr). Far above any legitimate output from the
+  # tools' invocations here (metadata JSON / `-q` strips / `-v error` muxes), so
+  # tripping it means a runaway, not a real result.
+  MAX_OUTPUT_BYTES = 64 * 1024 * 1024
+  READ_CHUNK = 64 * 1024
+
+  # Drop-in replacement for Open3.capture3 that returns the same [out, err,
+  # status] triple but kills the command (and anything it spawned) if it runs
+  # past `timeout` OR floods more than `max_output` bytes on either stream.
+  def self.capture3(*cmd, timeout: COMMAND_TIMEOUT, max_output: MAX_OUTPUT_BYTES)
+    Open3.popen3(*cmd, pgroup: true) do |stdin, stdout, stderr, wait_thr|
+      stdin.close
+      # Drain both pipes concurrently: a tool that fills one pipe buffer would
+      # otherwise block forever before exiting, and `join` below would never see
+      # it finish even though it isn't actually hung.
+      out_t = read_capped(stdout, max_output, wait_thr)
+      err_t = read_capped(stderr, max_output, wait_thr)
+
+      if wait_thr.join(timeout).nil?
+        kill_group(wait_thr)
+        out_t.join(2)
+        err_t.join(2)
+        raise Error, "#{cmd.first} timed out after #{timeout}s"
+      end
+
+      out, out_over = out_t.value
+      err, err_over = err_t.value
+      raise Error, "#{cmd.first} exceeded the #{max_output}-byte output limit" if out_over || err_over
+
+      [out, err, wait_thr.value]
+    end
+  end
+
+  # Read an IO into a String in a thread, but stop accumulating once it passes
+  # `limit` bytes — and kill the command then, so a flooding stream is cut off
+  # promptly instead of waiting out the full timeout. After the cap is hit it
+  # keeps draining (discarding) so the dying child isn't blocked on a full pipe.
+  # Returns [string, overflowed?].
+  def self.read_capped(io, limit, wait_thr)
+    Thread.new do
+      buf = +''
+      over = false
+      while (chunk = io.read(READ_CHUNK))
+        next if over # past the cap: drain & discard so the child can exit
+
+        buf << chunk
+        next unless buf.bytesize > limit
+
+        over = true
+        buf = buf.byteslice(0, limit)
+        kill_group(wait_thr)
+      end
+      [buf, over]
+    end
+  end
+
+  # SIGTERM the child's whole process group — pgroup:true made the child the
+  # group leader, so any helpers it forked are signalled too — escalating to
+  # SIGKILL if it ignores TERM. A negative pid targets the group.
+  def self.kill_group(wait_thr)
+    Process.kill('-TERM', wait_thr.pid)
+    Process.kill('-KILL', wait_thr.pid) unless wait_thr.join(2)
+  rescue Errno::ESRCH, Errno::EPERM
+    nil # already gone, or not permitted to signal it — nothing more to do
+  end
+
   # Lower-cased, dot-stripped extension used for FORMAT ROUTING decisions
   # (Strategy#tools_for, Strategy#mat2_essential?, Mat2.supports?). One
   # definition so every routing path normalizes the extension identically —

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: metaclean
 version: !ruby/object:Gem::Version
-  version: 4.0.1
+  version: 4.1.0
 platform: ruby
 authors:
 - Laurent Zogaj