fair_champion_harvester 0.1.14 → 0.1.15

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ecc509041da6bb1b6e39fec571b281b8b78fc3726fe886e6720e7ca8f505bf6d
-  data.tar.gz: 82bb48606f99990171318569dfa9cd22c593967401b0373771c05b0f93bf7c1b
+  metadata.gz: d47d4ea410de9aeb76e8e746409aaa0fa07e9fd6941cbd6bb5075d0639d34622
+  data.tar.gz: d9eb03395dc25d6025dcbf849b56de37561d839b267bdaaf6aa365fbd42bc51c
 SHA512:
-  metadata.gz: 7b9746031f2bde80c3487a06612ec6b588f0ad3098b622ec7e8a41c8d816f70b3c8dd1b742e0bedfe1c77f1dbfd58f2e4e4855da886717958b2ac66aaacae7d1
-  data.tar.gz: 1e4e0a02c9ff6b502a56de0b76d27e501b4904975554e91f9891aed3df95db7a9dc8a7ff540de720b7c30be3c1567c8e5914521c6b5d5f773567fe13849c663b
+  metadata.gz: 5a4a6dd6d40cd24ea5e4b0e555f52d09d40cb8d45b5fe4e50456f29b537f9274b8fd22a799cf44103e718c185722786529336edccbb19397b8ea3d40e054e870
+  data.tar.gz: 51f912518ea9e126de18d8e2199faa451e24afb92cc4b57461bc0ccb89e747c35e1bf8944a76e668a6b2f74eb12481850d1f5c653db4e934749e918f4d4b380d

data/CHANGELOG.md

@@ -2,6 +2,13 @@
 
 ## [Unreleased]
 
+## [0.1.15] - 2026-08-05
+
+### Fixed
+
+- `Extruct.do_extruct` had no timeout on the external `extruct` subprocess (`Open3.capture3`). The content-type and magic-byte checks catch most binary responses, but not all (unreliable/missing headers, formats outside `BINARY_MAGIC_BYTES`, or just very large/slow-to-parse HTML) — a single such request blocked the calling thread indefinitely and leaked an orphaned Python process, since Puma's `worker_timeout` only restarts a worker whose *every* thread has stopped responding, not one with a single permanently-stuck thread among several. Diagnosed on `tests.ostrails.eu` after multiple hours-long outages: load average 40-50, several `python3 extruct <uri>` processes pegged at high CPU for many minutes, and 68 zombie processes. Fixed by prepending the `timeout` coreutil (`-k 5`: escalate to `SIGKILL` 5s after the initial `SIGTERM` if ignored) to the invocation, configurable via `EXTRUCT_TIMEOUT_SECONDS` (default 30s).
+- Same change also closes a latent command-injection hole: the subprocess was previously invoked as a single shell-interpolated string (`"#{ExtructCommand} #{uri}"`), so a crafted `uri` could inject shell metacharacters. It's now passed as separate `argv` elements (`Open3.capture3("timeout", ..., *command_parts, uri)`), which never goes through a shell.
+
 ## [0.1.14] - 2026-06-30
 
 ### Fixed

data/lib/extruct.rb

@@ -8,6 +8,15 @@ module FAIRChampionHarvester
     }ix
     BINARY_MAGIC_BYTES = /\A(%PDF|PK\x03\x04|GIF8|\x89PNG|\xFF\xD8\xFF|\xD0\xCF|\x1F\x8B)/n
 
+    # Hard ceiling, in seconds, on how long the external `extruct` process may
+    # run. The content-type and magic-byte checks above catch most binary
+    # responses, but not all of them (unreliable/missing headers, formats
+    # outside BINARY_MAGIC_BYTES, or just very large/slow-to-parse HTML) —
+    # without this, a single such request blocks the calling thread (and
+    # leaks an orphaned Python process) indefinitely. Configurable via
+    # EXTRUCT_TIMEOUT_SECONDS.
+    TIMEOUT_SECONDS = ENV.fetch("EXTRUCT_TIMEOUT_SECONDS", 30).to_i
+
     def self.do_extruct(meta, uri, content_type: nil, body_prefix: nil)
       if content_type&.match?(BINARY_CONTENT_TYPE)
         meta.comments << "INFO: Skipping extruct for #{uri} — " \
@@ -21,10 +30,25 @@ module FAIRChampionHarvester
       meta.comments << "INFO:  Using 'extruct' to try to extract metadata from return value (message body) of #{uri}.\n"
       warn "begin open3"
       # binding.pry
-      stdout, stderr, status = Open3.capture3(FAIRChampionHarvester::Utils::ExtructCommand + " " + uri)
+      # Pass argv as separate elements (not one interpolated string) so this
+      # never goes through a shell — `uri` can be attacker/publisher-supplied
+      # and must not be shell-interpolated — and prepend the `timeout`
+      # coreutil so a hung/slow extruct process is forcibly killed (-k 5:
+      # escalate to SIGKILL 5s after the initial SIGTERM if it's ignored)
+      # instead of blocking this thread and leaking a child process forever.
+      command_parts = Shellwords.split(FAIRChampionHarvester::Utils::ExtructCommand)
+      stdout, stderr, status = Open3.capture3(
+        "timeout", "-k", "5", TIMEOUT_SECONDS.to_s, *command_parts, uri
+      )
       warn ""
       # sleep 5
       warn "open3 status: #{status} #{stdout}"
+
+      if [124, 137].include?(status.exitstatus)
+        meta.comments << "WARN: extruct timed out after #{TIMEOUT_SECONDS}s parsing #{uri} — skipping.\n"
+        return
+      end
+
       result = stderr # absurd that the output comes over stderr!  LOL!
 
       # result = %x{#{FAIRChampionHarvester::Utils::ExtructCommand} #{uri} 2>&1}

data/lib/fair_champion_harvester/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module FairChampionHarvester
-  VERSION = "0.1.14"
+  VERSION = "0.1.15"
 end

data/lib/fair_champion_harvester.rb

@@ -21,6 +21,7 @@ require "parseconfig"
 require "cgi"
 require "digest"
 require "open3"
+require "shellwords"
 require "require_all"
 # require 'pry'
 require_relative "fair_champion_harvester/version"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fair_champion_harvester
 version: !ruby/object:Gem::Version
-  version: 0.1.14
+  version: 0.1.15
 platform: ruby
 authors:
 - markwilkinson