safe_image 0.1.0 → 0.3.0

data/lib/safe_image/remote.rb

@@ -2,12 +2,15 @@
 
 require "fileutils"
 require "ipaddr"
-require "net/http"
-require "resolv"
 require "tempfile"
 require "time"
 require "tmpdir"
 require "uri"
+# net/http and resolv are required lazily inside the methods that fetch, not at
+# load time: requiring them pulls in resolv, which (Ruby 3.4+) reads
+# /etc/resolv.conf eagerly. The Landlock-sandboxed worker loads this file but
+# never performs remote fetches, and on hosts where /etc/resolv.conf is a
+# symlink into /run the sandbox denies that read and the worker dies at boot.
 
 module SafeImage
   module Remote
@@ -39,10 +42,50 @@ module SafeImage
       "image/avif" => ".avif",
       "image/x-icon" => ".ico",
       "image/vnd.microsoft.icon" => ".ico",
+      "image/jxl" => ".jxl",
       "image/svg+xml" => ".svg"
     }.freeze
 
-    EXTENSIONS = %w[.jpg .jpeg .png .gif .webp .heic .heif .avif .ico .svg].freeze
+    EXTENSIONS = %w[.jpg .jpeg .png .gif .webp .heic .heif .avif .ico .jxl .svg].freeze
+
+    # First-bytes signatures per downloaded extension, checked as soon as the
+    # first SIGNATURE_HEAD_BYTES of the body arrive so an obviously mislabeled
+    # response is dropped without downloading the rest. Each entry lists
+    # alternative candidates; a candidate is a list of [offset, bytes] pairs
+    # that must all match. The check rejects only on a definite mismatch of
+    # every candidate against fully-available bytes, so it can never reject an
+    # image the configured backend could decode — decoders sniff these same
+    # magic bytes to pick a loader. SVG has no usable signature and is exempt.
+    SIGNATURES = {
+      ".jpg" => [[[0, "\xFF\xD8\xFF".b]]],
+      ".jpeg" => [[[0, "\xFF\xD8\xFF".b]]],
+      ".png" => [[[0, "\x89PNG\r\n\x1A\n".b]]],
+      ".gif" => [[[0, "GIF8".b]]],
+      ".webp" => [[[0, "RIFF".b], [8, "WEBP".b]]],
+      ".ico" => [[[0, "\x00\x00\x01\x00".b]]],
+      ".heic" => [[[4, "ftyp".b]]],
+      ".heif" => [[[4, "ftyp".b]]],
+      ".avif" => [[[4, "ftyp".b]]],
+      ".jxl" => [[[0, "\xFF\x0A".b]], [[0, "\x00\x00\x00\x0CJXL \r\n\x87\n".b]]]
+    }.freeze
+
+    SIGNATURE_HEAD_BYTES = 12
+
+    # The metadata helpers probe the partially-downloaded file at these
+    # growing byte thresholds and abort the transfer once the answer is
+    # stable, instead of always downloading up to max_bytes.
+    PREFIX_PROBE_INITIAL_BYTES = 64 * 1024
+    PREFIX_PROBE_GROWTH_FACTOR = 4
+
+    # SVG is excluded from prefix probing: SvgMetadata enforces a total-size
+    # cap (MAX_SVG_BYTES) that probing a prefix would bypass, and remote SVGs
+    # are small enough that downloading them fully costs little.
+    PREFIX_PROBE_EXTENSIONS = (EXTENSIONS - [".svg"]).freeze
+
+    # Sentinel a metadata_fetch block returns when the prefix parsed but its
+    # answer could still change with more data (e.g. "not animated", which a
+    # truncated file can report for an animated one).
+    CONTINUE_DOWNLOAD = Object.new.freeze
 
     BLOCKED_IP_RANGES = [
       # IPv4 special-use / non-public ranges. Default remote fetching is for
@@ -113,7 +156,7 @@ module SafeImage
         )
         file.flush
 
-        ext = extension_for(response.fetch(:uri), response.fetch(:content_type))
+        ext = response.fetch(:ext)
         path = file.path
         if File.extname(path) != ext
           renamed = path.sub(/\.bin\z/, ext)
@@ -132,8 +175,18 @@ module SafeImage
     end
 
     def info(url, max_bytes: DEFAULT_MAX_BYTES, max_redirects: DEFAULT_MAX_REDIRECTS, open_timeout: DEFAULT_OPEN_TIMEOUT, read_timeout: DEFAULT_READ_TIMEOUT, total_timeout: DEFAULT_TOTAL_TIMEOUT, allow_private: false, allowed_ports: DEFAULT_ALLOWED_PORTS, headers: {}, max_pixels: nil, animated: false, orientation: false)
-      fetch(url, max_bytes: max_bytes, max_redirects: max_redirects, open_timeout: open_timeout, read_timeout: read_timeout, total_timeout: total_timeout, allow_private: allow_private, allowed_ports: allowed_ports, headers: headers) do |path|
-        SafeImage.info(path, max_pixels: max_pixels, animated: animated, orientation: orientation)
+      metadata_fetch(url, max_bytes: max_bytes, max_redirects: max_redirects, open_timeout: open_timeout, read_timeout: read_timeout, total_timeout: total_timeout, allow_private: allow_private, allowed_ports: allowed_ports, headers: headers) do |path, eof|
+        result = SafeImage.info(path, max_pixels: max_pixels, animated: animated, orientation: orientation)
+        # A truncated file can undercount frames but never overcount, so
+        # "animated" is final as soon as it is true; "not animated" is only
+        # provable from the complete file. Type, dimensions and orientation
+        # come from the header the successful probe just parsed, so they
+        # cannot change with more data.
+        if animated && result.animated != true && !eof
+          CONTINUE_DOWNLOAD
+        else
+          result
+        end
       end
     end
 
@@ -146,12 +199,97 @@ module SafeImage
     end
 
     def animated?(url, max_bytes: DEFAULT_MAX_BYTES, max_redirects: DEFAULT_MAX_REDIRECTS, open_timeout: DEFAULT_OPEN_TIMEOUT, read_timeout: DEFAULT_READ_TIMEOUT, total_timeout: DEFAULT_TOTAL_TIMEOUT, allow_private: false, allowed_ports: DEFAULT_ALLOWED_PORTS, headers: {}, max_pixels: nil)
+      metadata_fetch(url, max_bytes: max_bytes, max_redirects: max_redirects, open_timeout: open_timeout, read_timeout: read_timeout, total_timeout: total_timeout, allow_private: allow_private, allowed_ports: allowed_ports, headers: headers) do |path, eof|
+        answer = SafeImage.animated?(path, max_pixels: max_pixels)
+        next CONTINUE_DOWNLOAD if !eof && answer != true
+
+        answer
+      end
+    end
+
+    def dominant_color(url, max_bytes: DEFAULT_MAX_BYTES, max_redirects: DEFAULT_MAX_REDIRECTS, open_timeout: DEFAULT_OPEN_TIMEOUT, read_timeout: DEFAULT_READ_TIMEOUT, total_timeout: DEFAULT_TOTAL_TIMEOUT, allow_private: false, allowed_ports: DEFAULT_ALLOWED_PORTS, headers: {}, max_pixels: nil)
       fetch(url, max_bytes: max_bytes, max_redirects: max_redirects, open_timeout: open_timeout, read_timeout: read_timeout, total_timeout: total_timeout, allow_private: allow_private, allowed_ports: allowed_ports, headers: headers) do |path|
-        SafeImage.animated?(path, max_pixels: max_pixels)
+        SafeImage.dominant_color(path, max_pixels: max_pixels)
+      end
+    end
+
+    # Single-GET download that re-attempts the local metadata probe as bytes
+    # arrive (at PREFIX_PROBE_INITIAL_BYTES, then growing by
+    # PREFIX_PROBE_GROWTH_FACTOR) and aborts the transfer as soon as the
+    # block's answer is final. The block receives (path, eof) and must return
+    # CONTINUE_DOWNLOAD while its answer could still change with more data.
+    #
+    # Safety contract: any error from a pre-EOF probe means "not enough bytes
+    # yet" — it is swallowed and the download continues, so the complete file
+    # always gets the last word with exactly the validation and error
+    # behaviour of the full-download path (validate_downloaded_image! plus an
+    # un-rescued final probe). A file that never early-exits is handled
+    # byte-for-byte like Remote.fetch handles it.
+    def metadata_fetch(url, max_bytes:, max_redirects:, open_timeout:, read_timeout:, total_timeout:, allow_private:, allowed_ports:, headers:, &compute)
+      uri = parse_uri(url)
+      started_at = monotonic_time
+
+      Tempfile.create(["safe-image-remote", ".bin"], binmode: true) do |file|
+        original_path = file.path
+        path = original_path
+        ext = nil
+        next_probe_at = PREFIX_PROBE_INITIAL_BYTES
+
+        begin
+          early = catch(:metadata_answer) do
+            request(
+              uri,
+              io: file,
+              max_bytes: max_bytes,
+              max_redirects: max_redirects,
+              open_timeout: open_timeout,
+              read_timeout: read_timeout,
+              total_timeout: total_timeout,
+              started_at: started_at,
+              allow_private: allow_private,
+              allowed_ports: allowed_ports,
+              headers: headers,
+              # The extension is known before the body: give the tempfile its
+              # final name up front so probes dispatch on the right loader.
+              on_headers: ->(response_ext) do
+                ext = response_ext
+                renamed = original_path.sub(/\.bin\z/, ext)
+                FileUtils.mv(original_path, renamed)
+                path = renamed
+              end,
+              on_progress: ->(bytes) do
+                next unless PREFIX_PROBE_EXTENSIONS.include?(ext)
+                next if bytes < next_probe_at
+
+                next_probe_at *= PREFIX_PROBE_GROWTH_FACTOR while bytes >= next_probe_at
+                file.flush
+                answer =
+                  begin
+                    compute.call(path, false)
+                  rescue StandardError
+                    CONTINUE_DOWNLOAD
+                  end
+                throw :metadata_answer, [answer] unless CONTINUE_DOWNLOAD.equal?(answer)
+              end
+            )
+            nil
+          end
+
+          if early
+            early.first
+          else
+            file.flush
+            validate_downloaded_image!(path, ext)
+            compute.call(path, true)
+          end
+        ensure
+          FileUtils.rm_f(path) unless path == original_path
+        end
       end
     end
 
-    def request(uri, io:, max_bytes:, max_redirects:, open_timeout:, read_timeout:, total_timeout:, started_at:, allow_private:, allowed_ports:, headers: {})
+    def request(uri, io:, max_bytes:, max_redirects:, open_timeout:, read_timeout:, total_timeout:, started_at:, allow_private:, allowed_ports:, headers: {}, on_headers: nil, on_progress: nil)
+      require "net/http"
       raise ArgumentError, "too many redirects" if max_redirects < 0
       check_deadline!(started_at, total_timeout)
       ipaddr = validate_uri!(uri, allow_private: allow_private, allowed_ports: allowed_ports)
@@ -170,6 +308,7 @@ module SafeImage
 
       bytes = 0
       content_type = nil
+      ext = nil
 
       http.request(request) do |response|
         check_deadline!(started_at, total_timeout)
@@ -192,25 +331,63 @@ module SafeImage
             started_at: started_at,
             allow_private: allow_private,
             allowed_ports: allowed_ports,
-            headers: redirect_headers(headers, from: uri, to: redirected)
+            headers: redirect_headers(headers, from: uri, to: redirected),
+            on_headers: on_headers,
+            on_progress: on_progress
           )
         when Net::HTTPSuccess
           content_length = response["content-length"].to_i
           raise LimitError, "remote image exceeds #{max_bytes} bytes" if content_length > max_bytes
 
           content_type = response["content-type"].to_s.split(";", 2).first.to_s.downcase
+          # Everything the content-type and extension-agreement checks need is
+          # in the headers: reject unsupported or mismatched responses before
+          # reading a single body byte.
+          ext = extension_for(uri, content_type)
+          on_headers&.call(ext)
+
+          head = "".b
+          head_checked = false
           response.read_body do |chunk|
             check_deadline!(started_at, total_timeout)
             bytes += chunk.bytesize
             raise LimitError, "remote image exceeds #{max_bytes} bytes" if bytes > max_bytes
+            unless head_checked
+              head << chunk
+              if head.bytesize >= SIGNATURE_HEAD_BYTES
+                verify_signature!(ext, head)
+                head_checked = true
+                head = nil
+              end
+            end
             io.write(chunk)
+            on_progress&.call(bytes)
           end
+          verify_signature!(ext, head) if !head_checked && !head.empty?
         else
           raise Error, "remote image request failed: HTTP #{response.code}"
         end
       end
 
-      { uri: uri, content_type: content_type, bytes: bytes }
+      { uri: uri, content_type: content_type, ext: ext, bytes: bytes }
+    end
+
+    # Rejects a body whose first bytes definitively cannot belong to the
+    # format the response claimed. Formats without a fixed signature (SVG)
+    # and bytes not yet downloaded are never grounds for rejection.
+    def verify_signature!(ext, head)
+      candidates = SIGNATURES[ext]
+      return unless candidates
+
+      compatible = candidates.any? do |candidate|
+        candidate.all? do |offset, bytes|
+          slice = head.byteslice(offset, bytes.bytesize).to_s
+          slice.empty? || slice == bytes.byteslice(0, slice.bytesize)
+        end
+      end
+      return if compatible
+
+      raise InvalidImageError, "remote image first bytes do not match #{ext.delete_prefix(".")} signature"
     end
 
     def parse_uri(url)
@@ -228,6 +405,7 @@ module SafeImage
       end
       return nil if allow_private
 
+      require "resolv"
       resolver = Resolv::DNS.new
       resolver.timeouts = [2, 2]
       addresses = resolver.getaddresses(uri.host).map(&:to_s)

data/lib/safe_image/runner.rb

@@ -28,7 +28,15 @@ module SafeImage
     IMAGEMAGICK_POLICY_FILE = File.join(IMAGEMAGICK_POLICY_PATH, "policy.xml").freeze
     BASE_ENV = {
       "PATH" => TRUSTED_PATH,
-      "VIPS_BLOCK_UNTRUSTED" => "1"
+      "VIPS_BLOCK_UNTRUSTED" => "1",
+      # Cap glibc's per-thread malloc arenas. Multithreaded tools (oxipng's
+      # rayon pool, ImageMagick's OpenMP) otherwise reserve an arena per thread
+      # — up to 8x64MB of *address space* per core — which, combined with the
+      # sandbox's RLIMIT_AS memory cap, spuriously fails the tool under
+      # concurrency even though real memory use is tiny. AS counts reservations,
+      # not RSS; bounding arenas is the standard mitigation and costs nothing
+      # for these compute-bound tools.
+      "MALLOC_ARENA_MAX" => "2"
     }.freeze
 
     def run!(argv, timeout: DEFAULT_TIMEOUT, env: {}, sandbox: false, read: [], write: [])
@@ -40,7 +48,7 @@ module SafeImage
       Dir.mktmpdir("safe-image-command-") do |tmpdir|
         child_env = command_env(tmpdir, env)
 
-        if sandbox || SafeImage.sandbox_enabled?
+        if sandbox || SafeImage.sandbox?
           return Sandbox.capture_command!(argv, read: read, write: [*write, tmpdir], timeout: timeout, env: child_env)
         end
 

data/lib/safe_image/sandbox.rb

@@ -16,7 +16,7 @@ module SafeImage
     }.freeze
 
     OPERATIONS = %w[
-      probe thumbnail type size dimensions info orientation optimize resize crop downsize convert convert_to_jpeg fix_orientation
+      probe thumbnail type size dimensions info orientation dominant_color optimize resize crop downsize convert convert_to_jpeg fix_orientation
       convert_favicon_to_png frame_count animated? letter_avatar optimize_image!
       sanitize_svg!
     ].freeze
@@ -50,7 +50,7 @@ module SafeImage
       raise Error, "landlock sandbox requested but the landlock gem is unavailable"
     rescue Landlock::SafeExec::CommandError => e
       raise CommandError.new(
-        "sandboxed command failed",
+        "sandboxed command failed: #{failure_detail(e)}",
         command: argv,
         status: e.status&.exitstatus,
         stdout: e.stdout,
@@ -61,24 +61,34 @@ module SafeImage
     def public_call!(operation, args:, kwargs:)
       operation = operation.to_s
       raise ArgumentError, "unsupported sandbox operation: #{operation}" unless OPERATIONS.include?(operation)
-      result = run_worker!(operation, { args: args, kwargs: kwargs })
+      request = { args: args, kwargs: kwargs }
+      result =
+        if Zygote.enabled?
+          Zygote.call!(operation, request)
+        else
+          run_worker!(operation, request)
+        end
       operation == "type" && result ? result.to_sym : result
     end
 
-    def thumbnail(request)
-      public_call!(
-        :thumbnail,
-        args: [],
-        kwargs: request.merge(execution: :inline)
-      )
-    end
-
     def run_worker!(operation, request)
       operation = operation.to_s
       raise ArgumentError, "unsupported sandbox operation: #{operation}" unless OPERATIONS.include?(operation)
 
       require "landlock"
-      payload = JSON.dump({ operation: operation, request: request })
+      config = SafeImage.config
+      payload = JSON.dump(
+        {
+          operation: operation,
+          # JSON has no symbol type; wrap symbol values so the worker can restore
+          # them (e.g. id_namespace: :standalone must not arrive as the string
+          # "standalone", which resolve_namespace would treat as a real namespace).
+          request: deep_encode_symbols(request),
+          # The worker is a fresh process and must be configured like the
+          # parent — minus landlock, since it already runs inside the sandbox.
+          config: { backend: config.backend, max_pixels: config.max_pixels }
+        }
+      )
       code = <<~'RUBY'
         require "json"
         require "safe_image"
@@ -86,6 +96,8 @@ module SafeImage
         def deep_symbolize(value)
           case value
           when Hash
+            # {"__sym__" => "x"} is a symbol value the parent wrapped for transport.
+            return value[:__sym__].to_sym if value.size == 1 && value[:__sym__].is_a?(String)
             value.each_with_object({}) { |(k, v), h| h[k.to_sym] = deep_symbolize(v) }
           when Array
             value.map { |v| deep_symbolize(v) }
@@ -97,18 +109,23 @@ module SafeImage
         payload = JSON.parse(ARGV.fetch(0), symbolize_names: true)
         operation = payload.fetch(:operation).to_s
         allowed_operations = %w[
-          probe thumbnail type size dimensions info orientation optimize resize crop downsize convert convert_to_jpeg fix_orientation
+          probe thumbnail type size dimensions info orientation dominant_color optimize resize crop downsize convert convert_to_jpeg fix_orientation
           convert_favicon_to_png frame_count animated? letter_avatar optimize_image! sanitize_svg!
         ]
         raise ArgumentError, "unsupported sandbox operation: #{operation}" unless allowed_operations.include?(operation)
 
-        request = payload.fetch(:request)
+        request = deep_symbolize(payload.fetch(:request))
         args = request[:args] || []
-        kwargs = deep_symbolize(request[:kwargs] || {})
+        kwargs = request[:kwargs] || {}
 
-        result = SafeImage.with_sandbox_disabled do
-          SafeImage.__send__(operation, *args, **kwargs)
-        end
+        config = payload.fetch(:config)
+        SafeImage.configure!(
+          backend: config.fetch(:backend).to_sym,
+          landlock: false,
+          max_pixels: config.fetch(:max_pixels)
+        )
+
+        result = SafeImage.__send__(operation, *args, **kwargs)
 
         if defined?(SafeImage::Result) && result.is_a?(SafeImage::Result)
           puts JSON.dump({ __type: "Result", data: result.to_h })
@@ -146,22 +163,13 @@ module SafeImage
           max_output_bytes: 512 * 1024,
           truncate_output: false
         )
-        response = JSON.parse(stdout, symbolize_names: true)
-        if response[:__type] == "Result"
-          data = response.fetch(:data)
-          Result.new(**data)
-        elsif response[:__type] == "Info"
-          data = response.fetch(:data)
-          Info.new(**data)
-        else
-          response[:data]
-        end
+        decode_payload(JSON.parse(stdout, symbolize_names: true))
       end
     rescue LoadError
       raise Error, "landlock sandbox requested but the landlock gem is unavailable"
     rescue Landlock::SafeExec::CommandError => e
       raise CommandError.new(
-        "sandboxed worker failed",
+        "sandboxed worker failed: #{failure_detail(e)}",
         command: [RbConfig.ruby, "-e", "..."],
         status: e.status&.exitstatus,
         stdout: e.stdout,
@@ -169,6 +177,31 @@ module SafeImage
       )
     end
 
+    # Rebuilds a worker's {__type:, data:} JSON reply into the value the
+    # caller would have received inline.
+    def decode_payload(response)
+      case response[:__type]
+      when "Result" then Result.new(**response.fetch(:data))
+      when "Info" then Info.new(**response.fetch(:data))
+      else response[:data]
+      end
+    end
+
+    # JSON cannot represent symbols, so wrap symbol values as {"__sym__" => name}
+    # for the worker's deep_symbolize to restore. Mirrors that decoder.
+    def deep_encode_symbols(value)
+      case value
+      when Symbol
+        { "__sym__" => value.to_s }
+      when Hash
+        value.transform_values { |v| deep_encode_symbols(v) }
+      when Array
+        value.map { |v| deep_encode_symbols(v) }
+      else
+        value
+      end
+    end
+
     def sandbox_paths(request, operation)
       read = []
       write = []
@@ -221,7 +254,18 @@ module SafeImage
       paths << RbConfig::CONFIG["rubyarchdir"]
       paths << RbConfig::CONFIG["sitearchdir"]
       paths << RbConfig::CONFIG["vendorarchdir"]
+      # An --enable-shared Ruby installed outside the default read roots
+      # (e.g. GitHub Actions' /opt/hostedtoolcache builds) keeps libruby in
+      # libdir; without read access the worker dies at dynamic-link time
+      # before any Ruby code runs.
+      paths << RbConfig::CONFIG["libdir"]
       paths << File.dirname(RbConfig.ruby)
+      # Pango/fontconfig need the font directories and configs for the native
+      # letter_avatar text rendering inside the worker.
+      paths << "/etc/fonts"
+      paths << "/usr/share/fonts"
+      paths << "/usr/local/share/fonts"
+      paths << "/var/cache/fontconfig"
       paths
     end
 
@@ -229,6 +273,15 @@ module SafeImage
       paths.flatten.compact.map(&:to_s).reject(&:empty?).select { |path| File.exist?(path) }.uniq
     end
 
+    # Sandbox failures often happen before the child can run any Ruby (e.g. a
+    # denied shared-library read kills it at dynamic-link time); without the
+    # child's stderr in the message they are undiagnosable from a CI log.
+    def failure_detail(error)
+      detail = error.stderr.to_s.strip
+      detail = "exit status #{error.status&.exitstatus.inspect}" if detail.empty?
+      detail[0, 2000]
+    end
+
     def symbolize(hash)
       hash.transform_keys(&:to_sym)
     end