activestorage-hotcell-server 0.3.1 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 89dcb6743b4bb87dacc1351b9dec256ef7dd1c887caf13c9242c4c4abc05c1de
-  data.tar.gz: 93b579fc94d6670fd6008b58e98c10fba06d50c3b63b8bf691ea6f35ae1526bf
+  metadata.gz: f8f31eda3f64a83828d024483bd31e733927fdf2d7ea0b02796bbef8ead35509
+  data.tar.gz: 0e9975a3262d57ea6caecc8351903297ef6043412ac2bae7ad45a4f5783efaeb
 SHA512:
-  metadata.gz: 999c462832530390ac5cc300a69b2673246e1a99759bdb9a92625af721523ec72ab16988107a1b4049e4a8e41cdb8f7d6612a87b7e33d647e5947a98325d2b78
-  data.tar.gz: c076edca15955f2cd668a43ac54a3c23011f6f8ca935ab4d08a7db7a1623fc2b1d2906d719070a4a09f4ea99a9d75c2356abac236bdc5cb4df746eec1d438b06
+  metadata.gz: '078512cbc8c4bc0559effd79d63f3e5b30b24c308fe5183bc85e054dea4a5829d34b28ebdc43173503893c94babd16e8de91b9217b76229150fd01c633cad575'
+  data.tar.gz: afb813f18f6e85c61389af36c9dc731ca3ff64ac18c24523a0015f8f1f25a5616ebfd307f235e15fbe5038c5fc6d7ebe7776a9d8a44c713630e6ff9b22beff0f

data/lib/active_storage/hot_cell/server/analyzers/image/magick.rb

@@ -25,7 +25,7 @@ module ActiveStorage
 
             def perform(inputs, _outputs)
               source, = inputs
-              frames = identified(source.path)
+              frames = identified(source)
               width, height, orientation = frames.first
 
               { **dimensions(width, height, orientation), pages: frames.size, animated: frames.size > 1,
@@ -36,10 +36,10 @@ module ActiveStorage
               # One `identify` run reports every frame's dimensions and orientation, where MiniMagick::Image's
               # accessors (`valid?`, `width`, `pages`) each spawn an identify of their own — four execs per
               # analysis, two of them decoding every frame.
-              def identified(path)
-                frames = MiniMagick.identify do |identify|
+              def identified(source)
+                frames = MiniMagick.identify(inherit_fds: [ source.to_io ]) do |identify|
                   identify.format "%w %h %[orientation]\n"
-                  identify << path
+                  identify << source.fd_path
                 end.lines.map(&:split)
 
                 raise MiniMagick::Invalid, "ImageMagick does not recognise this as an image" if frames.empty?

data/lib/active_storage/hot_cell/server/magick_operation.rb

@@ -8,29 +8,6 @@ require "active_storage/hot_cell/server/operation"
 # toolchains' loads in the same place is what makes a single cell carrying both legible.
 require "image_processing/mini_magick"
 
-# Invariant 9: a tool sees only the environment its operation wrote for it. `Operation#run_tool` holds it
-# with `unsetenv_others: true`, and these operations do not use it — mini_magick spawns `magick` itself.
-# `MiniMagick.restricted_env` is `false` by default, and with it false mini_magick calls
-# `Open3.popen3({}, *command, unsetenv_others: false)`, so `magick` inherited this worker's whole
-# environment. `bin/conformance` did not catch it, because its environment check drives an operation that
-# goes through `run_tool`.
-#
-# `cli_env` names the locale for the reason `tool_environment` does: a tool's output must not shift under
-# it. mini_magick passes `HOME`, `PATH` and `LANG` and does not pass `LC_ALL`, so both go here.
-#
-# Set at require time rather than in `before_worker_boot`, so that the binary lookup this require performs
-# is covered too. Note what this is: mini_magick filters the environment it was given, where `run_tool`
-# writes a fresh one. Driving `magick` directly would make it ours, and that is the separate enhancement
-# this class already names.
-#
-# The OpenMP variables come from the cell's environment rather than being named here: the number belongs
-# to the image, which is configured to match the container's CPU quota. Without them the restriction would
-# hand `magick` the pool the image's bound was meant to take away. `run_tool` carries the same pair.
-MiniMagick.restricted_env = true
-MiniMagick.cli_env = { "LANG" => "C.UTF-8", "LC_ALL" => "C.UTF-8",
-                       "OMP_NUM_THREADS" => ENV["OMP_NUM_THREADS"],
-                       "OMP_THREAD_LIMIT" => ENV["OMP_THREAD_LIMIT"] }.compact
-
 module ActiveStorage
   module HotCell
     module Server
@@ -41,14 +18,46 @@ module ActiveStorage
       # own output — dimensions from `identify`, an exit status — which is a number and a string, not a
       # decoder. The distinction is the same one probe_media draws for ffprobe.
       #
-      # Unlike the vips operations, the input is staged onto scratch: mini_magick spawns its own `magick` and
-      # does not inherit this worker's descriptors, so an input cannot be handed to it as /dev/fd. That
-      # returns the file_size ceiling to this path — an input larger than the operation's file_size dies
-      # being staged — which the vips operations shed. Removing it means driving `magick` directly rather
-      # than through mini_magick, and is a separate enhancement.
+      # Both operations hand mini_magick the input descriptor with `inherit_fds:` and name it by its
+      # Descriptor#fd_path, so neither stages its input and neither carries the file_size ceiling the vips
+      # operations never had.
       class MagickOperation < Operation
         abstract_operation
 
+        # Invariant 9: a tool sees only the environment its operation wrote for it. `Operation#run_tool` holds
+        # it with `unsetenv_others: true`, and these operations do not use it — mini_magick spawns `magick`
+        # itself. `MiniMagick.restricted_env` is `false` by default, and with it false mini_magick calls
+        # `Open3.popen3({}, *command, unsetenv_others: false)`, so `magick` inherited this worker's whole
+        # environment. `bin/conformance` did not catch it, because its environment check drives an operation
+        # that goes through `run_tool`. Note what this is: mini_magick filters the environment it was given,
+        # where `run_tool` writes a fresh one. Driving `magick` directly would make it ours, and that is the
+        # separate enhancement this class already names.
+        #
+        # The locale is named for the reason `tool_environment` names it: a tool's output must not shift under
+        # it. mini_magick passes `HOME`, `PATH` and `LANG` and does not pass `LC_ALL`, so both go here.
+        #
+        # Everything else comes from the worker's environment rather than being named here. The OpenMP pair
+        # belongs to the image, which is configured to match the container's CPU quota. ImageMagick's own
+        # resource ceilings, `MAGICK_DISK_LIMIT` and its siblings, are read from its environment as it loads
+        # and the image sets them to a worker's share of the scratch; without them `magick` runs under
+        # ImageMagick's defaults, the host's RAM and an unbounded disk. `TMPDIR` and `MAGICK_TMPDIR` are where
+        # it spills that cache: the worker sets the first to the request's home after the fork and
+        # `Operation#initialize` maps the second from it — which is why this is read per request, after
+        # `super`, and not once at require: a hash built then hands `magick` the supervisor's `/tmp`, which is
+        # swept only at the next boot.
+        def self.cli_env
+          { "LANG" => "C.UTF-8", "LC_ALL" => "C.UTF-8",
+            "OMP_NUM_THREADS" => ENV["OMP_NUM_THREADS"], "OMP_THREAD_LIMIT" => ENV["OMP_THREAD_LIMIT"],
+            "TMPDIR" => ENV["TMPDIR"], "MAGICK_TMPDIR" => ENV["MAGICK_TMPDIR"] }
+            .merge(ENV.select { |name, _| name.match?(/\AMAGICK_\w+_LIMIT\z/) })
+            .compact
+        end
+
+        def initialize
+          super
+          MiniMagick.cli_env = self.class.cli_env
+        end
+
         # MiniMagick::Error is how mini_magick reports a `magick` that exited non-zero — the common shape of an
         # input it cannot decode. MiniMagick::Invalid is an input `identify` rejects outright.
         # ImageProcessing::Error is the pipeline's own verdict on the input against the requested
@@ -71,3 +80,7 @@ module ActiveStorage
     end
   end
 end
+
+# Also at require time, so that the binary lookup the require above performs is covered too.
+MiniMagick.restricted_env = true
+MiniMagick.cli_env = ActiveStorage::HotCell::Server::MagickOperation.cli_env

data/lib/active_storage/hot_cell/server/operation.rb

@@ -30,6 +30,15 @@ module ActiveStorage
           "tiff" => "image/tiff",
         }.freeze
 
+        # The cell describes the request's scratch as `TMPDIR`, and knows nothing about the libraries that run
+        # here. ImageMagick reads `MAGICK_TMPDIR` first, so the mapping is this gem's — for the `magick` the
+        # magick operations exec and for the `magickload` libvips delegates to in-process. Per request rather
+        # than at load: `TMPDIR` is the request's home, which is fresh every time.
+        def initialize
+          super
+          ENV["MAGICK_TMPDIR"] = ENV["TMPDIR"] if ENV.key?("TMPDIR")
+        end
+
         private
           # A caller breaking the protocol is a caller bug, not a bad document. Raising MessageError is what
           # makes the cell answer `invalid`, which is permanent and which the client raises rather than

data/lib/active_storage/hot_cell/server/transformers/image/magick.rb

@@ -29,9 +29,17 @@ module ActiveStorage
                 ImageProcessing::MiniMagick
               end
 
-              # The staged path rather than /dev/fd; MagickOperation's comment says why.
               def source_path(source)
-                source.path
+                source.fd_path
+              end
+
+              # `source_path` names the input as the source's `/dev/fd` path, which `magick` can open only if it
+              # inherits the descriptor, so name the IO for mini_magick to put in its spawn map. Through the
+              # loader options rather than by pre-building a MiniMagick::Tool: a pre-built tool reaches
+              # ImageProcessing's `load_image` by its first branch, which drops `page`, `loader` and `geometry`
+              # without a word.
+              def source_loader_options(source)
+                { inherit_fds: [ source.to_io ] }
               end
 
               def describe(path, format)

data/lib/active_storage/hot_cell/server/transforming.rb

@@ -21,23 +21,40 @@ module ActiveStorage
           # it to the library, which cannot reproduce those without restating them. Output#post makes the
           # one remaining copy, out through the caller's descriptor.
           encoded = destination.path(extension: format)
-          pipeline(source_path(source), format, operations).call(destination: encoded)
+          pipeline(source, format, operations).call(destination: encoded)
           destination.adopt encoded
 
           describe destination.path, format
         end
 
         private
-          # `loader(page: 0)` first is what Rails does, and it is what stops a multi-page TIFF or a
-          # hundred-frame GIF being decoded in full to produce one thumbnail. A caller's own `loader` arrives
-          # through `apply` and merges over it, which is also what Rails does — asking for every frame is
-          # `loader: { n: -1 }`.
-          def pipeline(path, format, operations)
+          # Three `loader` calls, and the order is the point: ImageProcessing merges them key by key, so the
+          # last one to write a key wins.
+          #
+          # `page: 0` is first, as a default the caller may replace. It is what Rails does, and what stops a
+          # multi-page TIFF or a hundred-frame GIF being decoded in full to produce one thumbnail; a caller
+          # who wants every frame sends `loader: { n: -1 }` through `apply` and gets it.
+          #
+          # `source_loader_options` is last, so the caller cannot replace it.
+          def pipeline(source, format, operations)
             processor
-              .source(path)
+              .source(source_path(source))
               .loader(page: 0)
               .convert(format)
               .apply(operations_for(operations))
+              .loader(**source_loader_options(source))
+          end
+
+          # What the toolchain needs in order to reach the source at all, which is not the same question as how
+          # the image is decoded. Empty here, because `source_path` normally answers a filename and a filename
+          # needs no help. An includer whose `source_path` names something the tool cannot open on its own
+          # overrides this to supply the rest — `Transformers::Image::Magick` is the one that does.
+          #
+          # It goes last in `pipeline` because it is not the caller's to change: a payload sending
+          # `loader: { inherit_fds: [] }` would otherwise leave the source naming a descriptor the tool has no
+          # way to open.
+          def source_loader_options(_source)
+            {}
           end
 
           # What Rails validates, and no more. `combine_options` is refused because it can never be one

data/lib/active_storage/hot_cell/server/version.rb

@@ -3,7 +3,7 @@
 module ActiveStorage
   module HotCell
     module Server
-      VERSION = "0.3.1"
+      VERSION = "0.4.0"
     end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: activestorage-hotcell-server
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.4.0
 platform: ruby
 authors:
 - Mike Dalessio
@@ -15,42 +15,42 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.3.1
+        version: 0.4.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.3.1
+        version: 0.4.0
 - !ruby/object:Gem::Dependency
   name: image_processing
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: 2.1.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: 2.1.0
 - !ruby/object:Gem::Dependency
   name: mini_magick
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 5.2.0
+        version: 5.4.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 5.2.0
+        version: 5.4.0
 - !ruby/object:Gem::Dependency
   name: ruby-vips
   requirement: !ruby/object:Gem::Requirement
@@ -99,8 +99,8 @@ licenses:
 - MIT
 metadata:
   homepage_uri: https://github.com/basecamp/hotcell
-  source_code_uri: https://github.com/basecamp/hotcell/tree/v0.3.1/activestorage-hotcell-server
-  changelog_uri: https://github.com/basecamp/hotcell/blob/v0.3.1/CHANGELOG.md
+  source_code_uri: https://github.com/basecamp/hotcell/tree/v0.4.0/activestorage-hotcell-server
+  changelog_uri: https://github.com/basecamp/hotcell/blob/v0.4.0/CHANGELOG.md
   bug_tracker_uri: https://github.com/basecamp/hotcell/issues
   rubygems_mfa_required: 'true'
 rdoc_options: []